From d9d166e500d95bdb3bc4fd13d63f453676338731 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 28 Nov 2023 17:27:32 -0500 Subject: [PATCH 01/58] Run routes using open api spec --- Cargo.lock | 2 +- .../service/web/storyteller_web/src/routes.rs | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 28ce8804f0..3f9e9f4c93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3792,7 +3792,7 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", "toml", ] diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9b6f920061..489d346d79 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -182,6 +182,9 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> app = add_subscription_routes(app); /* /v1/subscriptions/... */ app = add_voice_designer_routes(app); /* /v1/voice_designer */ + if server_environment == ServerEnvironment::Development { + app = add_weights_routes(app); /* /v1/stubs/... */ + } // ==================== Comments ==================== let mut app = RouteBuilder::from_app(app) @@ -1216,3 +1219,34 @@ fn add_voice_designer_routes (app:App)-> App ) } + +// ==================== Weights ROUTES ==================== +fn add_weights_routes(app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = () + > +{ + app.service( + web + ::scope("/v1/weights") + .route("/upload", web::post().to(upload_weights_handler)) + .service( + web + ::resource("/weight/{weight_token}") + .route(web::get().to(get_weight_details_handler)) + .route(web::post().to(update_weight_metadata_handler)) + .route(web::delete().to(delete_weight_handler)) + ) + .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + .route( + "/by_user_and_type/{username}/{type}", + web::get().to(get_weights_by_user_and_type_handler) + ) + ) +} From 65d6f178bca8e00287072915c05844a2c2afb2ab Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 17:07:18 -0500 Subject: [PATCH 02/58] WIP - weights endpoints need to be tested Update weights endpoints had to be fixed. should be ready to unit test the 3 endpoints before using the list weights --- .../src/queries/model_weights/get_weight.rs | 4 +- .../queries/model_weights/update_weight.rs | 72 +++----- .../src/http_server/endpoints/mod.rs | 3 +- .../endpoints/weights/create_weight.rs | 0 .../endpoints/weights/delete_weight.rs | 166 +++++++++++++++++ .../endpoints/weights/get_weight.rs | 156 ++++++++++++++++ .../endpoints/weights/list_weights_by_user.rs | 0 .../src/http_server/endpoints/weights/mod.rs | 3 + .../endpoints/weights/update_weight.rs | 169 ++++++++++++++++++ .../service/web/storyteller_web/src/routes.rs | 20 ++- 10 files changed, 535 insertions(+), 58 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs index 1b29dfceed..050f4de251 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs @@ -157,7 +157,7 @@ async fn select_include_deleted( "#, weight_token.as_str() ) - .fetch_one(mysql_connection).await + .fetch_one(&mut **mysql_connection).await } async fn select_without_deleted( @@ -207,7 +207,7 @@ async fn select_without_deleted( "#, weight_token.as_str() ) - .fetch_one(mysql_connection).await + .fetch_one(&mut **mysql_connection).await } // RawWeight is the struct that is returned from the database in raw form. diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index ba84497b52..21430a06e5 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -1,56 +1,51 @@ use anyhow::anyhow; use sqlx::MySqlPool; -use log::info; -use sqlx::mysql::MySqlPoolOptions; - use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::model_weights::ModelWeightToken; -use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; - pub struct UpdateWeightArgs<'a> { pub weight_token: &'a ModelWeightToken, pub title: Option<&'a str>, pub maybe_thumbnail_token: Option<&'a str>, - pub description_markdown: &'a str, - pub description_rendered_html: &'a str, - pub creator_set_visibility: &'a Visibility, + pub description_markdown: Option<&'a str>, + pub description_rendered_html: Option<&'a str>, + pub creator_set_visibility: Option<&'a Visibility>, pub mysql_pool: &'a MySqlPool, } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { let mut transaction = args.mysql_pool.begin().await?; - let query_result = sqlx - ::query!( - r#" - UPDATE model_weights - SET - title = ?, - description_markdown = ?, - maybe_thumbnail_token = ?, - description_rendered_html = ?, - creator_set_visibility = ?, - version = version + 1 - WHERE token = ? - LIMIT 1 - "#, - args.title, - args.description_markdown, - args.maybe_thumbnail_token, - args.description_rendered_html, - args.creator_set_visibility.to_str(), - args.weight_token.as_str() - ) - .execute(args.mysql_pool).await; + let query_result = sqlx::query!( + r#" + UPDATE model_weights + SET + title = COALESCE(?, title), + description_markdown = COALESCE(?, description_markdown), + maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token), + description_rendered_html = COALESCE(?, description_rendered_html), + creator_set_visibility = COALESCE(?, creator_set_visibility), + version = version + 1 + WHERE token = ? + LIMIT 1 + "#, + args.title.as_deref().unwrap_or(Some("")), + args.description_markdown.as_deref().unwrap_or(Some("")), + args.maybe_thumbnail_token.as_deref().unwrap_or(Some("")), + args.description_rendered_html.as_deref().unwrap_or(Some("")), + args.creator_set_visibility.as_deref().unwrap_or(Some("")), + args.weight_token.as_str() + ) + .execute(args.mysql_pool).await; transaction.commit().await?; match query_result { Ok(_) => Ok(()), Err(err) => { + transaction.rollback().await?; Err(anyhow!("weights update error: {:?}", err)) } } @@ -59,21 +54,4 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { #[cfg(test)] mod tests { - use tokio; - // Template - use sqlx::mysql::MySqlPoolOptions; - - use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING}; - use errors::AnyhowResult; - #[tokio::test] - async fn test_update_weights() -> AnyhowResult<()> { - let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; - - let pool = MySqlPoolOptions::new() - .max_connections(3) - .connect(&db_connection_string) - .await?; - - Ok(()) - } } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 81af5ee4b2..d4c32ebd1d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -25,4 +25,5 @@ pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; pub mod w2l; -pub mod favorites; \ No newline at end of file +pub mod favorites; +pub mod weights; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs new file mode 100644 index 0000000000..91dfcbe94c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -0,0 +1,166 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, log, warn}; +use enums::by_table::audit_logs::audit_log_entity_action::AuditLogEntityAction::Delete; + +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::delete_weights::{delete_weights_as_user,delete_weights_as_mod,undelete_weights_as_user,undelete_weights_as_mod}; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; + +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + + +#[derive(Deserialize)] +pub struct DeleteWeightRequest { + set_delete: bool, + as_mod: bool, + weight_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DeleteWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteWeightError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteWeightError::NotFound => StatusCode::NOT_FOUND, + DeleteWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn delete_weight_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> + ) -> Result{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(DeleteWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to delete this weight: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + + let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(path.as_mod)); + + let query_result = if path.set_delete { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_weights_as_mod( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + } else { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to undelete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update weight mod approval status DB error: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + Ok(simple_json_success()) + } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs new file mode 100644 index 0000000000..9ca0ed207e --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -0,0 +1,156 @@ +use std::fmt; +use std::sync::Arc; +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +use tokens::tokens::model_weights::ModelWeightToken; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use enums::by_table::model_weights::weights_types::WeightsType; +use enums::by_table::model_weights::weights_category::WeightsCategory; + +#[derive(Serialize, Clone)] +pub struct GetWeightResponse { + success: bool, + weight_token: ModelWeightToken, + title: String, + weights_type: WeightsType, + weights_category: WeightsCategory, + maybe_thumbnail_token: Option, + description_markdown: String, + description_rendered_html: String, + creator_set_visibility: Visibility, + + file_size_bytes: i32, + file_checksum_sha2: String, + + cached_user_ratings_negative_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_total_count: u32, + + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + version: i32, + created_at: DateTime, +} + +#[derive(Deserialize)] +pub struct GetWeightPathInfo { + weight_token: String, +} + +#[derive(Debug)] +pub enum GetWeightError { + NotAuthorized, + NotFound, + ServerError, +} + +impl fmt::Display for GetWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for GetWeightError { + fn status_code(&self) -> StatusCode { + match *self { + GetWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + GetWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetWeightError::NotFound => StatusCode::NOT_FOUND, + } + } +} + +pub async fn get_weight_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(GetWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(GetWeightError::ServerError); + } + }; + + // if the weight is private, only the creator can view it + let is_private = weight.creator_set_visibility == Visibility::Private; + + if is_private && creator_user_token.as_str() != &user_session.user_token { + warn!("user is not allowed to view this weight: {}", user_session.user_token); + return Err(GetWeightError::NotAuthorized); + } + + + let response = GetWeightResponse { + success: true, + weight_token: weight.token, + title: weight.title, + weights_type: weight.weights_type, + weights_category: weight.weights_category, + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + creator_set_visibility: weight.creator_set_visibility, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, + version: weight.version, + created_at: weight.created_at + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetWeightError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs new file mode 100644 index 0000000000..afc73c47a8 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -0,0 +1,3 @@ +pub mod get_weight; +pub mod delete_weight; +pub mod update_weight; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs new file mode 100644 index 0000000000..5b6c301f53 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -0,0 +1,169 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, warn}; + +use enums::common::visibility::Visibility; +use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; +use mysql_queries::queries::model_weights::update_weight::{update_weights, UpdateWeightArgs}; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::server_state::ServerState; + +/// TODO will eventually be polymorphic +#[derive(Deserialize)] +pub struct UpdateWeightRequest { + pub weight_token: String, + pub title: Option, + pub description_markdown: Option, + pub description_rendered_html: Option, + pub weight_type: Option, + pub weight_category: Option, +} + +#[derive(Serialize)] +pub struct UpdateWeightResponse { + pub success: bool, +} + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct UpdateWeightPathInfo { + weight_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum UpdateWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for UpdateWeightError { + fn status_code(&self) -> StatusCode { + match *self { + UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateWeightError::NotFound => StatusCode::NOT_FOUND, + UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UpdateWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn update_weight_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result + { + let my_sql_pool = &server_state.mysql_pool; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UpdateWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + + // TODO wouldn't we want to instead use a function that will query the DB for the user and determine if they are a mod? + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(UpdateWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to edit this weight: {}", user_session.user_token); + return Err(UpdateWeightError::NotAuthorized); + } + + let mut weight_type = None; + let mut weight_category = None; + + if let Some(wt) = request.weight_type.as_deref() { + weight_type = Some(wt.to_string()); + } + + if let Some(wc) = request.weight_category.as_deref() { + weight_category = Some(wc.to_string()); + } + + let mut maybe_mod_user_token = None; + + if is_mod { + maybe_mod_user_token = Some(user_session.user_token.clone()); + } + + let query_result = update_weights( + UpdateWeightArgs { + weight_token: &ModelWeightToken::new(weight_token.clone()), + mysql_pool: &server_state.mysql_pool, + title:request.title, + maybe_thumbnail_token: request.thumbnail_token, + description_markdown: request.description_markdown, + description_rendered_html: request.description_rendered_html, + creator_set_visibility: request.creator_set_visibility, + mysql_pool:my_sql_pool, + } + ).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update Weight DB error: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + }; + + Ok(simple_json_success()) + } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 489d346d79..db590ab3da 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -149,6 +149,10 @@ use crate::http_server::endpoints::w2l::list_user_w2l_templates::list_user_w2l_t use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::weights::get_weight::get_weight_handler; +use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; +use crate::http_server::endpoints::weights::update_weight::update_weight_handler; + pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where B: MessageBody, @@ -1235,18 +1239,18 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - .route("/upload", web::post().to(upload_weights_handler)) + // .route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") - .route(web::get().to(get_weight_details_handler)) - .route(web::post().to(update_weight_metadata_handler)) + .route(web::get().to(get_weight_handler)) + .route(web::post().to(update_weight_handler)) .route(web::delete().to(delete_weight_handler)) ) - .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) - .route( - "/by_user_and_type/{username}/{type}", - web::get().to(get_weights_by_user_and_type_handler) - ) + // .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + // .route( + // "/by_user_and_type/{username}/{type}", + // web::get().to(get_weights_by_user_and_type_handler) + // ) ) } From 32331f9890e8370284072e4226c90ce93edf8773 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 21:46:42 -0500 Subject: [PATCH 03/58] compiles working on list queries --- .../queries/model_weights/update_weight.rs | 19 +- .../endpoints/weights/update_weight.rs | 181 +++++++++--------- 2 files changed, 102 insertions(+), 98 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 21430a06e5..2b1d4a9c0f 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -12,12 +12,19 @@ pub struct UpdateWeightArgs<'a> { pub description_markdown: Option<&'a str>, pub description_rendered_html: Option<&'a str>, pub creator_set_visibility: Option<&'a Visibility>, + pub weight_type: Option, + pub weight_category: Option, pub mysql_pool: &'a MySqlPool, } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { let mut transaction = args.mysql_pool.begin().await?; + let visbility: &str = match args.creator_set_visibility { + Some(visibility) => visibility.to_str(), + None => "", + }; + let query_result = sqlx::query!( r#" UPDATE model_weights @@ -29,13 +36,12 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { creator_set_visibility = COALESCE(?, creator_set_visibility), version = version + 1 WHERE token = ? - LIMIT 1 "#, - args.title.as_deref().unwrap_or(Some("")), - args.description_markdown.as_deref().unwrap_or(Some("")), - args.maybe_thumbnail_token.as_deref().unwrap_or(Some("")), - args.description_rendered_html.as_deref().unwrap_or(Some("")), - args.creator_set_visibility.as_deref().unwrap_or(Some("")), + args.title.as_deref().unwrap_or(""), + args.description_markdown.as_deref().unwrap_or(""), + args.maybe_thumbnail_token.as_deref().unwrap_or(""), + args.description_rendered_html.as_deref().unwrap_or(""), + visbility, args.weight_token.as_str() ) .execute(args.mysql_pool).await; @@ -45,7 +51,6 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { match query_result { Ok(_) => Ok(()), Err(err) => { - transaction.rollback().await?; Err(anyhow!("weights update error: {:?}", err)) } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 5b6c301f53..4ab26cab50 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -1,73 +1,75 @@ use std::fmt; use std::sync::Arc; -use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::{ HttpRequest, HttpResponse, web }; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; -use log::{error, warn}; +use log::{ error, warn }; use enums::common::visibility::Visibility; use http_server_common::response::response_success_helpers::simple_json_success; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; -use mysql_queries::queries::model_weights::update_weight::{update_weights, UpdateWeightArgs}; +use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; -/// TODO will eventually be polymorphic +/// TODO will eventually be polymorphic #[derive(Deserialize)] pub struct UpdateWeightRequest { - pub weight_token: String, - pub title: Option, - pub description_markdown: Option, - pub description_rendered_html: Option, - pub weight_type: Option, - pub weight_category: Option, + pub weight_token: String, + pub title: Option, + pub thumbnail_token: Option, + pub description_markdown: Option, + pub description_rendered_html: Option, + pub weight_type: Option, + pub weight_category: Option, + pub visibility: Option, } #[derive(Serialize)] pub struct UpdateWeightResponse { - pub success: bool, + pub success: bool, } /// For the URL PathInfo #[derive(Deserialize)] pub struct UpdateWeightPathInfo { - weight_token: String, + weight_token: String, } // =============== Error Response =============== #[derive(Debug, Serialize)] pub enum UpdateWeightError { - BadInput(String), - NotFound, - NotAuthorized, - ServerError, + BadInput(String), + NotFound, + NotAuthorized, + ServerError, } impl ResponseError for UpdateWeightError { - fn status_code(&self) -> StatusCode { - match *self { - UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, - UpdateWeightError::NotFound => StatusCode::NOT_FOUND, - UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + fn status_code(&self) -> StatusCode { + match *self { + UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateWeightError::NotFound => StatusCode::NOT_FOUND, + UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } } - } - fn error_response(&self) -> HttpResponse { - serialize_as_json_error(self) - } + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } } // NB: Not using derive_more::Display since Clion doesn't understand it. impl fmt::Display for UpdateWeightError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } } // =============== Handler =============== @@ -76,94 +78,91 @@ pub async fn update_weight_handler( http_request: HttpRequest, path: Path, request: web::Json, - server_state: web::Data>) -> Result - { + server_state: web::Data> +) -> Result { let my_sql_pool = &server_state.mysql_pool; - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool) - .await + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { - warn!("Session checker error: {:?}", e); - UpdateWeightError::ServerError + warn!("Session checker error: {:?}", e); + UpdateWeightError::ServerError })?; - + let user_session = match maybe_user_session { - Some(session) => session, - None => { - warn!("not logged in"); - return Err(UpdateWeightError::NotAuthorized); - } + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateWeightError::NotAuthorized); + } }; - + let weight_token = path.weight_token.clone(); // TODO wouldn't we want to instead use a function that will query the DB for the user and determine if they are a mod? let is_mod = user_session.can_ban_users; - + let weight_lookup_result = get_weight_by_token( - &ModelWeightToken::new(weight_token.clone()), - is_mod, - &server_state.mysql_pool, + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool ).await; - + let weight = match weight_lookup_result { - Ok(Some(weight)) => weight, - Ok(None) => { - warn!("Weight not found: {:?}", weight_token); - return Err(UpdateWeightError::NotFound); - }, - Err(err) => { - warn!("Error looking up weight: {:?}", err); - return Err(UpdateWeightError::ServerError); - } + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(UpdateWeightError::NotFound); + } + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(UpdateWeightError::ServerError); + } }; - + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; - + if !is_creator && !is_mod { - warn!("user is not allowed to edit this weight: {}", user_session.user_token); - return Err(UpdateWeightError::NotAuthorized); + warn!("user is not allowed to edit this weight: {}", user_session.user_token); + return Err(UpdateWeightError::NotAuthorized); } - + let mut weight_type = None; let mut weight_category = None; - + if let Some(wt) = request.weight_type.as_deref() { - weight_type = Some(wt.to_string()); + weight_type = Some(wt.to_string()); } - + if let Some(wc) = request.weight_category.as_deref() { - weight_category = Some(wc.to_string()); + weight_category = Some(wc.to_string()); } - + let mut maybe_mod_user_token = None; - + if is_mod { - maybe_mod_user_token = Some(user_session.user_token.clone()); + maybe_mod_user_token = Some(user_session.user_token.clone()); } - - let query_result = update_weights( - UpdateWeightArgs { + + let query_result = update_weights(UpdateWeightArgs { weight_token: &ModelWeightToken::new(weight_token.clone()), - mysql_pool: &server_state.mysql_pool, - title:request.title, - maybe_thumbnail_token: request.thumbnail_token, - description_markdown: request.description_markdown, - description_rendered_html: request.description_rendered_html, - creator_set_visibility: request.creator_set_visibility, - mysql_pool:my_sql_pool, - } - ).await; - + mysql_pool: &server_state.mysql_pool, + title: request.title.as_deref(), + maybe_thumbnail_token: request.thumbnail_token.as_deref(), + description_markdown: request.description_markdown.as_deref(), + description_rendered_html: request.description_rendered_html.as_deref(), + creator_set_visibility: request.visibility.as_ref(), + weight_type: request.weight_type.as_deref().map(|s| s.to_string()), + weight_category: request.weight_category.as_deref().map(|s| s.to_string()), + }).await; + match query_result { - Ok(_) => {}, - Err(err) => { - warn!("Update Weight DB error: {:?}", err); - return Err(UpdateWeightError::ServerError); - } - }; - + Ok(_) => {} + Err(err) => { + warn!("Update Weight DB error: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + } + Ok(simple_json_success()) - } \ No newline at end of file +} From 347fe685cd31abb363c2d7bbbf85b8505a53a604 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 23:11:47 -0500 Subject: [PATCH 04/58] List list_availible_weights needs testing but is written. --- .../weights/list_availible_weights.rs | 244 ++++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 4 +- 2 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs new file mode 100644 index 0000000000..3e134413d1 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -0,0 +1,244 @@ +use std::sync::Arc; + +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{ DateTime, Utc }; +use log::{ info, warn }; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; +use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct ListAvailibleWeightsQuery { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, +} + +#[derive(Serialize)] +pub struct ListAvailibleWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + pub title: String, + + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub creator_user_token: UserToken, + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + + pub maybe_last_update_user_token: Option, + + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub version: i32, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, +} + +#[derive(Debug)] +pub enum ListWeightError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListWeightError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn list_availible_weights_handler( + http_request: HttpRequest, + query: web::Query, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightError::ServerError + })?; + + let mut is_mod = false; + let user_session = match maybe_user_session { + Some(session) => { + is_mod = session.can_ban_users; + session + } + None => { + info!("not logged in"); + return Err(ListWeightError::NotAuthorized); + } + }; + + let limit = query.limit.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let include_user_hidden = is_mod; + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + + + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; + + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + + // generate parse a response + let response = ListAvailibleWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + + weights_type: WeightsType, + weights_category: WeightsCategory, + + title: String, + + maybe_thumbnail_token: Option, + + description_markdown: String, + description_rendered_html: String, + + creator_user_token: UserToken, + creator_ip_address: String, + creator_set_visibility: Visibility, + + maybe_last_update_user_token: Option, + + original_download_url: Option, + original_filename: Option, + + file_size_bytes: i32, + file_checksum_sha2: String, + + private_bucket_hash: String, + + maybe_private_bucket_prefix: Option, + maybe_private_bucket_extension: Option, + + cached_user_ratings_total_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_negative_count: u32, + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + version: i32, + + created_at: DateTime, + updated_at: DateTime, + + user_deleted_at: Option>, + mod_deleted_at: Option>, + + creator_username: String, + creator_display_name: String, + creator_email_gravatar_hash: String, + }).collect::>(), + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightError::ServerError)?; + + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index afc73c47a8..66601e374d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -1,3 +1,5 @@ pub mod get_weight; pub mod delete_weight; -pub mod update_weight; \ No newline at end of file +pub mod update_weight; +pub mod list_weights_by_user; +pub mod list_availible_weights; \ No newline at end of file From fba7d2a1ebaabc1b7a1d52cc9969e084319e4af1 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 00:48:54 -0500 Subject: [PATCH 05/58] needs some tests for users and query builds with pagination --- .../weights/list_availible_weights.rs | 73 ++++------ .../endpoints/weights/list_weights_by_user.rs | 136 ++++++++++++++++++ 2 files changed, 164 insertions(+), 45 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 3e134413d1..fa64de0d58 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -8,7 +8,12 @@ use log::{ info, warn }; use enums::common::visibility::Visibility; use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; -use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; + +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; +use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; @@ -33,7 +38,6 @@ pub struct ListAvailibleWeightsSuccessResponse { #[derive(Serialize)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, - pub title: String, pub weights_type: WeightsType, @@ -47,11 +51,8 @@ pub struct ModelWeightForList { pub description_rendered_html: String, pub creator_user_token: UserToken, - pub creator_ip_address: String, pub creator_set_visibility: Visibility, - pub maybe_last_update_user_token: Option, - pub file_size_bytes: i32, pub file_checksum_sha2: String, @@ -59,7 +60,6 @@ pub struct ModelWeightForList { pub cached_user_ratings_positive_count: u32, pub cached_user_ratings_negative_count: u32, pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, pub version: i32, @@ -184,53 +184,36 @@ pub async fn list_availible_weights_handler( success: true, weights: weights_page.weights.into_iter() .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - - weights_type: WeightsType, - weights_category: WeightsCategory, + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, - title: String, + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, - maybe_thumbnail_token: Option, + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, - description_markdown: String, - description_rendered_html: String, + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, - creator_user_token: UserToken, - creator_ip_address: String, - creator_set_visibility: Visibility, + - maybe_last_update_user_token: Option, + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - original_download_url: Option, - original_filename: Option, + version: weights.version, - file_size_bytes: i32, - file_checksum_sha2: String, + created_at: weights.created_at, + updated_at: weights.updated_at, - private_bucket_hash: String, - - maybe_private_bucket_prefix: Option, - maybe_private_bucket_extension: Option, - - cached_user_ratings_total_count: u32, - cached_user_ratings_positive_count: u32, - cached_user_ratings_negative_count: u32, - maybe_cached_user_ratings_ratio: Option, - cached_user_ratings_last_updated_at: DateTime, - - version: i32, - - created_at: DateTime, - updated_at: DateTime, - - user_deleted_at: Option>, - mod_deleted_at: Option>, - - creator_username: String, - creator_display_name: String, - creator_email_gravatar_hash: String, + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash }).collect::>(), cursor_next, cursor_previous, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index e69de29bb2..21ac39c7a8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -0,0 +1,136 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, Clone)] +pub struct Weight { + weight_token: ModelWeightToken, + title: String, + + creator: UserDetailsLight, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} + + +#[derive(Serialize)] +pub struct ListWeightsByUserSuccessResponse { + pub success: bool, + pub weights: Vec, +} + +#[derive(Deserialize)] +pub struct ListWeightsByUserPathInfo { + username: String, +} + +#[derive(Debug)] +pub enum ListWeightsByUserError { + NotAuthorized, + ServerError, +} + +impl fmt::Display for ListWeightsByUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightsByUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightsByUserError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightsByUserError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn list_weights_by_user_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightsByUserError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListWeightsByUserError::NotAuthorized); + } + }; + + let username = path.username.as_ref(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let query_results = list_weights_by_creator_username( + &server_state.mysql_pool, + &username, + is_mod, + ).await.map_err(|e| { + warn!("Error querying for weights: {:?}", e); + ListWeightsByUserError::ServerError + }); + let weights = match query_results { + Ok(weights) => weights, + Err(e) => { + warn!("Error querying for weights: {:?}", e); + + + return Err(ListWeightsByUserError::ServerError); + } + }; + + let weights = weights.into_iter().map(|weight| { + Weight { + weight_token: weight.token, + title: weight.title, + creator: UserDetailsLight::from_db_fields( + &weight.creator_user_token, + &weight.creator_username, + &weight.creator_display_name, + &weight.creator_email_gravatar_hash, + ), + creator_set_visibility: weight.creator_set_visibility, + created_at: weight.created_at, + updated_at: weight.updated_at, + } + }).collect(); + + let response = ListWeightsByUserSuccessResponse { + success: true, + weights, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightsByUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file From 72fdd75568cfe551b91351fb530decfd33152c10 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 17:33:25 -0500 Subject: [PATCH 06/58] added upload weights / create weight endpoint original file for the weights weights file type for safetensors or checkpoint Gotta add one more thing for record creation --- crates/_types/buckets/src/public/mod.rs | 3 +- .../src/public/weight_uploads/directory.rs | 77 ++++ .../buckets/src/public/weight_uploads/mod.rs | 2 + .../public/weight_uploads/original_file.rs | 104 ++++++ .../enums/src/by_table/model_weights/mod.rs | 1 + .../model_weights/weights_file_type.rs | 68 ++++ .../tokens/src/tokens/weight_uploads.rs | 14 + .../endpoints/weights/create_weight.rs | 335 ++++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 3 +- 9 files changed, 605 insertions(+), 2 deletions(-) create mode 100644 crates/_types/buckets/src/public/weight_uploads/directory.rs create mode 100644 crates/_types/buckets/src/public/weight_uploads/mod.rs create mode 100644 crates/_types/buckets/src/public/weight_uploads/original_file.rs create mode 100644 crates/_types/enums/src/by_table/model_weights/weights_file_type.rs create mode 100644 crates/_types/tokens/src/tokens/weight_uploads.rs diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index d85de56720..dc35c40ed4 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,4 +2,5 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod zs_voices; \ No newline at end of file +pub mod zs_voices; +pub mod weight_uploads; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/directory.rs b/crates/_types/buckets/src/public/weight_uploads/directory.rs new file mode 100644 index 0000000000..fcedda62dd --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/directory.rs @@ -0,0 +1,77 @@ +use crockford::crockford_entropy_lower; +// TODO Thoughts NOTE: +// I am wondering if it be easier if we just had a single struct and impl to handle downloads from public and private buckets. +// essentially, the only difference is the prefix and file postfix and the path of where the file goes and the path will be constructed from there. +// then you can tie in the downloader as a single object that can be used for both public and private buckets. + +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +const WEIGHT_UPLOAD_DIRECTORY : &str = "/weight_upload"; + +/// Directory for user media uploads. +/// Each uploaded file gets its own directory so that we can store the original +/// file alongside re-processed derivatives. +#[derive(Clone)] +pub struct WeightUploadDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for WeightUploadDirectory {} + +impl WeightUploadDirectory { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", WEIGHT_UPLOAD_DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } +} + +#[cfg(test)] +mod tests { + use crate::public::weight_uploads::directory::WeightUploadDirectory; + + #[test] + pub fn generate_new_entropy() { + let directory = WeightUploadDirectory::generate_new(); + assert_eq!(directory.get_object_hash().len(), 32); + } + + #[test] + pub fn get_directory_path_str() { + let directory = WeightUploadDirectory::from_object_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = WeightUploadDirectory::from_object_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/f/o/foo"); + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = WeightUploadDirectory::from_object_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } +} diff --git a/crates/_types/buckets/src/public/weight_uploads/mod.rs b/crates/_types/buckets/src/public/weight_uploads/mod.rs new file mode 100644 index 0000000000..42ede393cf --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/mod.rs @@ -0,0 +1,2 @@ +pub mod directory; +pub mod original_file; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/original_file.rs new file mode 100644 index 0000000000..7f0a40a6b7 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/original_file.rs @@ -0,0 +1,104 @@ +use std::path::PathBuf; + +use crockford::crockford_entropy_lower; + +use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::public_path::PublicPath; + +const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; + +// TODO: Generate these from a macro. + +/// The original user upload file. +/// It may have derivative files (down samples, crops, etc.) that live alongside it. +#[derive(Clone)] +pub struct MediaUploadOriginalFilePath { + directory: MediaUploadDirectory, + full_object_path: String, +} + +impl PublicPath for MediaUploadOriginalFilePath {} + +impl MediaUploadOriginalFilePath { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let directory = MediaUploadDirectory::from_object_hash(hash); + let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); + Self { + directory, + full_object_path, + } + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } + + pub fn to_full_object_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_object_path) + } + + pub fn get_directory(&self) -> &MediaUploadDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename() -> &'static str { + ORIGINAL_FILE_BASENAME + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + + #[test] + pub fn generate_new_entropy() { + let file = MediaUploadOriginalFilePath::generate_new(); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); + } + + #[test] + pub fn to_full_object_pathbuf() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = MediaUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/original_upload.bin"); + } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = MediaUploadOriginalFilePath::from_object_hash(hash); + assert_eq!(file.get_object_hash(), hash); + } +} diff --git a/crates/_types/enums/src/by_table/model_weights/mod.rs b/crates/_types/enums/src/by_table/model_weights/mod.rs index 5217cf739e..0755131487 100644 --- a/crates/_types/enums/src/by_table/model_weights/mod.rs +++ b/crates/_types/enums/src/by_table/model_weights/mod.rs @@ -1,2 +1,3 @@ pub mod weights_types; pub mod weights_category; +pub mod weights_file_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs new file mode 100644 index 0000000000..a7adeb72cd --- /dev/null +++ b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs @@ -0,0 +1,68 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; +// TODO we will need to scan the checkpoints for malicious code. We can't just trust the file extension. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +pub enum WeightsFileType { + #[serde(rename = "checkpoint")] + Checkpoint, + #[serde(rename = "safetensors")] + SafeTensors, +} + +impl WeightsFileType { + pub fn to_str(&self) -> &'static str { + match self { + Self::Checkpoint => "checkpoint", + Self::SafeTensors => "safetensors", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "checkpoint" => Ok(Self::Checkpoint), + "safetensors" => Ok(Self::SafeTensors), + _ => Err(format!("invalid value: {:?}", value)), + } + } + + pub fn all_variants() -> BTreeSet { + BTreeSet::from([ + Self::Checkpoint, + Self::SafeTensors, + ]) + } +} + +impl_enum_display_and_debug_using_to_str!(WeightsFileType); +impl_mysql_enum_coders!(WeightsFileType); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(WeightsFileType::Checkpoint.to_str(), "checkpoint"); + assert_eq!(WeightsFileType::SafeTensors.to_str(), "safetensors"); + } + + #[test] + fn test_from_str() { + assert_eq!(WeightsFileType::from_str("checkpoint").unwrap(), WeightsFileType::Checkpoint); + assert_eq!(WeightsFileType::from_str("safetensors").unwrap(), WeightsFileType::SafeTensors); + assert!(WeightsFileType::from_str("invalid").is_err()); + } + + #[test] + fn test_all_variants() { + let variants = WeightsFileType::all_variants(); + assert_eq!(variants.len(), 2); + assert!(variants.contains(&WeightsFileType::Checkpoint)); + assert!(variants.contains(&WeightsFileType::SafeTensors)); + } +} \ No newline at end of file diff --git a/crates/_types/tokens/src/tokens/weight_uploads.rs b/crates/_types/tokens/src/tokens/weight_uploads.rs new file mode 100644 index 0000000000..cd7c046fc8 --- /dev/null +++ b/crates/_types/tokens/src/tokens/weight_uploads.rs @@ -0,0 +1,14 @@ +use std::fmt::Debug; + +use serde::Deserialize; +use serde::Serialize; + +use crate::prefixes::TokenPrefix; + +/// The primary key for user weight uploads +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[sqlx(transparent)] +pub struct WeightUploadToken(pub String); + +impl_string_token!(WeightUploadToken); +impl_crockford_generator!(WeightUploadToken, 32usize, TokenPrefix::WeightUpload, CrockfordLower); \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index e69de29bb2..1db5217ba1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -0,0 +1,335 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use once_cell::sync::Lazy; + +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{error, info, warn}; + +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; + +#[derive(Serialize)] +pub struct UploadWeightsSuccessResponse { + pub success: bool, + pub upload_token: MediaUploadToken, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + "binary/octet-stream", + ]) +}); + +pub async fn upload_weights_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, +) -> Result { + + let response = handle_weight_upload( + &http_request, + &server_state, + multipart_payload, + &ALLOWED_MIME_TYPES).await?; + + let weight_upload_token = response.to_media_token(); + + let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { + success: true, + upload_token: weight_upload_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} + + +pub enum SuccessCase { + WeightAlreadyUploaded { + existing_upload_token: WeightUploadToken, + }, + WeightSuccessfullyUploaded { + upload_token: WeightUploadToken, + } +} + +impl SuccessCase { + pub fn to_media_token(self) -> WeightUploadToken { + match self { + SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, + SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, + } + } +} + +pub async fn handle_weight_upload( + http_request: &HttpRequest, + server_state: &web::Data>, + mut multipart_payload: Multipart, + allowed_mimetypes: &HashSet<&'static str>, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + UploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + UploadError::ServerError + })?; + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_weights_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + UploadError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token + .ok_or(UploadError::BadInput("no uuid".to_string()))?; + + let maybe_existing_upload = + get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + .await; + + match maybe_existing_upload { + Err(err) => { + error!("Error checking for previous upload: {:?}", err); + return Err(UploadError::ServerError); + } + Ok(Some(upload)) => { + // File already uploaded and frontend didn't protect us. + return Ok(SuccessCase::WeightAlreadyUploaded { + existing_upload_token: upload.token, + }); + } + Ok(None) => { + // Proceed. + } + } + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(UploadError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); + + let maybe_mimetype = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + let bytes = match upload_weights_request.file_bytes { + None => return Err(UploadError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + UploadError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_upload_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !allowed_mimetypes.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // NB: .aiff (audio/aiff) isn't supported by Symphonia: + // It contains uncompressed PCM-encoded audio similar to wav. + // See: https://github.com/pdeljanov/Symphonia/issues/75 + // NB: The following formats are not supported by Symphonia and + // do not have any open issues filed. They may simply be too old: + // - .wma (audio/x-ms-wma) + // - .avi (video/x-msvideo) + media_upload_type = match mimetype { + // Audio + "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + _ => None, + }; + + + let media_upload_type = match media_upload_type { + Some(m) => m, + None => { + warn!("Invalid mimetype: {:?}", maybe_mimetype); + return Err(UploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) + }, + }; + + // probably don't need this + let media_upload_source = match upload_weights_request.media_source { + MediaSource::Unknown => MediaUploadSource::Unknown, + MediaSource::UserFile => MediaUploadSource::File, + MediaSource::UserDeviceApi => MediaUploadSource::DeviceApi, + }; + + // This is fine for now we just don't expose the path! + let public_upload_path = MediaUploadOriginalFilePath::generate_new(); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + // need to upload based on visibility + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + UploadError::ServerError + })?; + + let token = create_weight(CreateModelWeightsArgs { + token: &MediaUploadToken::generate(), + weights_type: media_upload_type, + weights_category: media_upload_category, + title: upload_weights_request.file_name.as_deref(), + maybe_thumbnail_token: None, + description_markdown: None, + description_rendered_html: None, + creator_user_token: maybe_user_token.as_deref(), + creator_ip_address: &ip_address, + creator_set_visibility, + maybe_last_update_user_token: None, + original_download_url: None, + original_filename: upload_weights_request.file_name.as_deref(), + file_size_bytes: file_size_bytes as i32, + file_checksum_sha2: hash, + private_bucket_hash: private_bucket_hash, + maybe_private_bucket_prefix: None, + maybe_private_bucket_extension: None, + cached_user_ratings_total_count: 0, + cached_user_ratings_positive_count: 0, + cached_user_ratings_negative_count: 0, + maybe_cached_user_ratings_ratio: None, + version: 0, + mysql_pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + UploadError::ServerError + })?; + }).await?; + + + + let (token, record_id) = insert_media_upload(Args{ + uuid_idempotency_token: &uuid_idempotency_token, + media_type: media_upload_type, + media_source: media_upload_source, + maybe_original_filename: upload_weights_request.file_name.as_deref(), + original_file_size_bytes: file_size_bytes as u64, + maybe_original_duration_millis: maybe_duration_millis, + maybe_original_mime_type: maybe_mimetype, + maybe_original_audio_encoding: maybe_codec_name.as_deref(), + maybe_original_video_encoding: None, + maybe_original_frame_width: None, // TODO + maybe_original_frame_height: None, // TODO + checksum_sha2: &hash, + public_upload_path: &public_upload_path, + maybe_extra_file_modification_info: None, + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: None, + creator_ip_address: &ip_address, + creator_set_visibility, + mysql_pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + UploadError::ServerError + })?; + + info!("new media upload id: {} token: {:?}", record_id, &token); + + Ok(SuccessCase::WeightSuccessfullyUploaded { + upload_token: token, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index 66601e374d..b9e2bd05c2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -2,4 +2,5 @@ pub mod get_weight; pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; -pub mod list_availible_weights; \ No newline at end of file +pub mod list_availible_weights; +pub mod create_weight; \ No newline at end of file From 496ccd68f1a267060cf336742ea87d08d53bf435 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 23:05:02 -0500 Subject: [PATCH 07/58] Working on uploads gonna be testing the endpoints soon. Need a ui for the upload page to understand how to write it but it's similar to prior art just need to check that out. --- .../public/weight_uploads/original_file.rs | 39 +++---- .../media_uploads/media_upload_type.rs | 8 ++ .../queries/model_weights/create_weight.rs | 1 + .../endpoints/weights/create_weight.rs | 110 +++++------------- .../drain_multipart_weights_request.rs | 100 ++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 3 +- 6 files changed, 156 insertions(+), 105 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/original_file.rs index 7f0a40a6b7..42f203f849 100644 --- a/crates/_types/buckets/src/public/weight_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/weight_uploads/original_file.rs @@ -2,24 +2,20 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::weight_uploads::directory::WeightUploadDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; -// TODO: Generate these from a macro. - -/// The original user upload file. -/// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] -pub struct MediaUploadOriginalFilePath { - directory: MediaUploadDirectory, +pub struct WeightUploadOriginalFilePath { + directory: WeightUploadDirectory, full_object_path: String, } -impl PublicPath for MediaUploadOriginalFilePath {} +impl PublicPath for WeightUploadOriginalFilePath {} -impl MediaUploadOriginalFilePath { +impl WeightUploadOriginalFilePath { pub fn generate_new() -> Self { let entropy = crockford_entropy_lower(32); @@ -28,7 +24,7 @@ impl MediaUploadOriginalFilePath { pub fn from_object_hash(hash: &str) -> Self { // TODO: Path construction could be cleaner. - let directory = MediaUploadDirectory::from_object_hash(hash); + let directory = WeightUploadDirectory::from_object_hash(hash); let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); Self { directory, @@ -44,7 +40,7 @@ impl MediaUploadOriginalFilePath { PathBuf::from(&self.full_object_path) } - pub fn get_directory(&self) -> &MediaUploadDirectory { + pub fn get_directory(&self) -> &WeightUploadDirectory { &self.directory } @@ -60,37 +56,36 @@ impl MediaUploadOriginalFilePath { #[cfg(test)] mod tests { use std::path::PathBuf; - - use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + use crate::public::weight_uploads::original_file::WeightUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { - let file = MediaUploadOriginalFilePath::generate_new(); + let file = WeightUploadOriginalFilePath::generate_new(); assert_eq!(file.get_object_hash().len(), 32); assert_eq!(file.get_directory().get_object_hash().len(), 32); } #[test] pub fn get_full_object_path_str() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); - assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); } #[test] pub fn to_full_object_pathbuf() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); - assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); } #[test] pub fn get_full_object_path_str_short_name() { - let file = MediaUploadOriginalFilePath::from_object_hash("foo"); - assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/original_upload.bin"); + let file = WeightUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/f/o/foo/original_upload.bin"); } #[test] pub fn get_full_object_path_str_starts_with_directory() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); let full_path = file.get_full_object_path_str(); assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) } @@ -98,7 +93,7 @@ mod tests { #[test] pub fn get_object_hash() { let hash = "abcdefghijk"; - let file = MediaUploadOriginalFilePath::from_object_hash(hash); + let file = WeightUploadOriginalFilePath::from_object_hash(hash); assert_eq!(file.get_object_hash(), hash); } } diff --git a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs index 91f06ed283..42fd07ce41 100644 --- a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs +++ b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs @@ -20,6 +20,9 @@ pub enum MediaUploadType { /// Video files: mp4, etc. Video, + + // Binary files: safetensors ... weights, etc. + Binary, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -33,6 +36,7 @@ impl MediaUploadType { Self::Audio => "audio", Self::Image => "image", Self::Video => "video", + Self::Binary => "binary", } } @@ -41,6 +45,7 @@ impl MediaUploadType { "audio" => Ok(Self::Audio), "image" => Ok(Self::Image), "video" => Ok(Self::Video), + "binary" => Ok(Self::Binary), _ => Err(format!("invalid value: {:?}", value)), } } @@ -56,6 +61,7 @@ mod tests { assert_serialization(MediaUploadType::Audio, "audio"); assert_serialization(MediaUploadType::Image, "image"); assert_serialization(MediaUploadType::Video, "video"); + assert_serialization(MediaUploadType::Binary, "binary"); } #[test] @@ -63,6 +69,7 @@ mod tests { assert_eq!(MediaUploadType::Audio.to_str(), "audio"); assert_eq!(MediaUploadType::Image.to_str(), "image"); assert_eq!(MediaUploadType::Video.to_str(), "video"); + assert_eq!(MediaUploadType::Binary.to_str(), "binary"); } #[test] @@ -70,6 +77,7 @@ mod tests { assert_eq!(MediaUploadType::from_str("audio").unwrap(), MediaUploadType::Audio); assert_eq!(MediaUploadType::from_str("image").unwrap(), MediaUploadType::Image); assert_eq!(MediaUploadType::from_str("video").unwrap(), MediaUploadType::Video); + assert_eq!(MediaUploadType::from_str("binary").unwrap(), MediaUploadType::Binary); assert!(MediaUploadType::from_str("foo").is_err()); } } diff --git a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs index d21f8a789e..4827edd907 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs @@ -7,6 +7,7 @@ use enums::by_table::model_weights::{ use log::warn; use enums::common::visibility::Visibility; use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + pub struct CreateModelWeightsArgs<'a> { pub token: &'a ModelWeightToken, pub weights_type: WeightsType, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 1db5217ba1..6b9194135f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -3,17 +3,19 @@ use std::sync::Arc; use actix_multipart::Multipart; use actix_web::{HttpRequest, HttpResponse, web}; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; use once_cell::sync::Lazy; -use tokens::tokens::media_uploads::MediaUploadToken; +use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use log::{error, info, warn}; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; @@ -23,17 +25,14 @@ use media::decode_basic_audio_info::decode_basic_audio_bytes_info; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; -use tokens::tokens::media_uploads::MediaUploadToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; -use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; -use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; #[derive(Serialize)] pub struct UploadWeightsSuccessResponse { pub success: bool, - pub upload_token: MediaUploadToken, + pub upload_token: ModelWeightToken, } static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { @@ -53,8 +52,10 @@ pub async fn upload_weights_handler( &server_state, multipart_payload, &ALLOWED_MIME_TYPES).await?; + + // send a body with the required payload along side the multipart upload - let weight_upload_token = response.to_media_token(); + let weight_upload_token = response.to_model_token(); let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { success: true, @@ -69,18 +70,17 @@ pub async fn upload_weights_handler( .body(body)); } - pub enum SuccessCase { WeightAlreadyUploaded { - existing_upload_token: WeightUploadToken, + existing_upload_token: ModelWeightToken, }, WeightSuccessfullyUploaded { - upload_token: WeightUploadToken, + upload_token: ModelWeightToken, } } impl SuccessCase { - pub fn to_media_token(self) -> WeightUploadToken { + pub fn to_model_token(self) -> ModelWeightToken { match self { SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, @@ -190,6 +190,7 @@ pub async fn handle_weight_upload( .as_ref() .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) .flatten(); + let bytes = match upload_weights_request.file_bytes { None => return Err(UploadError::BadInput("missing file contents".to_string())), @@ -219,38 +220,14 @@ pub async fn handle_weight_upload( .collect::(); return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); } - - // NB: .aiff (audio/aiff) isn't supported by Symphonia: - // It contains uncompressed PCM-encoded audio similar to wav. - // See: https://github.com/pdeljanov/Symphonia/issues/75 - // NB: The following formats are not supported by Symphonia and - // do not have any open issues filed. They may simply be too old: - // - .wma (audio/x-ms-wma) - // - .avi (video/x-msvideo) media_upload_type = match mimetype { - // Audio - "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), _ => None, }; - - - let media_upload_type = match media_upload_type { - Some(m) => m, - None => { - warn!("Invalid mimetype: {:?}", maybe_mimetype); - return Err(UploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) - }, - }; - - // probably don't need this - let media_upload_source = match upload_weights_request.media_source { - MediaSource::Unknown => MediaUploadSource::Unknown, - MediaSource::UserFile => MediaUploadSource::File, - MediaSource::UserDeviceApi => MediaUploadSource::DeviceApi, - }; + } // This is fine for now we just don't expose the path! - let public_upload_path = MediaUploadOriginalFilePath::generate_new(); + let public_upload_path = WeightUploadOriginalFilePath::generate_new(); info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); @@ -266,18 +243,18 @@ pub async fn handle_weight_upload( })?; let token = create_weight(CreateModelWeightsArgs { - token: &MediaUploadToken::generate(), - weights_type: media_upload_type, - weights_category: media_upload_category, + token: &ModelWeightToken::generate(), + weights_type: WeightsType::LoRA, // to do parse multi part for this + weights_category: WeightsCategory::ImageGeneration, title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: None, - description_markdown: None, - description_rendered_html: None, - creator_user_token: maybe_user_token.as_deref(), + maybe_thumbnail_token: &ThumbnailToken::generate(), + description_markdown: "", + description_rendered_html: "", + creator_user_token: maybe_user_token.as_str(), creator_ip_address: &ip_address, creator_set_visibility, - maybe_last_update_user_token: None, - original_download_url: None, + maybe_last_update_user_token: "", + original_download_url: "", original_filename: upload_weights_request.file_name.as_deref(), file_size_bytes: file_size_bytes as i32, file_checksum_sha2: hash, @@ -290,44 +267,13 @@ pub async fn handle_weight_upload( maybe_cached_user_ratings_ratio: None, version: 0, mysql_pool: &server_state.mysql_pool, - }) - .await - .map_err(|err| { - warn!("New generic download creation DB error: {:?}", err); - UploadError::ServerError - })?; - }).await?; - - - - let (token, record_id) = insert_media_upload(Args{ - uuid_idempotency_token: &uuid_idempotency_token, - media_type: media_upload_type, - media_source: media_upload_source, - maybe_original_filename: upload_weights_request.file_name.as_deref(), - original_file_size_bytes: file_size_bytes as u64, - maybe_original_duration_millis: maybe_duration_millis, - maybe_original_mime_type: maybe_mimetype, - maybe_original_audio_encoding: maybe_codec_name.as_deref(), - maybe_original_video_encoding: None, - maybe_original_frame_width: None, // TODO - maybe_original_frame_height: None, // TODO - checksum_sha2: &hash, - public_upload_path: &public_upload_path, - maybe_extra_file_modification_info: None, - maybe_creator_user_token: maybe_user_token.as_ref(), - maybe_creator_anonymous_visitor_token: None, - creator_ip_address: &ip_address, - creator_set_visibility, - mysql_pool: &server_state.mysql_pool, - }) - .await - .map_err(|err| { - warn!("New generic download creation DB error: {:?}", err); + }).await + .map_err(|e| { + error!("Error creating weights upload: {:?}", e); UploadError::ServerError })?; - info!("new media upload id: {} token: {:?}", record_id, &token); + info!("new weights upload id: {} token: {:?}", record_id, &token); Ok(SuccessCase::WeightSuccessfullyUploaded { upload_token: token, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs new file mode 100644 index 0000000000..c87a9d82c5 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs @@ -0,0 +1,100 @@ +use actix_multipart::Multipart; +use actix_web::web::BytesMut; +use futures::TryStreamExt; +use log::warn; + +use errors::AnyhowResult; +use tokens::tokens::weights::ModelWeightToken; + +use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; + +pub struct WeightUploadRequest { + pub uuid_idempotency_token: Option, + pub model_weights_token: Option, + pub file_name: Option, + pub file_bytes: Option, + pub media_source: MediaSource, +} + +/// Where the frontend tells us the file came from. +#[derive(Eq, PartialEq, Clone, Copy)] +pub enum MediaSource { + Unknown, + UserFile, + /// Eg. the web audio API in Javascript. + UserDeviceApi, +} + +/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. +pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { + let mut uuid_idempotency_token = None; + let mut model_weight_token = None; + let mut file_bytes = None; + let mut file_name = None; + let mut media_source = None; + + while let Ok(Some(mut field)) = multipart_payload.try_next().await { + let mut field_name = None; + let mut field_filename = None; + + if let content_disposition = field.content_disposition() { + field_name = content_disposition.get_name() + .map(|s| s.to_string()); + field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. + .map(|s| s.to_string()); + } + + match field_name.as_deref() { + Some("uuid_idempotency_token") => { + uuid_idempotency_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading idempotency token: {:}", e); + e + })?; + }, + Some("dataset_token") => { + model_weight_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading dataset token: {:}", e); + e + })?; + }, + Some("file") => { + file_name = field_filename.clone(); + file_bytes = checked_read_multipart_bytes(&mut field).await + .map_err(|e| { + warn!("Error reading audio upload: {:}", e); + e + })?; + }, + Some("source") => { + media_source = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading source: {:}", e); + e + })?; + }, + _ => continue, + } + } + + let media_source = match media_source.as_deref() { + Some("device") => MediaSource::UserDeviceApi, + Some("file") => MediaSource::UserFile, + _ => { + if file_name.as_deref() == Some("blob") { + MediaSource::UserDeviceApi + } else { + MediaSource::Unknown + } + }, + }; + + Ok(UploadSampleRequest { + uuid_idempotency_token, + dataset_token, + file_name, + file_bytes, + media_source, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index b9e2bd05c2..ba1f88eb2a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -3,4 +3,5 @@ pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; pub mod list_availible_weights; -pub mod create_weight; \ No newline at end of file +pub mod create_weight; +pub mod drain_multipart_weights_request; \ No newline at end of file From 1d937ab98133d87f235153c3d2d88c9407f24c71 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 1 Dec 2023 15:22:32 -0500 Subject: [PATCH 08/58] Initial weights_endpoints need testing right now. --- crates/service/web/storyteller_web/src/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index db590ab3da..015192966c 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1239,7 +1239,7 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - // .route("/upload", web::post().to(upload_weights_handler)) + // .route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") From 8072d135962b52505cadf4b17e94b77ad2446355 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Sun, 3 Dec 2023 01:35:04 -0500 Subject: [PATCH 09/58] Fixed Delete and Updates for Weights --- .../queries/model_weights/update_weight.rs | 16 ++++++---------- .../endpoints/weights/delete_weight.rs | 17 +++++++++++------ .../src/http_server/endpoints/weights/mod.rs | 4 ++-- .../endpoints/weights/update_weight.rs | 19 +------------------ .../service/web/storyteller_web/src/routes.rs | 6 ++++-- 5 files changed, 24 insertions(+), 38 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 2b1d4a9c0f..0d15d4e38f 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -18,12 +18,8 @@ pub struct UpdateWeightArgs<'a> { } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; - let visbility: &str = match args.creator_set_visibility { - Some(visibility) => visibility.to_str(), - None => "", - }; let query_result = sqlx::query!( r#" @@ -37,11 +33,11 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { version = version + 1 WHERE token = ? "#, - args.title.as_deref().unwrap_or(""), - args.description_markdown.as_deref().unwrap_or(""), - args.maybe_thumbnail_token.as_deref().unwrap_or(""), - args.description_rendered_html.as_deref().unwrap_or(""), - visbility, + args.title.as_deref(), + args.description_markdown.as_deref(), + args.maybe_thumbnail_token.as_deref(), + args.description_rendered_html.as_deref(), + args.creator_set_visibility.as_deref(), args.weight_token.as_str() ) .execute(args.mysql_pool).await; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index 91dfcbe94c..f8f547b3b3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -23,11 +23,15 @@ use enums::common::visibility::Visibility; use tokens::tokens::users::UserToken; +#[derive(Deserialize)] +pub struct DeleteWeightPathInfo { + weight_token: String, +} + #[derive(Deserialize)] pub struct DeleteWeightRequest { set_delete: bool, - as_mod: bool, - weight_token: String, + as_mod: bool } // =============== Error Response =============== @@ -64,7 +68,8 @@ impl fmt::Display for DeleteWeightError { pub async fn delete_weight_handler( http_request: HttpRequest, - path: Path, + path: Path, + request: web::Json, server_state: web::Data> ) -> Result{ let maybe_user_session = server_state @@ -112,9 +117,9 @@ pub async fn delete_weight_handler( return Err(DeleteWeightError::NotAuthorized); } - let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(path.as_mod)); - - let query_result = if path.set_delete { + let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(request.as_mod)); + + let query_result = if request.set_delete { match delete_role { DeleteRole::ErrorDoNotDelete => { warn!("user is not allowed to delete weights: {}", user_session.user_token); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index ba1f88eb2a..f457767f25 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -3,5 +3,5 @@ pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; pub mod list_availible_weights; -pub mod create_weight; -pub mod drain_multipart_weights_request; \ No newline at end of file +// pub mod create_weight; +// pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 4ab26cab50..9f159ab417 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -19,7 +19,6 @@ use crate::server_state::ServerState; /// TODO will eventually be polymorphic #[derive(Deserialize)] pub struct UpdateWeightRequest { - pub weight_token: String, pub title: Option, pub thumbnail_token: Option, pub description_markdown: Option, @@ -127,25 +126,9 @@ pub async fn update_weight_handler( return Err(UpdateWeightError::NotAuthorized); } - let mut weight_type = None; - let mut weight_category = None; - - if let Some(wt) = request.weight_type.as_deref() { - weight_type = Some(wt.to_string()); - } - - if let Some(wc) = request.weight_category.as_deref() { - weight_category = Some(wc.to_string()); - } - - let mut maybe_mod_user_token = None; - - if is_mod { - maybe_mod_user_token = Some(user_session.user_token.clone()); - } let query_result = update_weights(UpdateWeightArgs { - weight_token: &ModelWeightToken::new(weight_token.clone()), + weight_token: &ModelWeightToken::new(path.weight_token.clone()), mysql_pool: &server_state.mysql_pool, title: request.title.as_deref(), maybe_thumbnail_token: request.thumbnail_token.as_deref(), diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 015192966c..4415eb75e8 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -152,6 +152,9 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; +use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; + pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where @@ -1223,7 +1226,6 @@ fn add_voice_designer_routes (app:App)-> App ) } - // ==================== Weights ROUTES ==================== fn add_weights_routes(app: App) -> App where @@ -1239,7 +1241,7 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - // .route("/upload", web::post().to(upload_weights_handler)) + //.route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") From 9c8ee8dbbbd3d7a3230e55dcaf7b45228f644cd2 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Sun, 3 Dec 2023 22:55:56 -0500 Subject: [PATCH 10/58] Ok list_weights_by_user works properly now. --- .../model_weights/list_weights_by_user.rs | 6 +- .../src/queries/model_weights/tests.rs | 4 ++ .../endpoints/weights/list_weights_by_user.rs | 56 +++++++++++++++---- .../service/web/storyteller_web/src/routes.rs | 4 +- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs index 49d5e52155..e0ef62baee 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs @@ -77,11 +77,7 @@ pub async fn list_weights_by_creator_username( let raw_weights: Vec = get_raw_weights_by_creator_username(&mut connection, creator_username, can_see_deleted).await?; let weights_records: Vec = map_to_weights(raw_weights).await; - let filtered_weights: Vec = weights_records.into_iter().filter(|weight| { - weight.creator_username == creator_username && weight.creator_set_visibility == Visibility::Public && can_see_deleted - }).collect(); - - Ok(filtered_weights) + Ok(weights_records) } async fn get_raw_weights_by_creator_username( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index dca6d0cab6..9b054e4b30 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -292,6 +292,10 @@ mod tests { seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,5,&creator_username).await?; let weights_by_username = list_weights_by_creator_username(&pool, &creator_username, can_see_deleted).await?; + for weight in weights_by_username.iter() { + // print weight + println!("weight: {:?}", weight.creator_username); + } assert_eq!(weights_by_username.len(), 5); Ok(()) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index 21ac39c7a8..c478ea68d9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use log::warn; use enums::common::visibility::Visibility; -use mysql_queries::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; +use mysql_queries::queries::model_weights::list_weights_by_user::{list_weights_by_creator_username, WeightsJoinUserRecord}; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; @@ -19,11 +19,23 @@ use crate::server_state::ServerState; pub struct Weight { weight_token: ModelWeightToken, title: String, - + creator: UserDetailsLight, - creator_set_visibility: Visibility, + maybe_thumbnail_token: Option, + + description_markdown: String, + description_rendered_html: String, + + file_size_bytes: i32, + file_checksum_sha2: String, + cached_user_ratings_total_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_negative_count: u32, + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + created_at: DateTime, updated_at: DateTime, } @@ -96,17 +108,16 @@ pub async fn list_weights_by_user_handler( warn!("Error querying for weights: {:?}", e); ListWeightsByUserError::ServerError }); + let weights = match query_results { Ok(weights) => weights, Err(e) => { warn!("Error querying for weights: {:?}", e); - - return Err(ListWeightsByUserError::ServerError); } }; - - let weights = weights.into_iter().map(|weight| { + + let weights:Vec = weights.into_iter().map(|weight| { Weight { weight_token: weight.token, title: weight.title, @@ -116,16 +127,41 @@ pub async fn list_weights_by_user_handler( &weight.creator_display_name, &weight.creator_email_gravatar_hash, ), + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, creator_set_visibility: weight.creator_set_visibility, created_at: weight.created_at, updated_at: weight.updated_at, } }).collect(); - - let response = ListWeightsByUserSuccessResponse { + + let final_weights:Vec; + + // if it's not the user ... then only show public weights else show private and public + if (creator_user_token != user_session.user_token) { + final_weights = weights.into_iter().filter(|weight| { + weight.creator_set_visibility == Visibility::Public + }).collect(); + + } + else { + final_weights = weights; + } + + + let response: ListWeightsByUserSuccessResponse = ListWeightsByUserSuccessResponse { success: true, - weights, + weights: final_weights, }; + let body = serde_json::to_string(&response) .map_err(|e| ListWeightsByUserError::ServerError)?; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4415eb75e8..4f05c00c9a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1249,10 +1249,12 @@ fn add_weights_routes(app: App) -> App .route(web::post().to(update_weight_handler)) .route(web::delete().to(delete_weight_handler)) ) - // .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) // .route( // "/by_user_and_type/{username}/{type}", // web::get().to(get_weights_by_user_and_type_handler) // ) ) } +// use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +// use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; \ No newline at end of file From e4756a4f2c2c26bd05773be15c8e426b8931ba74 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 01:52:14 -0500 Subject: [PATCH 11/58] Added more data seeding to the database for the model_weights table. --- .../src/seeding/model_weights.rs | 89 ++++++++++++++++++- .../endpoints/weights/update_weight.rs | 1 - .../service/web/storyteller_web/src/routes.rs | 10 +-- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index de9bcd90a9..07c8f2f155 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -1,10 +1,13 @@ +use std::vec; + +use buckets::public::media_files::original_file; use log::info; use tokens::tokens::users::UserToken; use sqlx::{ MySql, Pool }; use enums::by_table::model_weights::{ - weights_types::WeightsType, - weights_category::WeightsCategory, + weights_types::{WeightsType, self}, + weights_category::{WeightsCategory, self}, }; use errors::{ anyhow, AnyhowResult }; @@ -15,6 +18,86 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +pub async fn seed_weights_for_paging( + mysql_pool: &Pool) -> AnyhowResult<()> +{ + + let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { + None => { + return Err(anyhow!("could not find user hanashi")); + } + Some(token) => token, + }; + + // create a loop that loops from 1 to 100 + for i in 1..=100 { + // create a new weight + let model_weight_token = ModelWeightToken(format!("item:{}", i)); + let title = format!("Title {}", i); + let description = format!("Description {}", i); + let description_rendered_html = format!("

{}

", description); + let original_filename = format!("filename{}.txt", i); + let original_download_url = format!("http://example.com/download{}", i); + let thumbnail_token = format!("Thumbnail {}", i); + let private_bucket_hash = format!("bucket_hash{}", i); + let private_bucket_prefix = format!("_fake{}", i); + let private_bucket_extension = format!("rvc{}", i); + let cached_user_ratings_total_count = i; + let cached_user_ratings_positive_count = i; + let cached_user_ratings_negative_count = i; + let cached_user_ratings_ratio = i as u32 / 100; + let version = i as i32; + + let weights_category = match i { + 1..=20 => WeightsCategory::VoiceConversion, + 21..=40 => WeightsCategory::TextToSpeech, + 41..=60 => WeightsCategory::ImageGeneration, + 61..=80 => WeightsCategory::ImageGeneration, + 81..=100 => WeightsCategory::ImageGeneration, + _ => WeightsCategory::ImageGeneration + }; + + let weights_types = match i { + 1..=20 => WeightsType::RvcV2, + 21..=40 => WeightsType::HifiganTacotron2, + 41..=60 => WeightsType::StableDiffusion15, + 61..=80 => WeightsType::StableDiffusionXL, + 81..=100 => WeightsType::LoRA, + _ => WeightsType::LoRA + }; + + let args = CreateModelWeightsArgs { + token: &model_weight_token, // replace with actual ModelWeightToken + weights_type: weights_types, // replace with actual WeightsType + weights_category: weights_category, // replace with actual WeightsCategory + title: title, + maybe_thumbnail_token: Some(thumbnail_token), + description_markdown: description, + description_rendered_html: description_rendered_html, + creator_user_token: Some(&user_token), // replace with actual UserToken + creator_ip_address: "192.168.1.1", + creator_set_visibility: Visibility::Public, + maybe_last_update_user_token: Some(user_token.to_string()), + original_download_url: Some(original_download_url), + original_filename: Some(original_filename), + file_size_bytes: 1024, + file_checksum_sha2: "checksum1".to_string(), + private_bucket_hash: "bucket_hash1".to_string(), + maybe_private_bucket_prefix: Some("_fake".to_string()), + maybe_private_bucket_extension: Some("rvc".to_string()), + cached_user_ratings_total_count: cached_user_ratings_total_count, + cached_user_ratings_positive_count: cached_user_ratings_positive_count, + cached_user_ratings_negative_count: cached_user_ratings_ratio, + maybe_cached_user_ratings_ratio: Some(0.9), + version: version, + mysql_pool: &mysql_pool, // replace with actual MySqlPool + }; + + create_weight(args).await?; + } + Ok(()) + +} pub async fn seed_weights_for_user_token( mysql_pool: &Pool, user_token: UserToken @@ -330,6 +413,6 @@ pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { // adds this to HANASHI get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await?; seed_weights_for_user_token(mysql_pool, user_token).await?; - + seed_weights_for_paging(mysql_pool).await?; Ok(()) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 9f159ab417..0e287729a3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -126,7 +126,6 @@ pub async fn update_weight_handler( return Err(UpdateWeightError::NotAuthorized); } - let query_result = update_weights(UpdateWeightArgs { weight_token: &ModelWeightToken::new(path.weight_token.clone()), mysql_pool: &server_state.mysql_pool, diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4f05c00c9a..6c06107519 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1250,11 +1250,9 @@ fn add_weights_routes(app: App) -> App .route(web::delete().to(delete_weight_handler)) ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) - // .route( - // "/by_user_and_type/{username}/{type}", - // web::get().to(get_weights_by_user_and_type_handler) - // ) + .route( + "/by_user_and_type/{username}/{type}", + web::get().to(list_availible_weights_handler) + ) ) } -// use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; -// use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; \ No newline at end of file From 241f648baf724e8cc976bdd33dc6c5010e5490f8 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 03:38:51 -0500 Subject: [PATCH 12/58] Add type filtering in there and category filtering --- .../list_weights_query_builder.rs | 8 +- .../weights/list_availible_weights.rs | 205 +++++++++++------- 2 files changed, 125 insertions(+), 88 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs index 69ccf3617a..e1d79b89ab 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs @@ -133,13 +133,13 @@ impl ListWeightsQueryBuilder { self } - pub fn weights_type(mut self, weights_type: WeightsType) -> Self { - self.weights_type = Some(weights_type); + pub fn weights_type(mut self, weights_type: Option) -> Self { + self.weights_type = weights_type; self } - pub fn weights_category(mut self, weights_category: WeightsCategory) -> Self { - self.weights_category = Some(weights_category); + pub fn weights_category(mut self, weights_category: Option) -> Self { + self.weights_category = weights_category; self } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index fa64de0d58..26257fc60a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -35,11 +35,17 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } +#[derive(Deserialize)] +pub struct ListWeightsByPathInfo { + pub username: String, + pub weights_type: Option, + pub weights_category: Option, +} + #[derive(Serialize)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, - pub weights_type: WeightsType, pub weights_category: WeightsCategory, @@ -73,31 +79,31 @@ pub struct ModelWeightForList { #[derive(Debug)] pub enum ListWeightError { - NotAuthorized, - ServerError, + NotAuthorized, + ServerError, } impl std::fmt::Display for ListWeightError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } } impl ResponseError for ListWeightError { - fn status_code(&self) -> StatusCode { - match *self { - ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } } - } } pub async fn list_availible_weights_handler( http_request: HttpRequest, + path: web::Path, query: web::Query, server_state: web::Data> ) -> Result { - let maybe_user_session = server_state.session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { @@ -123,103 +129,134 @@ pub async fn list_availible_weights_handler( let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) .map_err(|e| { warn!("crypto error: {:?}", e); ListWeightError::ServerError - })?; - Some(cursor) + })?; + Some(cursor) } else { - None + None }; - let include_user_hidden = is_mod; + let include_user_hidden = is_mod; - let mut query_builder = ListWeightsQueryBuilder::new() - .sort_ascending(sort_ascending) - .scope_creator_username(None) - .include_user_hidden(include_user_hidden) - .include_user_deleted_results(is_mod) - .include_mod_deleted_results(is_mod) - .limit(limit) - .cursor_is_reversed(cursor_is_reversed) - .offset(cursor); + let mut weights_category: Option = None; + let mut weights_type: Option = None; - let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + if let Some(weights_category_string) = path.weights_category.as_ref() { + let result = WeightsCategory::from_str(&weights_category_string); + match result { + Ok(category) => { + weights_category = Some(category) + }, + Err(e) => { + warn!("invalid weights_category: {:?}", e); + weights_category = None + } + } + } + + if let Some(weights_type_string) = path.weights_type.as_ref() { + let result = WeightsType::from_str(&weights_type_string); + match result { + Ok(wtype) => { + weights_type = Some(wtype) + }, + Err(e) => { + warn!("invalid weights_type: {:?}", e); + weights_type = None + } + } + } + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .weights_category(weights_category) + .weights_type(weights_type) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - let weights_page = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListWeightError::ServerError); - } - }; + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; - let cursor_next = if let Some(id) = weights_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; - let cursor_previous = if let Some(id) = weights_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; - // generate parse a response - let response = ListAvailibleWeightsSuccessResponse { - success: true, - weights: weights_page.weights.into_iter() - .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - weights_type: weights.weights_type, - weights_category: weights.weights_category, + // generate parse a response + let response = ListAvailibleWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, - maybe_thumbnail_token:weights.maybe_thumbnail_token, - description_markdown: weights.description_markdown, - description_rendered_html: weights.description_rendered_html, + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, - creator_user_token: weights.creator_user_token, - creator_set_visibility: weights.creator_set_visibility, + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, - file_size_bytes:weights.file_size_bytes, - file_checksum_sha2: weights.file_checksum_sha2, + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, - cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, - cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, - maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - version: weights.version, + version: weights.version, - created_at: weights.created_at, - updated_at: weights.updated_at, + created_at: weights.created_at, + updated_at: weights.updated_at, - creator_username: weights.creator_username, - creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash - }).collect::>(), - cursor_next, - cursor_previous, - }; - - let body = serde_json::to_string(&response) + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash + }).collect::>(), + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) .map_err(|e| ListWeightError::ServerError)?; From ae5eea53fe67f3a5b619f080a23cdddc42662b09 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 03:57:40 -0500 Subject: [PATCH 13/58] Something wrong with the query parameters not passing off the limits etc --- crates/service/web/storyteller_web/src/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 6c06107519..33130372c3 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1251,7 +1251,7 @@ fn add_weights_routes(app: App) -> App ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) .route( - "/by_user_and_type/{username}/{type}", + "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) ) From 8c7b10015039937d53f25a0c44ec004fa79f7663 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 18:06:11 -0500 Subject: [PATCH 14/58] Adding new seed then have to switch tests back. --- crates/cli/dev_database_seed/src/main.rs | 1 - .../src/seeding/model_weights.rs | 392 +++++++++++++++--- 2 files changed, 327 insertions(+), 66 deletions(-) diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index ed29579867..0288c5aca8 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -61,7 +61,6 @@ pub async fn main() -> AnyhowResult<()> { seed_voice_conversion(&pool).await?; seed_weights(&pool).await?; - seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 07c8f2f155..2de742f766 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -18,36 +18,79 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; -pub async fn seed_weights_for_paging( - mysql_pool: &Pool) -> AnyhowResult<()> -{ +pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { + let sd1_5_markdown_description = r#" +# Dragonfruit AI Models Update and Workflow - let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { - None => { - return Err(anyhow!("could not find user hanashi")); - } - Some(token) => token, - }; +## Last Update for Dragonfruit Models +- **Note:** This might be the last update for a while, focusing next on a realistic merge (WIP). +- **Future Plans:** Possible update to Z.5D v2 after completing the realistic merge. + +## New Models Announcement +- **Upcoming Models:** Separate from the Dragonfruit branch, so stay tuned! + +## DragonfruitGT v1 (Z v2) +- **Focus:** More fantasy, richer color range, improved landscapes, more masculine men. +- **Alternative:** Use DragonfruitOG v2.0 for thicker lines, simpler colors. + +## GT v1.0 (Z v2.0) Features +- Improved backgrounds +- Deeper contrast and color +- Bolder lines and more detail + +### Settings for GT +- Steps: 30-50 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +## Workflow for Images +1. **BerryMix VAE:** Ensure correct color. +2. **Adetailer Extension:** Improve faces, bodies, and hands. Also available on Civitai and Huggingface. +3. **Hires Fix:** Denoising strength 0.3 - 0.35, using upscaler "x_NMKD-Superscale-SP_178000_G". +4. **Img2img Transfer:** Use denoising strength 0.3-0.35 with the same prompt and settings. Adjust height after hires. +5. **Second Img2img Transfer:** Low denoising strength 0.1-3.0. Use SD upscale script with "RealESRGAN_x4plus_anime_6B" sampler for size increase. + +## DragonfruitZ.5D v1.0 +- **Style:** 2.5D model, merging DragonfruitZ with realistic elements. +- **Features:** Skin color variety, masculine men, landscapes, fantasy, NSFW content, detailed faces. + +### Settings for Z.5D +- Steps: 30 +- Clip skip: 2 +- Upscaler: Euler a +- Extensions: Adetailer, VAE: berrysmix + +## DragonfruitOG v2.0 +- **Refinement:** Of the original Dragonfruit model. +- **Style:** Bold, clean lines and bold colors with decreased detail for simplicity. +- **Improvements:** Teeth, skin color variety, landscapes, fantasy elements, NSFW content. + +## DragonfruitZ v1.0 +- **Alternative Version:** More fantasy and color range, better landscapes, more masculine men. +- **Suggestion:** Use Dragonfruit (Unisex) v2 for thicker lines and simpler colors. + +## Dragonfruit OG v1.0 +- **Focus:** More unisex, suitable for creating male characters. +- **Personal Note:** First merge, aimed at 2D, flat, colorful, thick lined styles. + +### Personal Settings +- Steps: 20-30 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +*Thank you for your continued support and feedback!* +"#; + +let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + let user_token = user_token; // create a loop that loops from 1 to 100 for i in 1..=100 { // create a new weight - let model_weight_token = ModelWeightToken(format!("item:{}", i)); - let title = format!("Title {}", i); - let description = format!("Description {}", i); - let description_rendered_html = format!("

{}

", description); - let original_filename = format!("filename{}.txt", i); - let original_download_url = format!("http://example.com/download{}", i); - let thumbnail_token = format!("Thumbnail {}", i); - let private_bucket_hash = format!("bucket_hash{}", i); - let private_bucket_prefix = format!("_fake{}", i); - let private_bucket_extension = format!("rvc{}", i); - let cached_user_ratings_total_count = i; - let cached_user_ratings_positive_count = i; - let cached_user_ratings_negative_count = i; - let cached_user_ratings_ratio = i as u32 / 100; - let version = i as i32; - + let weights_category = match i { 1..=20 => WeightsCategory::VoiceConversion, 21..=40 => WeightsCategory::TextToSpeech, @@ -66,37 +109,250 @@ pub async fn seed_weights_for_paging( _ => WeightsType::LoRA }; - let args = CreateModelWeightsArgs { - token: &model_weight_token, // replace with actual ModelWeightToken - weights_type: weights_types, // replace with actual WeightsType - weights_category: weights_category, // replace with actual WeightsCategory - title: title, - maybe_thumbnail_token: Some(thumbnail_token), - description_markdown: description, - description_rendered_html: description_rendered_html, - creator_user_token: Some(&user_token), // replace with actual UserToken - creator_ip_address: "192.168.1.1", - creator_set_visibility: Visibility::Public, - maybe_last_update_user_token: Some(user_token.to_string()), - original_download_url: Some(original_download_url), - original_filename: Some(original_filename), - file_size_bytes: 1024, - file_checksum_sha2: "checksum1".to_string(), - private_bucket_hash: "bucket_hash1".to_string(), - maybe_private_bucket_prefix: Some("_fake".to_string()), - maybe_private_bucket_extension: Some("rvc".to_string()), - cached_user_ratings_total_count: cached_user_ratings_total_count, - cached_user_ratings_positive_count: cached_user_ratings_positive_count, - cached_user_ratings_negative_count: cached_user_ratings_ratio, - maybe_cached_user_ratings_ratio: Some(0.9), - version: version, - mysql_pool: &mysql_pool, // replace with actual MySqlPool - }; + let mut model_weight_token; + let mut title; + let mut description; + let mut description_rendered_html; + let mut original_filename; + let mut original_download_url; + let mut thumbnail_token; + + let mut private_bucket_hash:String = "".to_string(); + let mut private_bucket_prefix; + let mut private_bucket_extension; - create_weight(args).await?; + let mut cached_user_ratings_total_count; + let mut cached_user_ratings_positive_count; + let mut cached_user_ratings_negative_count; + let mut cached_user_ratings_ratio; + let mut version; + + match i { + 1..=20 => { + // voice conversion model + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding RVCv2 model {}", i); + }, + 21..=40 =>{ + // TTS + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + let tts_markdown_description = r#" + # HifiGAN Tacotron2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Goku MOS 4 + "#; + + description_rendered_html = tts_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("tt2"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding TT2 model {}", i); + }, + 41..=60 => { + // SD 1.5 + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = sd1_5_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; //sd1_5_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sd15"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SD15 model {}", i); + }, + 61..=80 => { + // SD XL + let sdxl_markdown_description = r#" + # Special Edition Release Announcement + + ## What's New? + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + + ## Tips for Use + 1. **Simple Prompts:** Avoid adding words like "detailed, realistic..." or specifying a specific artist. Simply describe what you want to see. + 2. **CFG Scale:** A range between 3 and 8.5 is effective (recommended: 7). + 3. **Minimum Steps:** Ensure at least 36 steps for optimal results. + 4. **Compatibility:** + - Works excellently with Hires fix. + - Great results with unaestheticXLv31 embedding, although sometimes better without it. + 5. **Sampling Method:** Either DPM++ 2M SDE Karras or DPM++ 2M Karras. + + Have fun! 😊 + "#; + let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("SDXL_Niji_Special Edition: {}", i); + description = format!("Description Number {}:", i); + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sdxl"); + + description_rendered_html = sdxl_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SDXL model {}", i); + }, + 81..=100 => { + // LoRA + let lora_markdown_description = r#" + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - LoRA Gawr Gura + "#; + + let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg"; + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura LoRA: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = lora_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; // lora_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("loRA"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding LoRA model {}", i); + }, + _ => { + // We went out of range so have more gwar gura + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + + println!("Out of range"); + } + } + + let args = CreateModelWeightsArgs { + token: &model_weight_token, // replace with actual ModelWeightToken + weights_type: weights_types, // replace with actual WeightsType + weights_category: weights_category, // replace with actual WeightsCategory + title: title, + maybe_thumbnail_token: Some(thumbnail_token.to_string()), + description_markdown: description, + description_rendered_html: description_rendered_html.to_string(), + creator_user_token: Some(&user_token), // replace with actual UserToken + creator_ip_address: "192.168.1.1", + creator_set_visibility: Visibility::Public, + maybe_last_update_user_token: Some(user_token.to_string()), + original_download_url: Some(original_download_url), + original_filename: Some(original_filename), + file_size_bytes: 1024, + file_checksum_sha2: "checksum1".to_string(), + private_bucket_hash: "bucket_hash1".to_string(), + maybe_private_bucket_prefix: Some(private_bucket_prefix), + maybe_private_bucket_extension: Some(private_bucket_extension), + cached_user_ratings_total_count: cached_user_ratings_total_count, + cached_user_ratings_positive_count: cached_user_ratings_positive_count, + cached_user_ratings_negative_count: cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32), + version: version, + mysql_pool: &mysql_pool, // replace with actual MySqlPool + }; + + create_weight(args).await; } Ok(()) - } pub async fn seed_weights_for_user_token( mysql_pool: &Pool, @@ -248,16 +504,9 @@ pub async fn seed_weights_for_user_token( } Ok(()) } -pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { - info!("Seeding weights..."); - let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { - None => { - return Err(anyhow!("could not find user hanashi")); - } - Some(token) => token, - }; +pub async fn original_seed_weights(mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { let model_weight_token1 = ModelWeightToken("1".to_string()); let creator_token1 = UserToken("creatorToken1".to_string()); @@ -410,9 +659,22 @@ pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { create_weight(model_weights_arg).await?; } - // adds this to HANASHI - get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await?; - seed_weights_for_user_token(mysql_pool, user_token).await?; - seed_weights_for_paging(mysql_pool).await?; + Ok(()) + +} +pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { + info!("Seeding weights..."); + + let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { + None => { + return Err(anyhow!("could not find user hanashi")); + } + Some(token) => token, + }; + + //original_seed_weights(mysql_pool,user_token).await?; + //seed_weights_for_user_token(mysql_pool, user_token).await?; + seed_weights_for_paging(mysql_pool,user_token).await?; + Ok(()) } From 91d0a1da8f02692b4f1bae743cd7ca8834c253fc Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 18:21:43 -0500 Subject: [PATCH 15/58] Tests pass again --- .../list_weights_query_builder.rs | 4 +-- .../src/queries/model_weights/tests.rs | 36 +++++++++---------- .../service/web/storyteller_web/src/routes.rs | 1 + 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs index e1d79b89ab..d1d18302b9 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs @@ -505,7 +505,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_type() { - let query_builder = ListWeightsQueryBuilder::new().weights_type(WeightsType::RvcV2); + let query_builder = ListWeightsQueryBuilder::new().weights_type(Some(WeightsType::RvcV2)); assert_eq!( &query_builder.build_predicates(), " WHERE model_weights.weights_type = ? \ @@ -520,7 +520,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_category() { let query_builder = ListWeightsQueryBuilder::new().weights_category( - WeightsCategory::VoiceConversion + Some(WeightsCategory::VoiceConversion) ); println!("Query HERE! {}", &query_builder.build_predicates()); assert_eq!( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 9b054e4b30..52dc725ed6 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -360,8 +360,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); @@ -372,8 +372,8 @@ mod tests { assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) .offset(Some(10)) @@ -384,8 +384,8 @@ mod tests { assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) .offset(Some(10)) @@ -409,8 +409,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -429,8 +429,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -460,8 +460,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -478,8 +478,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -576,8 +576,8 @@ mod tests { // try a query with all the filters let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::Tacotron2) - .weights_category(WeightsCategory::TextToSpeech) + .weights_type(Some(WeightsType::Tacotron2)) + .weights_category(Some(WeightsCategory::TextToSpeech)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); @@ -587,8 +587,8 @@ mod tests { assert_eq!(result.weights.len(), 2); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 33130372c3..91efc45771 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1254,5 +1254,6 @@ fn add_weights_routes(app: App) -> App "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) + .route("/list/{username}/{weights_category}/{weights_type}",list_weights_handler) ) } From 130e0daa084ddb2c5068c95ffdefadcfd83c85d7 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 22:49:33 -0500 Subject: [PATCH 16/58] paging works correctly. --- .../src/queries/model_weights/tests.rs | 389 +++++++++++++----- 1 file changed, 290 insertions(+), 99 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 2a24c7fda3..917fdd98b3 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -3,8 +3,9 @@ mod tests { use std::result; use anyhow::Ok; - use enums::by_table::model_weights::{weights_types, weights_category}; - use sqlx::{MySqlPool, Acquire}; + use chrono::offset; + use enums::by_table::model_weights::{ weights_types, weights_category }; + use sqlx::{ MySqlPool, Acquire }; use rand::Rng; use tokio; @@ -24,7 +25,12 @@ mod tests { use crate::queries::model_weights::create_weight::create_weight; use crate::queries::model_weights::get_weight::get_weight_by_token; - use crate::queries::model_weights::delete_weights::{ delete_weights_as_user, delete_weights_as_mod, undelete_weights_as_mod , undelete_weights_as_user}; + use crate::queries::model_weights::delete_weights::{ + delete_weights_as_user, + delete_weights_as_mod, + undelete_weights_as_mod, + undelete_weights_as_user, + }; use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; @@ -45,23 +51,19 @@ mod tests { pub async fn delete_all_weights_for_table(mysql_pool: &MySqlPool) -> AnyhowResult<()> { // write a query that deletes all weights - let _r = sqlx::query("DELETE FROM model_weights") - .execute(mysql_pool) - .await?; + let _r = sqlx::query("DELETE FROM model_weights").execute(mysql_pool).await?; Ok(()) } #[tokio::test] #[serial] async fn test_create_weights() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); let random_number: u32 = rng.gen(); let model_weight_token1 = ModelWeightToken(random_number.to_string()); - let creator_token1 = UserToken("creatorToken!1".to_string()); let args = CreateModelWeightsArgs { @@ -131,7 +133,6 @@ mod tests { #[tokio::test] #[serial] async fn test_delete_and_undelete_weights_user() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); @@ -170,7 +171,7 @@ mod tests { create_weight(args).await?; delete_weights_as_user(&model_weight_token1, &pool).await?; - + let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -261,7 +262,6 @@ mod tests { } } - undelete_weights_as_mod(&model_weight_token1, &pool).await?; let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -285,13 +285,22 @@ mod tests { #[serial] async fn list_all_weights_by_user() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); let can_see_deleted = true; - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,5,&creator_username).await?; + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 5, + &creator_username + ).await?; - let weights_by_username = list_weights_by_creator_username(&pool, &creator_username, can_see_deleted).await?; + let weights_by_username = list_weights_by_creator_username( + &pool, + &creator_username, + can_see_deleted + ).await?; for weight in weights_by_username.iter() { // print weight println!("weight: {:?}", weight.creator_username); @@ -301,23 +310,29 @@ mod tests { Ok(()) } - async fn seed_weights_with_category_and_type(weights_type:WeightsType,weights_category:WeightsCategory,number_of_items:u32,creator_username:&str) -> AnyhowResult<()> { - + async fn seed_weights_with_category_and_type( + weights_type: WeightsType, + weights_category: WeightsCategory, + number_of_items: u32, + creator_username: &str + ) -> AnyhowResult<()> { let pool = setup().await; - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..number_of_items { - let mut rng = rand::thread_rng(); let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -346,7 +361,6 @@ mod tests { } Ok(()) - } // Tests paging for the list weights query builder @@ -356,40 +370,45 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,20,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 20, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); - + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + let result = qb.perform_query_for_page(&pool).await?; - + // let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; @@ -398,27 +417,31 @@ mod tests { Ok(()) } - #[tokio::test] #[serial] async fn list_weights_query_build_test_asc_desc_cursor_reverse() -> AnyhowResult<()> { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true) - .cursor_is_reversed(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .cursor_is_reversed(false); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -428,19 +451,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false) - .cursor_is_reversed(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false) + .cursor_is_reversed(true); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -456,18 +478,23 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -477,19 +504,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -503,11 +529,14 @@ mod tests { #[serial] async fn list_weights_query_build_test() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..6 { @@ -519,7 +548,7 @@ mod tests { WeightsType::RvcV2 } else if i % 5 == 3 { WeightsType::StableDiffusionXL - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsType::SoVitsSvc } else { WeightsType::StableDiffusion15 @@ -533,7 +562,7 @@ mod tests { WeightsCategory::VoiceConversion } else if i % 5 == 3 { WeightsCategory::ImageGeneration - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsCategory::VoiceConversion } else { WeightsCategory::ImageGeneration @@ -543,10 +572,10 @@ mod tests { let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -581,18 +610,18 @@ mod tests { .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); - let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 2); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 1); assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); assert_eq!(result.weights[0].weights_type, WeightsType::LoRA); @@ -600,6 +629,168 @@ mod tests { Ok(()) } - -} + #[tokio::test] + #[serial] + async fn list_weights_query_builder_page_type_and_category() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + // using the seeded weights from the previous test + let creator_username = "hanashi".to_string(); + for i in 0..4 { + let weights_types = vec![ + WeightsType::HifiganTacotron2, + WeightsType::LoRA, + WeightsType::RvcV2, + WeightsType::StableDiffusionXL + ]; + + let weights_type = weights_types[i as usize]; + + let qb = ListWeightsQueryBuilder::new() + .weights_type(Some(weights_type)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + println!("weights_type: {:?}", weights_type); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 20, "Should be 20 weights"); + assert_eq!( + result.weights[0].weights_type, + weights_type, + "Should be the same weights type" + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_weight_query_builder_all() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .limit(11); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 11); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_limits() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), 10); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + let result = qb.perform_query_for_page(&pool).await?; + + // by default 25 + assert_eq!(result.weights.len(), 25); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + // over query limit + let qb: ListWeightsQueryBuilder = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(80); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 60); + // pick a random value in here + assert_eq!(result.weights[9].weights_category, WeightsCategory::ImageGeneration); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn page_across_a_category() -> AnyhowResult<()> { + // paging across a category and it works + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + // how we implement paging. + let page:u16 = 0; + let page_size: u16 = 20; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .sort_ascending(true) + .limit(page_size); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let page = 1; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .sort_ascending(true) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + let page = 2; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + Ok(()) + } +} From f99b5a92b81ccb6e9a6ce0f08fb819e67d41e81d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 13:44:50 -0500 Subject: [PATCH 17/58] Added weights paging to michael's spec missing the fields. --- .../endpoints/weights/list_availible_weights.rs | 17 +++++++++++------ .../service/web/storyteller_web/src/routes.rs | 1 - 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 26257fc60a..7587931d3b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -22,22 +22,23 @@ use crate::server_state::ServerState; #[derive(Deserialize)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, - pub limit: Option, - pub cursor: Option, - pub cursor_is_reversed: Option, + pub page_size: u16, + pub page_index : u16, } #[derive(Serialize)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, + pub page_size: u16, + pub page_index : u16, pub cursor_next: Option, pub cursor_previous: Option, } #[derive(Deserialize)] pub struct ListWeightsByPathInfo { - pub username: String, + pub username: Option, pub weights_type: Option, pub weights_category: Option, } @@ -75,6 +76,11 @@ pub struct ModelWeightForList { pub creator_username: String, pub creator_display_name: String, pub creator_email_gravatar_hash: String, + + // additional fields to be added when tables are around + pub likes: u32, + pub bookmarks: bool, + } #[derive(Debug)] @@ -123,7 +129,7 @@ pub async fn list_availible_weights_handler( } }; - let limit = query.limit.unwrap_or(25); + let limit = query.page_size.unwrap_or(10); let sort_ascending = query.sort_ascending.unwrap_or(false); let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); @@ -236,7 +242,6 @@ pub async fn list_availible_weights_handler( file_size_bytes:weights.file_size_bytes, file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9f202a55d7..0ef5dbd81a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1260,6 +1260,5 @@ fn add_weights_routes(app: App) -> App "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) - .route("/list/{username}/{weights_category}/{weights_type}",list_weights_handler) ) } From 49934107405a04e9e7a9e6ae4bbd4abc99649998 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 14:54:32 -0500 Subject: [PATCH 18/58] Built needs a simple endpoint query test --- .../weights/list_availible_weights.rs | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 7587931d3b..044b05b113 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -5,7 +5,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use chrono::{ DateTime, Utc }; use log::{ info, warn }; - +use rand::Rng; use enums::common::visibility::Visibility; use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; @@ -129,21 +129,9 @@ pub async fn list_availible_weights_handler( } }; - let limit = query.page_size.unwrap_or(10); + let limit = query.page_size; let sort_ascending = query.sort_ascending.unwrap_or(false); - let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); - - let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; let include_user_hidden = is_mod; @@ -185,12 +173,10 @@ pub async fn list_availible_weights_handler( .include_user_deleted_results(is_mod) .include_mod_deleted_results(is_mod) .limit(limit) - .cursor_is_reversed(cursor_is_reversed) - .offset(cursor); + .offset(Some((query.page_index as u64) * (query.page_size as u64))); let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - let weights_page = match query_results { Ok(results) => results, Err(e) => { @@ -221,6 +207,9 @@ pub async fn list_availible_weights_handler( None }; + let mut rng = rand::thread_rng(); + let random_float = rng.gen_range(0.0..1.0); + let random_bool = random_float >= 0.5; // generate parse a response let response = ListAvailibleWeightsSuccessResponse { @@ -242,7 +231,6 @@ pub async fn list_availible_weights_handler( file_size_bytes:weights.file_size_bytes, file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, @@ -255,10 +243,15 @@ pub async fn list_availible_weights_handler( creator_username: weights.creator_username, creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash + creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + + bookmarks: random_bool, + likes: rng.gen_range(0..1000), }).collect::>(), cursor_next, cursor_previous, + page_index: query.page_index, + page_size: query.page_size }; let body = serde_json::to_string(&response) From 1a3b02b5d2635eac00e876093f2cb8046a1b08f9 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 21:46:53 -0500 Subject: [PATCH 19/58] Adding docs --- Cargo.lock | 3 +- crates/_types/enums/Cargo.toml | 1 + .../model_weights/weights_category.rs | 4 ++- .../by_table/model_weights/weights_types.rs | 3 +- .../web/storyteller_web/src/api_doc.rs | 9 +++--- .../web/storyteller_web/src/api_doc_server.rs | 1 - .../inference/enqueue_tts_request.rs | 9 ++---- .../weights/list_availible_weights.rs | 28 +++++++++++++++---- 8 files changed, 38 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a95e836b72..00a71c2794 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,7 +1053,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2178,6 +2178,7 @@ dependencies = [ "sqlx-mysql", "strum", "toml", + "utoipa", ] [[package]] diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 1cb6c94409..4479a39957 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -21,6 +21,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } [dev-dependencies] diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index c2bca7f24d..5818ec59ab 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -5,8 +5,10 @@ use strum::EnumCount; #[cfg(test)] use strum::EnumIter; +use utoipia::ToSchema; + #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { #[serde(rename = "image_generation")] ImageGeneration, diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index e7ff3c3c97..0ef556d927 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -4,10 +4,11 @@ use std::collections::BTreeSet; use strum::EnumCount; #[cfg(test)] use strum::EnumIter; +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f76feb1e76..f90d681efd 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,11 +1,12 @@ -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; -use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use crate::http_server::endpoints::*; +use tokens::tokens::*; use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] + + #[openapi(paths(crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights), + components(schemas(ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index e3fc865736..8f305b120e 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -56,7 +56,6 @@ use std::{ pub mod api_doc; use api_doc::ApiDoc; - use futures::future::LocalBoxFuture; use utoipa::OpenApi; #[actix_web::main] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 32ae10d58d..41adb52569 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -43,23 +43,20 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; -#[derive(Deserialize)] -#[derive(ToSchema)] +#[derive(Deserialize,ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, voice_token: String, } -#[derive(Serialize)] -#[derive(ToSchema)] +#[derive(Serialize,ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } -#[derive(Debug)] -#[derive(ToSchema)] +#[derive(Debug,ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 044b05b113..f658f0a62a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -13,20 +13,23 @@ use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, }; + use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; -#[derive(Deserialize)] +use utoipia::Schema; + +#[derive(Deserialize,Schema)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, pub page_size: u16, pub page_index : u16, } -#[derive(Serialize)] +#[derive(Serialize,Schema)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, @@ -36,14 +39,14 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } -#[derive(Deserialize)] +#[derive(Deserialize,Schema)] pub struct ListWeightsByPathInfo { pub username: Option, pub weights_type: Option, pub weights_category: Option, } -#[derive(Serialize)] +#[derive(Serialize,Schema)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, @@ -83,7 +86,7 @@ pub struct ModelWeightForList { } -#[derive(Debug)] +#[derive(Debug,Schema)] pub enum ListWeightError { NotAuthorized, ServerError, @@ -104,6 +107,19 @@ impl ResponseError for ListWeightError { } } +#[utoipa::path( + get, + path = "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", + responses( + (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightError), + (status = 500, description = "Server error", body = ListWeightError), + ), + params( + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request") + ("path" = ListWeightsByPathInfo, description = "Path for Request") + ) +)] pub async fn list_availible_weights_handler( http_request: HttpRequest, path: web::Path, @@ -245,6 +261,7 @@ pub async fn list_availible_weights_handler( creator_display_name: weights.creator_display_name, creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + // TODO FIX THIS when we align. bookmarks: random_bool, likes: rng.gen_range(0..1000), }).collect::>(), @@ -257,6 +274,5 @@ pub async fn list_availible_weights_handler( let body = serde_json::to_string(&response) .map_err(|e| ListWeightError::ServerError)?; - Ok(HttpResponse::Ok().content_type("application/json").body(body)) } From 32cfb10323b8f4bdc0a6373d9f46173f5207a475 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 00:55:28 -0500 Subject: [PATCH 20/58] added weights list documentation --- .../model_weights/weights_category.rs | 2 +- crates/_types/enums/src/common/visibility.rs | 5 ++-- .../_types/tokens/src/tokens/model_weights.rs | 3 +- crates/_types/tokens/src/tokens/users.rs | 4 +-- .../web/get_inference_job_status.rs | 1 + .../web/storyteller_web/src/api_doc.rs | 28 ++++++++++++++----- .../weights/list_availible_weights.rs | 14 +++++----- 7 files changed, 37 insertions(+), 20 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index 5818ec59ab..f6c4e6ed3f 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -5,8 +5,8 @@ use strum::EnumCount; #[cfg(test)] use strum::EnumIter; -use utoipia::ToSchema; +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { diff --git a/crates/_types/enums/src/common/visibility.rs b/crates/_types/enums/src/common/visibility.rs index f7240ca332..7e9c0341f3 100644 --- a/crates/_types/enums/src/common/visibility.rs +++ b/crates/_types/enums/src/common/visibility.rs @@ -15,9 +15,10 @@ use strum::EnumIter; /// See also: https://docs.rs/sqlx/0.4.0-beta.1/sqlx/trait.Type.html /// /// *DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY!* -/// + +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type)] +#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type,ToSchema)] #[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum Visibility { diff --git a/crates/_types/tokens/src/tokens/model_weights.rs b/crates/_types/tokens/src/tokens/model_weights.rs index ba29c1f8fb..a63f2d9028 100644 --- a/crates/_types/tokens/src/tokens/model_weights.rs +++ b/crates/_types/tokens/src/tokens/model_weights.rs @@ -5,8 +5,9 @@ use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for the "model_weights" table. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct ModelWeightToken(pub String); diff --git a/crates/_types/tokens/src/tokens/users.rs b/crates/_types/tokens/src/tokens/users.rs index 2e28fca9fe..4a678ebf9f 100644 --- a/crates/_types/tokens/src/tokens/users.rs +++ b/crates/_types/tokens/src/tokens/users.rs @@ -4,9 +4,9 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; - +use utoipa::ToSchema; /// The primary key for users. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct UserToken(pub String); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 615ef68ca9..9f6137cf01 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; + use chrono::{DateTime, Utc}; use log::warn; use sqlx::{MySql, MySqlPool}; diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f90d681efd..f86ca46b48 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,12 +1,26 @@ -use crate::http_server::endpoints::*; -use tokens::tokens::*; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; +use crate::http_server::endpoints::weights::list_availible_weights::*; +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::generic_inference_jobs::*; +use tokens::tokens::model_weights::*; +use tokens::tokens::users::*; +use enums::common::visibility::Visibility; +use chrono::{DateTime, Utc}; use utoipa::OpenApi; -#[derive(OpenApi)] -#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), - components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] - #[openapi(paths(crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights), - components(schemas(ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, + crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler + ),components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + + ModelWeightToken,UserToken,Visibility, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] pub struct ApiDoc; + diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index f658f0a62a..5ababcdff9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -20,16 +20,16 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; -use utoipia::Schema; +use utoipa::ToSchema; -#[derive(Deserialize,Schema)] +#[derive(Deserialize,ToSchema)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, pub page_size: u16, pub page_index : u16, } -#[derive(Serialize,Schema)] +#[derive(Serialize,ToSchema)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, @@ -39,14 +39,14 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } -#[derive(Deserialize,Schema)] +#[derive(Deserialize,ToSchema)] pub struct ListWeightsByPathInfo { pub username: Option, pub weights_type: Option, pub weights_category: Option, } -#[derive(Serialize,Schema)] +#[derive(Serialize,ToSchema)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, @@ -86,7 +86,7 @@ pub struct ModelWeightForList { } -#[derive(Debug,Schema)] +#[derive(Debug,ToSchema)] pub enum ListWeightError { NotAuthorized, ServerError, @@ -116,7 +116,7 @@ impl ResponseError for ListWeightError { (status = 500, description = "Server error", body = ListWeightError), ), params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request") + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), ("path" = ListWeightsByPathInfo, description = "Path for Request") ) )] From f76928d0019b0f15075e6e8338731624d50679a5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 01:28:54 -0500 Subject: [PATCH 21/58] Delete weight documentation added --- .../web/storyteller_web/src/api_doc.rs | 16 +++++++--- .../endpoints/weights/delete_weight.rs | 30 ++++++++++++++++--- .../web_utils/response_success_helpers.rs | 4 +-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f86ca46b48..9eb6412d18 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,7 +1,10 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; -use crate::http_server::endpoints::weights::list_availible_weights::*; + +use crate::http_server::endpoints::weights::list_availible_weights::*; +use crate::http_server::web_utils::response_success_helpers::simple_json_success::SimpleGenericJsonSuccess; +use crate::http_server::endpoints::weights::delete_weight::*; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -16,11 +19,16 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, - crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler - ),components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, + crate::http_server::endpoints::weights::delete_weight::delete_weight_handler + ),components(schemas( + SimpleGenericJsonSuccess, + EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, ModelWeightToken,UserToken,Visibility, ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError + )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index f8f547b3b3..a589b1eefe 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -10,7 +10,12 @@ use enums::by_table::audit_logs::audit_log_entity_action::AuditLogEntityAction:: use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; -use mysql_queries::queries::model_weights::delete_weights::{delete_weights_as_user,delete_weights_as_mod,undelete_weights_as_user,undelete_weights_as_mod}; +use mysql_queries::queries::model_weights::delete_weights::{ + delete_weights_as_user, + delete_weights_as_mod, + undelete_weights_as_user, + undelete_weights_as_mod +}; use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use tokens::tokens::model_weights::ModelWeightToken; @@ -22,21 +27,23 @@ use crate::util::delete_role_disambiguation::{delete_role_disambiguation, Delete use enums::common::visibility::Visibility; use tokens::tokens::users::UserToken; +use utoipa::ToSchema; -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct DeleteWeightPathInfo { weight_token: String, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct DeleteWeightRequest { set_delete: bool, as_mod: bool } + // =============== Error Response =============== -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize,ToSchema)] pub enum DeleteWeightError { BadInput(String), NotFound, @@ -66,6 +73,21 @@ impl fmt::Display for DeleteWeightError { } } + +#[utoipa::path( + get, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = DeleteWeightError), + (status = 401, description = "Not authorized", body = DeleteWeightError), + (status = 500, description = "Server error", body = DeleteWeightError), + ), + params( + ("request" = DeleteWeightRequest, description = "Payload for Request"), + ("path" = DeleteWeightPathInfo, description = "Path for Request") + ) +)] pub async fn delete_weight_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs index dae959059f..16908c9ac9 100644 --- a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs +++ b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs @@ -1,6 +1,6 @@ use actix_web::HttpResponse; - -#[derive(Serialize)] +use utoipa::ToSchema; +#[derive(Serialize,ToSchema)] pub struct SimpleGenericJsonSuccess { pub success: bool, } From f7d5ad496cafb910fa344cc3cca53c91612fd501 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 14:01:31 -0500 Subject: [PATCH 22/58] Update docs --- .../web/storyteller_web/src/api_doc.rs | 19 +- .../endpoints/weights/create_weight.rs | 296 ++++-------------- .../endpoints/weights/get_weight.rs | 7 +- .../endpoints/weights/update_weight.rs | 23 +- 4 files changed, 101 insertions(+), 244 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 9eb6412d18..33c7054cd3 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -20,15 +20,18 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, - crate::http_server::endpoints::weights::delete_weight::delete_weight_handler + crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, + crate::http_server::endpoints::weights::update_weight::update_weight_handler, + crate::http_server::endpoints::weights::get_weight::get_weight_handler, + crate::http_server::endpoints::weights::create_weight::create_weight_handler, ),components(schemas( - SimpleGenericJsonSuccess, - EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + SimpleGenericJsonSuccess, + EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, - ModelWeightToken,UserToken,Visibility, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, - DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError - )))] + ModelWeightToken,UserToken,Visibility, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError + )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 6b9194135f..5e502e6a57 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -28,8 +28,9 @@ use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_me use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; -#[derive(Serialize)] +#[derive(Serialize,ToSchema)] pub struct UploadWeightsSuccessResponse { pub success: bool, pub upload_token: ModelWeightToken, @@ -41,241 +42,80 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { ]) }); -pub async fn upload_weights_handler( - http_request: HttpRequest, - server_state: web::Data>, - mut multipart_payload: Multipart, -) -> Result { +// pub async fn create_weight_handler( +// http_request: HttpRequest, +// server_state: web::Data>, +// mut multipart_payload: Multipart, +// ) -> Result { - let response = handle_weight_upload( - &http_request, - &server_state, - multipart_payload, - &ALLOWED_MIME_TYPES).await?; - - // send a body with the required payload along side the multipart upload + - let weight_upload_token = response.to_model_token(); +// let weight_upload_token = response.to_model_token(); - let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { - success: true, - upload_token: weight_upload_token, - }; +// let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { +// success: true, +// upload_token: weight_upload_token, +// }; - let body = serde_json::to_string(&response) - .map_err(|e| UploadError::ServerError)?; +// let body = serde_json::to_string(&response) +// .map_err(|e| UploadError::ServerError)?; - return Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)); -} +// return Ok(HttpResponse::Ok() +// .content_type("application/json") +// .body(body)); +// } -pub enum SuccessCase { - WeightAlreadyUploaded { - existing_upload_token: ModelWeightToken, - }, - WeightSuccessfullyUploaded { - upload_token: ModelWeightToken, - } -} +// pub enum SuccessCase { +// WeightAlreadyUploaded { +// existing_upload_token: ModelWeightToken, +// }, +// WeightSuccessfullyUploaded { +// upload_token: ModelWeightToken, +// } +// } -impl SuccessCase { - pub fn to_model_token(self) -> ModelWeightToken { - match self { - SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, - SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, - } - } -} +// impl SuccessCase { +// pub fn to_model_token(self) -> ModelWeightToken { +// match self { +// SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, +// SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, +// } +// } +// } -pub async fn handle_weight_upload( - http_request: &HttpRequest, - server_state: &web::Data>, - mut multipart_payload: Multipart, - allowed_mimetypes: &HashSet<&'static str>, -) -> Result { - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - error!("MySql pool error: {:?}", err); - UploadError::ServerError - })?; +// let token = create_weight(CreateModelWeightsArgs { +// token: &ModelWeightToken::generate(), +// weights_type: WeightsType::LoRA, // to do parse multi part for this +// weights_category: WeightsCategory::ImageGeneration, +// title: upload_weights_request.file_name.as_deref(), +// maybe_thumbnail_token: &ThumbnailToken::generate(), +// description_markdown: "", +// description_rendered_html: "", +// creator_user_token: maybe_user_token.as_str(), +// creator_ip_address: &ip_address, +// creator_set_visibility, +// maybe_last_update_user_token: "", +// original_download_url: "", +// original_filename: upload_weights_request.file_name.as_deref(), +// file_size_bytes: file_size_bytes as i32, +// file_checksum_sha2: hash, +// private_bucket_hash: private_bucket_hash, +// maybe_private_bucket_prefix: None, +// maybe_private_bucket_extension: None, +// cached_user_ratings_total_count: 0, +// cached_user_ratings_positive_count: 0, +// cached_user_ratings_negative_count: 0, +// maybe_cached_user_ratings_ratio: None, +// version: 0, +// mysql_pool: &server_state.mysql_pool, +// }).await +// .map_err(|e| { +// error!("Error creating weights upload: {:?}", e); +// UploadError::ServerError +// })?; - // ==================== READ SESSION ==================== // +// info!("new weights upload id: {} token: {:?}", record_id, &token); - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) - .await - .map_err(|e| { - error!("Session checker error: {:?}", e); - UploadError::ServerError - })?; - - // ==================== RATE LIMIT ==================== // - - let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, - }; - - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - - // ==================== READ MULTIPART REQUEST ==================== // - - let upload_weights_request = drain_multipart_request(multipart_payload) - .await - .map_err(|e| { - // TODO: Error handling could be nicer. - UploadError::BadInput("bad request".to_string()) - })?; - - let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token - .ok_or(UploadError::BadInput("no uuid".to_string()))?; - - let maybe_existing_upload = - get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) - .await; - - match maybe_existing_upload { - Err(err) => { - error!("Error checking for previous upload: {:?}", err); - return Err(UploadError::ServerError); - } - Ok(Some(upload)) => { - // File already uploaded and frontend didn't protect us. - return Ok(SuccessCase::WeightAlreadyUploaded { - existing_upload_token: upload.token, - }); - } - Ok(None) => { - // Proceed. - } - } - - if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { - return Err(UploadError::BadInput(reason)); - } - - let creator_set_visibility = maybe_user_session - .as_ref() - .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. - .unwrap_or(Visibility::default()); - - let ip_address = get_request_ip(&http_request); - - let maybe_user_token = maybe_user_session - .map(|session| session.get_strongly_typed_user_token()); - - let maybe_file_size_bytes = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| bytes.len()); - - info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); - - let maybe_mimetype = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) - .flatten(); - - - let bytes = match upload_weights_request.file_bytes { - None => return Err(UploadError::BadInput("missing file contents".to_string())), - Some(bytes) => bytes, - }; - - let hash = sha256_hash_bytes(&bytes) - .map_err(|io_error| { - error!("Problem hashing bytes: {:?}", io_error); - UploadError::ServerError - })?; - - let file_size_bytes = bytes.len(); - - let mut maybe_duration_millis = None; - let mut maybe_codec_name = None; - let mut media_upload_type = None; - - if let Some(mimetype) = maybe_mimetype.as_deref() { - - if !allowed_mimetypes.contains(mimetype) { - // NB: Don't let our error message inject malicious strings - let filtered_mimetype = mimetype - .chars() - .filter(|c| c.is_ascii()) - .filter(|c| c.is_alphanumeric() || *c == '/') - .collect::(); - return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); - } - media_upload_type = match mimetype { - "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), - _ => None, - }; - } - - // This is fine for now we just don't expose the path! - let public_upload_path = WeightUploadOriginalFilePath::generate_new(); - - info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); - - // need to upload based on visibility - server_state.public_bucket_client.upload_file_with_content_type( - public_upload_path.get_full_object_path_str(), - bytes.as_ref(), - mime_type) - .await - .map_err(|e| { - warn!("Upload media bytes to bucket error: {:?}", e); - UploadError::ServerError - })?; - - let token = create_weight(CreateModelWeightsArgs { - token: &ModelWeightToken::generate(), - weights_type: WeightsType::LoRA, // to do parse multi part for this - weights_category: WeightsCategory::ImageGeneration, - title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: &ThumbnailToken::generate(), - description_markdown: "", - description_rendered_html: "", - creator_user_token: maybe_user_token.as_str(), - creator_ip_address: &ip_address, - creator_set_visibility, - maybe_last_update_user_token: "", - original_download_url: "", - original_filename: upload_weights_request.file_name.as_deref(), - file_size_bytes: file_size_bytes as i32, - file_checksum_sha2: hash, - private_bucket_hash: private_bucket_hash, - maybe_private_bucket_prefix: None, - maybe_private_bucket_extension: None, - cached_user_ratings_total_count: 0, - cached_user_ratings_positive_count: 0, - cached_user_ratings_negative_count: 0, - maybe_cached_user_ratings_ratio: None, - version: 0, - mysql_pool: &server_state.mysql_pool, - }).await - .map_err(|e| { - error!("Error creating weights upload: {:?}", e); - UploadError::ServerError - })?; - - info!("new weights upload id: {} token: {:?}", record_id, &token); - - Ok(SuccessCase::WeightSuccessfullyUploaded { - upload_token: token, - }) -} +// Ok() +// } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 9ca0ed207e..6bb712c677 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -18,7 +18,8 @@ use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use enums::by_table::model_weights::weights_types::WeightsType; use enums::by_table::model_weights::weights_category::WeightsCategory; -#[derive(Serialize, Clone)] +use utoipa::ToSchema +#[derive(Serialize, Clone,ToSchema)] pub struct GetWeightResponse { success: bool, weight_token: ModelWeightToken, @@ -44,12 +45,12 @@ pub struct GetWeightResponse { created_at: DateTime, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct GetWeightPathInfo { weight_token: String, } -#[derive(Debug)] +#[derive(Debug,ToSchema)] pub enum GetWeightError { NotAuthorized, NotFound, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 0e287729a3..47f087f23a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -15,9 +15,9 @@ use mysql_queries::queries::model_weights::update_weight::{ update_weights, Upda use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; - +use utopia::ToSchema; /// TODO will eventually be polymorphic -#[derive(Deserialize)] +#[derive(Deserialize, ToSchema)] pub struct UpdateWeightRequest { pub title: Option, pub thumbnail_token: Option, @@ -28,7 +28,7 @@ pub struct UpdateWeightRequest { pub visibility: Option, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub struct UpdateWeightResponse { pub success: bool, } @@ -41,7 +41,7 @@ pub struct UpdateWeightPathInfo { // =============== Error Response =============== -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub enum UpdateWeightError { BadInput(String), NotFound, @@ -72,7 +72,20 @@ impl fmt::Display for UpdateWeightError { } // =============== Handler =============== - +#[utoipa::path( + post, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = UpdateWeightError), + (status = 401, description = "Not authorized", body = UpdateWeightError), + (status = 500, description = "Server error", body = UpdateWeightError), + ), + params( + ("request" = UpdateWeightRequest, description = "Payload for Request"), + ("path" = UpdateWeightPathInfo, description = "Path for Request") + ) + )] pub async fn update_weight_handler( http_request: HttpRequest, path: Path, From 3a4ca7b54835de1998f8009305d5a58b9f0587e5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 16:14:08 -0500 Subject: [PATCH 23/58] weights doc complete. --- crates/_types/enums/Cargo.toml | 2 +- crates/_types/tokens/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../web/storyteller_web/src/api_doc.rs | 20 +- .../web/storyteller_web/src/api_doc_server.rs | 1 + .../endpoints/weights/delete_weight.rs | 2 +- .../endpoints/weights/get_weight.rs | 20 +- .../endpoints/weights/update_weight.rs | 4 +- .../endpoints/weights/upload_weight.rs | 271 ++++++++++++++++++ 9 files changed, 308 insertions(+), 16 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 4479a39957..71e7726e57 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -21,7 +21,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index e85c19e259..308f2e9a3f 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -24,7 +24,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index ffe732cdc1..d466063034 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -160,7 +160,7 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 33c7054cd3..9db38a306a 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,10 +1,11 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; - use crate::http_server::endpoints::weights::list_availible_weights::*; -use crate::http_server::web_utils::response_success_helpers::simple_json_success::SimpleGenericJsonSuccess; +use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; +use crate::http_server::endpoints::weights::get_weight::*; +use crate::http_server::endpoints::weights::update_weight::*; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -23,15 +24,20 @@ use utoipa::OpenApi; crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, - crate::http_server::endpoints::weights::create_weight::create_weight_handler, ),components(schemas( + // Tokens + UserToken,Visibility, + // SimpleGenericJsonSuccess, + // Inference EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, - - ModelWeightToken,UserToken,Visibility, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + //Model Weights + ModelWeightToken,WeightsCategory,WeightsType, + GetWeightPathInfo,GetWeightResponse,GetWeightError, + UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError, ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, - DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index 8f305b120e..e3fc865736 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -56,6 +56,7 @@ use std::{ pub mod api_doc; use api_doc::ApiDoc; + use futures::future::LocalBoxFuture; use utoipa::OpenApi; #[actix_web::main] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index a589b1eefe..7806fa8583 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -75,7 +75,7 @@ impl fmt::Display for DeleteWeightError { #[utoipa::path( - get, + delete, path = "/weight/{weight_token}", responses( (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 6bb712c677..1c0857b5b7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -18,8 +18,9 @@ use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use enums::by_table::model_weights::weights_types::WeightsType; use enums::by_table::model_weights::weights_category::WeightsCategory; -use utoipa::ToSchema -#[derive(Serialize, Clone,ToSchema)] +use utoipa::ToSchema; + +#[derive(Serialize, Clone, ToSchema)] pub struct GetWeightResponse { success: bool, weight_token: ModelWeightToken, @@ -72,7 +73,20 @@ impl ResponseError for GetWeightError { } } } - +#[utoipa::path( + get, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = GetWeightResponse), + (status = 400, description = "Bad input", body = GetWeightError), + (status = 401, description = "Not authorized", body = GetWeightError), + (status = 500, description = "Server error", body = GetWeightError), + ), + params( + ("request" = UpdateWeightRequest, description = "Payload for Request"), + ("path" = GetWeightPathInfo, description = "Path for Request") + ) + )] pub async fn get_weight_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 47f087f23a..8ddddca33e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -15,7 +15,7 @@ use mysql_queries::queries::model_weights::update_weight::{ update_weights, Upda use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; -use utopia::ToSchema; +use utoipa::ToSchema; /// TODO will eventually be polymorphic #[derive(Deserialize, ToSchema)] pub struct UpdateWeightRequest { @@ -34,7 +34,7 @@ pub struct UpdateWeightResponse { } /// For the URL PathInfo -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct UpdateWeightPathInfo { weight_token: String, } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs new file mode 100644 index 0000000000..2d66aa7684 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs @@ -0,0 +1,271 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use once_cell::sync::Lazy; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{error, info, warn}; + +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; + +pub async fn upload_weights_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, + ) -> Result { + + let response = handle_weight_upload( + &http_request, + &server_state, + multipart_payload, + &ALLOWED_MIME_TYPES).await?; + + // send a body with the required payload along side the multipart upload + + let weight_upload_token = response.to_model_token(); + + let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { + success: true, + upload_token: weight_upload_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); + } + + pub enum SuccessCase { + WeightAlreadyUploaded { + existing_upload_token: ModelWeightToken, + }, + WeightSuccessfullyUploaded { + upload_token: ModelWeightToken, + } + } + + impl SuccessCase { + pub fn to_model_token(self) -> ModelWeightToken { + match self { + SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, + SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, + } + } + } + + pub async fn handle_weight_upload( + http_request: &HttpRequest, + server_state: &web::Data>, + mut multipart_payload: Multipart, + allowed_mimetypes: &HashSet<&'static str>, + ) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + UploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + UploadError::ServerError + })?; + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_weights_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + UploadError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token + .ok_or(UploadError::BadInput("no uuid".to_string()))?; + + let maybe_existing_upload = + get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + .await; + + match maybe_existing_upload { + Err(err) => { + error!("Error checking for previous upload: {:?}", err); + return Err(UploadError::ServerError); + } + Ok(Some(upload)) => { + // File already uploaded and frontend didn't protect us. + return Ok(SuccessCase::WeightAlreadyUploaded { + existing_upload_token: upload.token, + }); + } + Ok(None) => { + // Proceed. + } + } + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(UploadError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); + + let maybe_mimetype = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + + let bytes = match upload_weights_request.file_bytes { + None => return Err(UploadError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + UploadError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_upload_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !allowed_mimetypes.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + media_upload_type = match mimetype { + "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), + _ => None, + }; + } + + // This is fine for now we just don't expose the path! + let public_upload_path = WeightUploadOriginalFilePath::generate_new(); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + // need to upload based on visibility + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + UploadError::ServerError + })?; + + let token = create_weight(CreateModelWeightsArgs { + token: &ModelWeightToken::generate(), + weights_type: WeightsType::LoRA, // to do parse multi part for this + weights_category: WeightsCategory::ImageGeneration, + title: upload_weights_request.file_name.as_deref(), + maybe_thumbnail_token: &ThumbnailToken::generate(), + description_markdown: "", + description_rendered_html: "", + creator_user_token: maybe_user_token.as_str(), + creator_ip_address: &ip_address, + creator_set_visibility, + maybe_last_update_user_token: "", + original_download_url: "", + original_filename: upload_weights_request.file_name.as_deref(), + file_size_bytes: file_size_bytes as i32, + file_checksum_sha2: hash, + private_bucket_hash: private_bucket_hash, + maybe_private_bucket_prefix: None, + maybe_private_bucket_extension: None, + cached_user_ratings_total_count: 0, + cached_user_ratings_positive_count: 0, + cached_user_ratings_negative_count: 0, + maybe_cached_user_ratings_ratio: None, + version: 0, + mysql_pool: &server_state.mysql_pool, + }).await + .map_err(|e| { + error!("Error creating weights upload: {:?}", e); + UploadError::ServerError + })?; + + info!("new weights upload id: {} token: {:?}", record_id, &token); + + Ok(SuccessCase::WeightSuccessfullyUploaded { + upload_token: token, + }) + } + \ No newline at end of file From af6370eea1e490def21a783e29025774d0add58d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 19:53:52 -0500 Subject: [PATCH 24/58] Issue with the api doc not sure. --- .../web/storyteller_web/src/api_doc.rs | 8 +- .../endpoints/weights/create_weight.rs | 2 - .../endpoints/weights/get_weight.rs | 1 - .../weights/list_availible_weights.rs | 278 ------------------ .../src/http_server/endpoints/weights/mod.rs | 6 +- .../service/web/storyteller_web/src/routes.rs | 6 +- 6 files changed, 10 insertions(+), 291 deletions(-) delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 9db38a306a..40c351e55c 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,7 +1,7 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; -use crate::http_server::endpoints::weights::list_availible_weights::*; +use crate::http_server::endpoints::weights::list_available_weights::*; use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; use crate::http_server::endpoints::weights::get_weight::*; @@ -20,7 +20,7 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, - crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, + crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler, crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, @@ -36,8 +36,8 @@ use utoipa::OpenApi; GetWeightPathInfo,GetWeightResponse,GetWeightError, UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError, DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + ListWeightError,ModelWeightForList,ListWeightError, + ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 5e502e6a57..1bd1a0e50d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -48,7 +48,6 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { // mut multipart_payload: Multipart, // ) -> Result { - // let weight_upload_token = response.to_model_token(); @@ -83,7 +82,6 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { // } // } - // let token = create_weight(CreateModelWeightsArgs { // token: &ModelWeightToken::generate(), // weights_type: WeightsType::LoRA, // to do parse multi part for this diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 1c0857b5b7..6eb2f9849f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -83,7 +83,6 @@ impl ResponseError for GetWeightError { (status = 500, description = "Server error", body = GetWeightError), ), params( - ("request" = UpdateWeightRequest, description = "Payload for Request"), ("path" = GetWeightPathInfo, description = "Path for Request") ) )] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs deleted file mode 100644 index 5ababcdff9..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ /dev/null @@ -1,278 +0,0 @@ -use std::sync::Arc; - -use actix_web::{ HttpRequest, HttpResponse, web }; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; -use chrono::{ DateTime, Utc }; -use log::{ info, warn }; -use rand::Rng; -use enums::common::visibility::Visibility; -use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; - -use enums::by_table::model_weights::{ - weights_types::WeightsType, - weights_category::WeightsCategory, -}; - -use tokens::tokens::users::UserToken; -use tokens::tokens::model_weights::ModelWeightToken; - -use crate::http_server::common_responses::user_details_lite::UserDetailsLight; -use crate::server_state::ServerState; - -use utoipa::ToSchema; - -#[derive(Deserialize,ToSchema)] -pub struct ListAvailibleWeightsQuery { - pub sort_ascending: Option, - pub page_size: u16, - pub page_index : u16, -} - -#[derive(Serialize,ToSchema)] -pub struct ListAvailibleWeightsSuccessResponse { - pub success: bool, - pub weights: Vec, - pub page_size: u16, - pub page_index : u16, - pub cursor_next: Option, - pub cursor_previous: Option, -} - -#[derive(Deserialize,ToSchema)] -pub struct ListWeightsByPathInfo { - pub username: Option, - pub weights_type: Option, - pub weights_category: Option, -} - -#[derive(Serialize,ToSchema)] -pub struct ModelWeightForList { - pub weight_token: ModelWeightToken, - - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, - - pub title: String, - - pub maybe_thumbnail_token: Option, - - pub description_markdown: String, - pub description_rendered_html: String, - - pub creator_user_token: UserToken, - pub creator_set_visibility: Visibility, - - pub file_size_bytes: i32, - pub file_checksum_sha2: String, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - - pub version: i32, - - pub created_at: DateTime, - pub updated_at: DateTime, - - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, - - // additional fields to be added when tables are around - pub likes: u32, - pub bookmarks: bool, - -} - -#[derive(Debug,ToSchema)] -pub enum ListWeightError { - NotAuthorized, - ServerError, -} - -impl std::fmt::Display for ListWeightError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -impl ResponseError for ListWeightError { - fn status_code(&self) -> StatusCode { - match *self { - ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - } - } -} - -#[utoipa::path( - get, - path = "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", - responses( - (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), - (status = 401, description = "Not authorized", body = ListWeightError), - (status = 500, description = "Server error", body = ListWeightError), - ), - params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), - ("path" = ListWeightsByPathInfo, description = "Path for Request") - ) -)] -pub async fn list_availible_weights_handler( - http_request: HttpRequest, - path: web::Path, - query: web::Query, - server_state: web::Data> -) -> Result { - let maybe_user_session = server_state.session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool).await - .map_err(|e| { - warn!("Session checker error: {:?}", e); - ListWeightError::ServerError - })?; - - let mut is_mod = false; - let user_session = match maybe_user_session { - Some(session) => { - is_mod = session.can_ban_users; - session - } - None => { - info!("not logged in"); - return Err(ListWeightError::NotAuthorized); - } - }; - - let limit = query.page_size; - - let sort_ascending = query.sort_ascending.unwrap_or(false); - - let include_user_hidden = is_mod; - - let mut weights_category: Option = None; - let mut weights_type: Option = None; - - if let Some(weights_category_string) = path.weights_category.as_ref() { - let result = WeightsCategory::from_str(&weights_category_string); - match result { - Ok(category) => { - weights_category = Some(category) - }, - Err(e) => { - warn!("invalid weights_category: {:?}", e); - weights_category = None - } - } - } - - if let Some(weights_type_string) = path.weights_type.as_ref() { - let result = WeightsType::from_str(&weights_type_string); - match result { - Ok(wtype) => { - weights_type = Some(wtype) - }, - Err(e) => { - warn!("invalid weights_type: {:?}", e); - weights_type = None - } - } - } - - let mut query_builder = ListWeightsQueryBuilder::new() - .sort_ascending(sort_ascending) - .weights_category(weights_category) - .weights_type(weights_type) - .scope_creator_username(None) - .include_user_hidden(include_user_hidden) - .include_user_deleted_results(is_mod) - .include_mod_deleted_results(is_mod) - .limit(limit) - .offset(Some((query.page_index as u64) * (query.page_size as u64))); - - let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - - let weights_page = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListWeightError::ServerError); - } - }; - - let cursor_next = if let Some(id) = weights_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let cursor_previous = if let Some(id) = weights_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let mut rng = rand::thread_rng(); - let random_float = rng.gen_range(0.0..1.0); - let random_bool = random_float >= 0.5; - - // generate parse a response - let response = ListAvailibleWeightsSuccessResponse { - success: true, - weights: weights_page.weights.into_iter() - .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - weights_type: weights.weights_type, - weights_category: weights.weights_category, - - maybe_thumbnail_token:weights.maybe_thumbnail_token, - description_markdown: weights.description_markdown, - description_rendered_html: weights.description_rendered_html, - - creator_user_token: weights.creator_user_token, - creator_set_visibility: weights.creator_set_visibility, - - file_size_bytes:weights.file_size_bytes, - file_checksum_sha2: weights.file_checksum_sha2, - - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, - cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, - cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, - maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - - version: weights.version, - - created_at: weights.created_at, - updated_at: weights.updated_at, - - creator_username: weights.creator_username, - creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash, - - // TODO FIX THIS when we align. - bookmarks: random_bool, - likes: rng.gen_range(0..1000), - }).collect::>(), - cursor_next, - cursor_previous, - page_index: query.page_index, - page_size: query.page_size - }; - - let body = serde_json::to_string(&response) - .map_err(|e| ListWeightError::ServerError)?; - - Ok(HttpResponse::Ok().content_type("application/json").body(body)) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index f457767f25..3fbd1cb09d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -2,6 +2,6 @@ pub mod get_weight; pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; -pub mod list_availible_weights; -// pub mod create_weight; -// pub mod drain_multipart_weights_request; \ No newline at end of file +pub mod list_available_weights; +// pub mod create_weight; this is a private api call internally +// pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4bf7d273ac..19dc95b7e9 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -155,7 +155,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; -use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; @@ -1266,8 +1266,8 @@ fn add_weights_routes(app: App) -> App ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) .route( - "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", - web::get().to(list_availible_weights_handler) + "/list", + web::get().to(list_available_weights_handler) ) ) } From 330f24413ddcc8c0901385182f748ca891e18d07 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 19:54:21 -0500 Subject: [PATCH 25/58] added missing file --- .../weights/list_available_weights.rs | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs new file mode 100644 index 0000000000..a0c29bc14f --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -0,0 +1,273 @@ +use std::sync::Arc; +use actix_web::web::Query; +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{ DateTime, Utc }; +use log::{ info, warn }; +use rand::Rng; +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; + +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::users::UserToken; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +use utoipa::ToSchema; + + +#[derive(Deserialize,ToSchema)] +pub struct ListAvailableWeightsQuery { + pub sort_ascending: Option, + pub page_size: u16, + pub page_index : u16, + pub username: Option, + pub weights_type: Option, + pub weights_category: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ListAvailableWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, + pub page_size: u16, + pub page_index : u16, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub creator_user_token: UserToken, + pub creator_set_visibility: Visibility, + + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + + pub version: i32, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + // additional fields to be added when tables are around + pub likes: u32, + pub bookmarks: bool, + +} + +#[derive(Debug,ToSchema)] +pub enum ListWeightError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListWeightError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/list/", + responses( + (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightError), + (status = 500, description = "Server error", body = ListWeightError), + ), + params( + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), + ) +)] +pub async fn list_available_weights_handler( + http_request: HttpRequest, + query: web::Query, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightError::ServerError + })?; + + let mut is_mod = false; + let user_session = match maybe_user_session { + Some(session) => { + is_mod = session.can_ban_users; + session + } + None => { + info!("not logged in"); + return Err(ListWeightError::NotAuthorized); + } + }; + + let limit = query.page_size; + + let sort_ascending = query.sort_ascending.unwrap_or(false); + + let include_user_hidden = is_mod; + + let mut weights_category: Option = None; + let mut weights_type: Option = None; + + if let Some(weights_category_string) = query.weights_category.as_ref() { + let result = WeightsCategory::from_str(&weights_category_string); + match result { + Ok(category) => { + weights_category = Some(category) + }, + Err(e) => { + warn!("invalid weights_category: {:?}", e); + weights_category = None + } + } + } + + if let Some(weights_type_string) = query.weights_type.as_ref() { + let result = WeightsType::from_str(&weights_type_string); + match result { + Ok(wtype) => { + weights_type = Some(wtype) + }, + Err(e) => { + warn!("invalid weights_type: {:?}", e); + weights_type = None + } + } + } + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .weights_category(weights_category) + .weights_type(weights_type) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .offset(Some((query.page_index as u64) * (query.page_size as u64))); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; + + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let mut rng = rand::thread_rng(); + let random_float = rng.gen_range(0.0..1.0); + let random_bool = random_float >= 0.5; + + // generate parse a response + let response = ListAvailableWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, + + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, + + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, + + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, + + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, + + version: weights.version, + + created_at: weights.created_at, + updated_at: weights.updated_at, + + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + + // TODO: FIX THIS when we align again. + bookmarks: random_bool, + likes: rng.gen_range(0..1000), + }).collect::>(), + cursor_next, + cursor_previous, + page_index: query.page_index, + page_size: query.page_size + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} From 43b6808f23cf58c08261fea8c9a6352efb05ff6b Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 20:05:23 -0500 Subject: [PATCH 26/58] Ok fixed docs and avaoilable weights endpoint --- .../http_server/endpoints/weights/list_available_weights.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index a0c29bc14f..34962dbd5a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -108,12 +108,12 @@ impl ResponseError for ListWeightError { get, path = "/list/", responses( - (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), (status = 500, description = "Server error", body = ListWeightError), ), params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), + ("request" = ListAvailableWeightsQuery, description = "Payload for Request"), ) )] pub async fn list_available_weights_handler( From f4176bf84a8f8d4d8a552afaeaddb240312936dc Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 21:31:37 -0500 Subject: [PATCH 27/58] Alright should fix the generation now... --- .../src/http_server/endpoints/weights/delete_weight.rs | 2 +- .../src/http_server/endpoints/weights/get_weight.rs | 2 +- .../src/http_server/endpoints/weights/list_available_weights.rs | 2 +- .../src/http_server/endpoints/weights/update_weight.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index 7806fa8583..33f9ac75d7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -76,7 +76,7 @@ impl fmt::Display for DeleteWeightError { #[utoipa::path( delete, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), (status = 400, description = "Bad input", body = DeleteWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 6eb2f9849f..02be96fef3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -75,7 +75,7 @@ impl ResponseError for GetWeightError { } #[utoipa::path( get, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Update", body = GetWeightResponse), (status = 400, description = "Bad input", body = GetWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index 34962dbd5a..f9e8d7f367 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -106,7 +106,7 @@ impl ResponseError for ListWeightError { #[utoipa::path( get, - path = "/list/", + path = "/v1/weights/list/", responses( (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 8ddddca33e..e091c30322 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -74,7 +74,7 @@ impl fmt::Display for UpdateWeightError { // =============== Handler =============== #[utoipa::path( post, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Update", body = SimpleGenericJsonSuccess), (status = 400, description = "Bad input", body = UpdateWeightError), From f327dfcdbb372940d5796c8e6a98cab8b00ddbc2 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 23:27:34 -0500 Subject: [PATCH 28/58] Builds and docs work --- .../web/storyteller_web/src/api_doc.rs | 3 +++ .../weights/list_available_weights.rs | 4 +++- .../endpoints/weights/list_weights_by_user.rs | 21 +++++++++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 40c351e55c..178e8871ae 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -18,12 +18,14 @@ use enums::common::visibility::Visibility; use chrono::{DateTime, Utc}; use utoipa::OpenApi; + #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler, crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, + crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler ),components(schemas( // Tokens UserToken,Visibility, @@ -38,6 +40,7 @@ use utoipa::OpenApi; DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, ListWeightError,ModelWeightForList,ListWeightError, ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, + ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index f9e8d7f367..c41e38de99 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -106,7 +106,7 @@ impl ResponseError for ListWeightError { #[utoipa::path( get, - path = "/v1/weights/list/", + path = "/v1/weights/list", responses( (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), @@ -139,6 +139,8 @@ pub async fn list_available_weights_handler( return Err(ListWeightError::NotAuthorized); } }; + let page_size: u16 = 10; + let page_index : u16 = 0; let limit = query.page_size; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index c478ea68d9..364a14fdde 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -14,8 +14,9 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; +use utoipa::ToSchema; -#[derive(Serialize, Clone)] +#[derive(Serialize, Clone,ToSchema)] pub struct Weight { weight_token: ModelWeightToken, title: String, @@ -41,18 +42,18 @@ pub struct Weight { } -#[derive(Serialize)] +#[derive(Serialize,ToSchema)] pub struct ListWeightsByUserSuccessResponse { pub success: bool, pub weights: Vec, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct ListWeightsByUserPathInfo { username: String, } -#[derive(Debug)] +#[derive(Debug,ToSchema)] pub enum ListWeightsByUserError { NotAuthorized, ServerError, @@ -73,6 +74,18 @@ impl ResponseError for ListWeightsByUserError { } } +#[utoipa::path( + get, + path = "/v1/weights/by_user/{username}", + responses( + (status = 200, description = "List Weights by user", body = ListWeightsByUserSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightsByUserError), + (status = 500, description = "Server error", body = ListWeightsByUserError), + ), + params( + ("path" = ListWeightsByUserPathInfo, description = "Payload for Request"), + ) +)] pub async fn list_weights_by_user_handler( http_request: HttpRequest, path: Path, From b26276cfda47c883abe5e01b1d62c6f28dbc58dd Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 23:36:20 -0500 Subject: [PATCH 29/58] done now for code --- crates/service/web/storyteller_web/src/api_doc.rs | 5 ++++- .../src/http_server/common_responses/user_details_lite.rs | 8 +++++--- .../endpoints/weights/list_available_weights.rs | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 178e8871ae..1d96b220e0 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -6,6 +6,8 @@ use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; use crate::http_server::endpoints::weights::get_weight::*; use crate::http_server::endpoints::weights::update_weight::*; +use crate::http_server::endpoints::weights::list_weights_by_user::*; +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -40,7 +42,8 @@ use utoipa::OpenApi; DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, ListWeightError,ModelWeightForList,ListWeightError, ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, - ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight + ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight, + UserDetailsLight,DefaultAvatarInfo )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index d1353f60f7..c325ce0585 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -2,9 +2,9 @@ use tokens::tokens::users::UserToken; use crate::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username; use crate::user_avatars::default_avatar_from_username::default_avatar_from_username; - +use utoipa::ToSchema; /// Everything we need to refer to a user on the public web interface. -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize,ToSchema)] pub struct UserDetailsLight { /// The token for the user pub user_token: UserToken, @@ -82,7 +82,9 @@ impl UserDetailsLight { } } -#[derive(Clone, Serialize)] + + +#[derive(Clone, Serialize,ToSchema)] pub struct DefaultAvatarInfo { pub image_index: u8, pub color_index: u8, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index c41e38de99..7886625891 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -17,7 +17,7 @@ use enums::by_table::model_weights::{ use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; -use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::http_server::common_responses::user_details_lite::{UserDetailsLight,DefaultAvatarInfo}; use crate::server_state::ServerState; use utoipa::ToSchema; From dda2e0840d509d170c0a5021c3435a0c43807ad5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 7 Dec 2023 00:25:57 -0500 Subject: [PATCH 30/58] Updated docs and weights_types --- .../enums/src/by_table/model_weights/weights_types.rs | 8 ++++++++ .../src/http_server/common_responses/user_details_lite.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 0ef556d927..e094c62874 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -24,6 +24,8 @@ pub enum WeightsType { Tacotron2, #[serde(rename = "loRA")] LoRA, + #[serde(rename = "vall-e")] + VallE, } @@ -37,6 +39,7 @@ impl WeightsType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tt2", Self::LoRA => "loRA", + Self::VallE => "vall-e", } } @@ -49,6 +52,7 @@ impl WeightsType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tt2" => Ok(Self::Tacotron2), "loRA" => Ok(Self::LoRA), + "vall-e" => Ok(Self::VallE), _ => Err(format!("invalid value: {:?}", value)), } } @@ -62,6 +66,7 @@ impl WeightsType { Self::SoVitsSvc, Self::Tacotron2, Self::LoRA, + Self::VallE ]) } } @@ -83,6 +88,7 @@ mod tests { assert_eq!(WeightsType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(WeightsType::Tacotron2.to_str(), "tt2"); assert_eq!(WeightsType::LoRA.to_str(), "loRA"); + assert_eq!(WeightsType::VallE.to_str(), "vall-e"); } #[test] @@ -94,6 +100,7 @@ mod tests { assert_eq!(WeightsType::from_str("so_vits_svc").unwrap(), WeightsType::SoVitsSvc); assert_eq!(WeightsType::from_str("tt2").unwrap(), WeightsType::Tacotron2); assert_eq!(WeightsType::from_str("loRA").unwrap(), WeightsType::LoRA); + assert_eq!(WeightsType::from_str("vall-e").unwrap(), WeightsType::VallE); assert!(WeightsType::from_str("invalid").is_err()); } @@ -108,5 +115,6 @@ mod tests { assert!(variants.contains(&WeightsType::SoVitsSvc)); assert!(variants.contains(&WeightsType::Tacotron2)); assert!(variants.contains(&WeightsType::LoRA)); + assert!(variants.contains(&WeightsType::VallE)); } } diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index c325ce0585..582296e109 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -83,7 +83,6 @@ impl UserDetailsLight { } - #[derive(Clone, Serialize,ToSchema)] pub struct DefaultAvatarInfo { pub image_index: u8, From 8afd10c86459abaeb906a242bba67585f2a1e12b Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 7 Dec 2023 17:52:21 -0500 Subject: [PATCH 31/58] minor fixes to spelling --- .../enums/src/by_table/model_weights/weights_types.rs | 11 +++++------ .../src/queries/model_weights/update_weight.rs | 4 ++-- .../endpoints/weights/list_available_weights.rs | 2 -- .../http_server/endpoints/weights/update_weight.rs | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index e094c62874..905dc39e47 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -6,7 +6,6 @@ use strum::EnumCount; use strum::EnumIter; use utoipa::ToSchema; - #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] pub enum WeightsType { @@ -24,7 +23,7 @@ pub enum WeightsType { Tacotron2, #[serde(rename = "loRA")] LoRA, - #[serde(rename = "vall-e")] + #[serde(rename = "vall_e")] VallE, } @@ -39,7 +38,7 @@ impl WeightsType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tt2", Self::LoRA => "loRA", - Self::VallE => "vall-e", + Self::VallE => "vall_e", } } @@ -52,7 +51,7 @@ impl WeightsType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tt2" => Ok(Self::Tacotron2), "loRA" => Ok(Self::LoRA), - "vall-e" => Ok(Self::VallE), + "vall_e" => Ok(Self::VallE), _ => Err(format!("invalid value: {:?}", value)), } } @@ -88,7 +87,7 @@ mod tests { assert_eq!(WeightsType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(WeightsType::Tacotron2.to_str(), "tt2"); assert_eq!(WeightsType::LoRA.to_str(), "loRA"); - assert_eq!(WeightsType::VallE.to_str(), "vall-e"); + assert_eq!(WeightsType::VallE.to_str(), "vall_e"); } #[test] @@ -100,7 +99,7 @@ mod tests { assert_eq!(WeightsType::from_str("so_vits_svc").unwrap(), WeightsType::SoVitsSvc); assert_eq!(WeightsType::from_str("tt2").unwrap(), WeightsType::Tacotron2); assert_eq!(WeightsType::from_str("loRA").unwrap(), WeightsType::LoRA); - assert_eq!(WeightsType::from_str("vall-e").unwrap(), WeightsType::VallE); + assert_eq!(WeightsType::from_str("vall_e").unwrap(), WeightsType::VallE); assert!(WeightsType::from_str("invalid").is_err()); } diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 9d95767d88..33bb1ace32 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -12,8 +12,8 @@ pub struct UpdateWeightArgs<'a> { pub description_markdown: Option<&'a str>, pub description_rendered_html: Option<&'a str>, pub creator_set_visibility: Option<&'a Visibility>, - pub weight_type: Option, - pub weight_category: Option, + pub weights_type: Option, + pub weights_category: Option, pub mysql_pool: &'a MySqlPool, } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index 7886625891..f191e40410 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -139,8 +139,6 @@ pub async fn list_available_weights_handler( return Err(ListWeightError::NotAuthorized); } }; - let page_size: u16 = 10; - let page_index : u16 = 0; let limit = query.page_size; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index e091c30322..8f69470054 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -147,8 +147,8 @@ pub async fn update_weight_handler( description_markdown: request.description_markdown.as_deref(), description_rendered_html: request.description_rendered_html.as_deref(), creator_set_visibility: request.visibility.as_ref(), - weight_type: request.weight_type.as_deref().map(|s| s.to_string()), - weight_category: request.weight_category.as_deref().map(|s| s.to_string()), + weights_type: request.weight_type.as_deref().map(|s| s.to_string()), + weights_category: request.weight_category.as_deref().map(|s| s.to_string()), }).await; match query_result { From d39962491f596f09fa011f6745ae45e8e75aba5d Mon Sep 17 00:00:00 2001 From: salt Date: Thu, 7 Dec 2023 18:36:51 -0800 Subject: [PATCH 32/58] fix some warnings (mostly unnecessary mut) --- .../src/searches/search_tts_models.rs | 2 +- .../list_available_email_sender_jobs.rs | 2 +- .../job/list_available_generic_inference_jobs.rs | 2 +- .../job/mark_generic_inference_job_failure.rs | 2 +- .../src/queries/media_files/update_media_file.rs | 2 +- .../model_categories/list_categories_query_builder.rs | 6 +++--- .../src/queries/model_weights/update_weight.rs | 2 +- .../list_trending_tts_models.rs | 2 +- .../list_assigned_tts_categories_query_builder.rs | 2 +- .../tts/tts_download_jobs/tts_download_job_queries.rs | 2 +- .../tts/tts_results/list_tts_results_query_builder.rs | 2 +- .../twitch_event_rules/reorder_twitch_event_rules.rs | 2 +- .../src/queries/twitch/twitch_oauth/find.rs | 2 +- .../src/queries/users/user/list_users_query_builder.rs | 10 +++++----- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs index 1ae2cc18ca..30055f1774 100644 --- a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs +++ b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs @@ -27,7 +27,7 @@ pub async fn search_tts_models( let mut response_json = search_response.json::().await?; - let mut hits = response_json.get_mut("hits") + let hits = response_json.get_mut("hits") .map(|hits| hits.take()); let hits = hits.map(|mut hits| { diff --git a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs index 55669b4422..8420295eb5 100644 --- a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs @@ -167,7 +167,7 @@ FROM email_sender_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 4cb3b5fa7e..1f72d607cb 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -336,7 +336,7 @@ FROM generic_inference_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs index a1df90a95a..5a3ea2fb6d 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs @@ -20,7 +20,7 @@ pub async fn mark_generic_inference_job_failure( // statuses: "attempt_failed", "complete_failure", "dead" let mut next_status = "attempt_failed"; - let mut maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { + let maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { let mut reason = reason.trim().to_string(); reason.truncate(512); // Max length of column is 512 reason diff --git a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs index 10ea2024c4..8faa739b0f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs @@ -14,7 +14,7 @@ pub struct UpdateMediaFileArgs<'a> { pub async fn update_media_file(args: UpdateMediaFileArgs<'_>) -> AnyhowResult<()>{ // TODO: enforce checks for idempotency token - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; let query_result = sqlx::query!( r#" UPDATE media_files diff --git a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs index 136751050d..845b7e60f1 100644 --- a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs @@ -184,7 +184,7 @@ impl ListCategoriesQueryBuilder { query = query.bind(model_type); } - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) @@ -239,7 +239,7 @@ LEFT OUTER JOIN users let mut query = "".to_string(); - if let Some(username) = self.scope_creator_user_token.as_deref() { + if let Some(_username) = self.scope_creator_user_token.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.creator_user_token = ?"); first_predicate_added = true; @@ -248,7 +248,7 @@ LEFT OUTER JOIN users } } - if let Some(model_type) = self.scope_model_type.as_deref() { + if let Some(_model_type) = self.scope_model_type.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.model_type = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index cbe0ba55d7..eb95ef40d7 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -16,7 +16,7 @@ pub struct UpdateWeightArgs<'a> { } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; let query_result = sqlx ::query!( diff --git a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs index e81d9c339a..5ec24e4eeb 100644 --- a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs +++ b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs @@ -63,7 +63,7 @@ pub async fn list_trending_tts_models( ]; let query = query_parts.join(" UNION "); - let mut query = sqlx::query_as::<_, RawTrendingModel>(&query); + let query = sqlx::query_as::<_, RawTrendingModel>(&query); let results = query.fetch_all(&mut **mysql_connection).await?; diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs index 23dabea29c..f5bd643639 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs @@ -159,7 +159,7 @@ impl ListAssignedTtsCategoriesQueryBuilder { // NB: Model token is always used for scoping. query = query.bind(&self.scope_tts_model_token); - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs index 490493cf27..0a97d15436 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs @@ -192,7 +192,7 @@ pub async fn mark_tts_upload_job_done( { let status = if success { "complete_success" } else { "complete_failure" }; - let query_result = sqlx::query!( + let _query_result = sqlx::query!( r#" UPDATE tts_model_upload_jobs SET diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs index d11c2b921c..0ed9b446a7 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs @@ -268,7 +268,7 @@ LEFT OUTER JOIN users } } - if let Some(username) = self.scope_creator_username.as_deref() { + if let Some(_username) = self.scope_creator_username.as_deref() { if !first_predicate_added { query.push_str(" WHERE users.username = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs index 1842de1a2a..f2cafbbef1 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs @@ -93,7 +93,7 @@ SET "#, query_inner_positions, query_inner_ip_addresses, user_token, rule_tokens); // NB: This, unfortunately, cannot be statically checked. - let mut query = sqlx::query(&query); + let query = sqlx::query(&query); let result = query.execute(mysql_pool).await; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs index 55d93b4246..015e52f891 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs @@ -145,7 +145,7 @@ impl TwitchOauthTokenFinder { query = query.bind(twitch_username); } - let mut result = query.fetch_optional(mysql_pool).await; + let result = query.fetch_optional(mysql_pool).await; match result { Ok(Some(record)) => { diff --git a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs index e00ed36aa6..d9f623e33e 100644 --- a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs @@ -65,10 +65,10 @@ impl ListUsersQueryBuilder { let internal_results = self.internal_query(mysql_pool).await?; - let mut first_id = internal_results.first() + let first_id = internal_results.first() .map(|raw_result| raw_result.user_id); - let mut last_id = internal_results.last() + let last_id = internal_results.last() .map(|raw_result| raw_result.user_id); let users = internal_results @@ -106,17 +106,17 @@ impl ListUsersQueryBuilder { ) -> AnyhowResult> { let query = self.build_query_string(); - let mut query = sqlx::query_as::<_, UserForListRaw>(&query); + let query = sqlx::query_as::<_, UserForListRaw>(&query); - let mut results = query.fetch_all(mysql_pool) + let results = query.fetch_all(mysql_pool) .await?; Ok(results) } pub fn build_query_string(&self) -> String { - let mut query_string = r#" + let query_string = r#" SELECT id as user_id, token as user_token, From e18b86a55a3b8663ab7ef3f42e5a661906fcbfac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:25:49 -0500 Subject: [PATCH 33/58] everything needs to refer to workplace-hack --- crates/_types/buckets/Cargo.toml | 2 ++ crates/_types/enums/Cargo.toml | 6 +++++- crates/_types/reusable_types/Cargo.toml | 4 +++- crates/_types/tokens/Cargo.toml | 8 +++++++- crates/cli/database_backup/Cargo.toml | 4 +++- crates/cli/database_cleanup/Cargo.toml | 4 +++- crates/cli/dev_database_seed/Cargo.toml | 4 +++- crates/cli/dev_upload_media_file/Cargo.toml | 4 +++- crates/cli/elasticsearch-cli/Cargo.toml | 4 +++- crates/cli/kubectl-pod-cleanup/Cargo.toml | 4 +++- crates/cli/lru-delete-models/Cargo.toml | 4 +++- crates/cli/scrape_test/Cargo.toml | 4 +++- crates/lib/actix_helpers/Cargo.toml | 4 +++- crates/lib/api_clients/cloud_storage/Cargo.toml | 4 +++- crates/lib/api_clients/google_drive_common/Cargo.toml | 4 +++- crates/lib/api_clients/twitch_common/Cargo.toml | 4 +++- crates/lib/bootstrap/Cargo.toml | 4 +++- crates/lib/caching/memory_caching/Cargo.toml | 4 +++- crates/lib/caching/redis_caching/Cargo.toml | 4 +++- crates/lib/collections/Cargo.toml | 2 ++ crates/lib/concurrency/Cargo.toml | 3 +++ crates/lib/cookies/Cargo.toml | 4 +++- crates/lib/crockford/Cargo.toml | 8 +++++++- crates/lib/datetimes/Cargo.toml | 8 +++++++- crates/lib/deprecated/config/Cargo.toml | 4 +++- crates/lib/deprecated/container_common/Cargo.toml | 4 +++- crates/lib/deprecated/http_server_common/Cargo.toml | 4 +++- crates/lib/easyenv/Cargo.toml | 4 +++- crates/lib/email_sender/Cargo.toml | 8 +++++++- crates/lib/errors/Cargo.toml | 2 ++ crates/lib/files/archives/Cargo.toml | 4 +++- crates/lib/files/filesys/Cargo.toml | 5 ++++- crates/lib/files/hashing/Cargo.toml | 4 +++- crates/lib/files/images/Cargo.toml | 4 +++- crates/lib/files/media/Cargo.toml | 5 +++-- crates/lib/files/mimetypes/Cargo.toml | 5 ++++- crates/lib/idempotency/Cargo.toml | 4 +++- crates/lib/jobs_common/Cargo.toml | 4 +++- crates/lib/logging/Cargo.toml | 8 +++++++- crates/lib/mysql_queries/Cargo.toml | 4 +++- crates/lib/password/Cargo.toml | 4 +++- crates/lib/redis_common/Cargo.toml | 4 +++- crates/lib/server_environment/Cargo.toml | 8 +++++++- crates/lib/sqlx_mysql_helpers/Cargo.toml | 5 ++++- crates/lib/storyteller_root/Cargo.toml | 8 +++++++- crates/lib/testing/Cargo.toml | 2 ++ crates/lib/tts_common/Cargo.toml | 4 +++- crates/lib/url_config/Cargo.toml | 4 +++- crates/lib/user_input_common/Cargo.toml | 3 ++- crates/lib/web_scrapers/Cargo.toml | 4 +++- crates/service/job/analytics_job/Cargo.toml | 4 +++- crates/service/job/download_job/Cargo.toml | 4 +++- crates/service/job/email_sender_job/Cargo.toml | 4 +++- crates/service/job/inference_job/Cargo.toml | 4 +++- .../inference_job/config/inference-job.development.env | 1 + crates/service/job/tts_download_job/Cargo.toml | 4 +++- crates/service/job/tts_inference_job/Cargo.toml | 4 +++- crates/service/job/w2l_download_job/Cargo.toml | 4 +++- crates/service/job/w2l_inference_job/Cargo.toml | 4 +++- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 4 +++- crates/service/plugins/billing/Cargo.toml | 4 +++- crates/service/plugins/inference_workloads/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 4 +++- crates/service/web/dummy_service/Cargo.toml | 4 +++- crates/service/web/storyteller_web/Cargo.toml | 1 - crates/service/web/websocket_gateway/Cargo.toml | 6 +++--- 66 files changed, 219 insertions(+), 63 deletions(-) diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml index 80145886da..866ad39853 100644 --- a/crates/_types/buckets/Cargo.toml +++ b/crates/_types/buckets/Cargo.toml @@ -17,6 +17,8 @@ path = "src/lib.rs" # Internal crockford = { path = "../../lib/crockford" } errors = { path = "../../lib/errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index f5c925fa3e..255ed1a634 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -13,6 +13,11 @@ name = "enums" path = "src/lib.rs" [dependencies] +# Internal + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -21,7 +26,6 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 3fee6d8cfa..3ef6dd4af7 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal # (none) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } @@ -26,7 +29,6 @@ serde_derive = "1.0.125" # MySql Database sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # Test serialization diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index 896b2053d6..581f8c65de 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -13,6 +13,13 @@ name = "tokens" path = "src/lib.rs" [dependencies] + +# Internal +# (none) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -22,7 +29,6 @@ rand = "0.8.5" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } diff --git a/crates/cli/database_backup/Cargo.toml b/crates/cli/database_backup/Cargo.toml index 691890dfae..02bd043b23 100644 --- a/crates/cli/database_backup/Cargo.toml +++ b/crates/cli/database_backup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/database_cleanup/Cargo.toml b/crates/cli/database_cleanup/Cargo.toml index 1600fd11f6..16485809b8 100644 --- a/crates/cli/database_cleanup/Cargo.toml +++ b/crates/cli/database_cleanup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index c07d0431ac..ae63bd60b6 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -28,6 +28,9 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated @@ -39,7 +42,6 @@ tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 5831756152..ff572d4512 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -25,6 +25,9 @@ mimetypes = { path = "../../lib/files/mimetypes" } mysql_queries = { path = "../../lib/mysql_queries" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated log = "0.4.14" @@ -34,7 +37,6 @@ log = "0.4.14" tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index f6ca014556..c8f0841c69 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated @@ -41,7 +44,6 @@ elasticsearch = "8.5.0-alpha.1" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/kubectl-pod-cleanup/Cargo.toml b/crates/cli/kubectl-pod-cleanup/Cargo.toml index a0bb79e162..73d842a561 100644 --- a/crates/cli/kubectl-pod-cleanup/Cargo.toml +++ b/crates/cli/kubectl-pod-cleanup/Cargo.toml @@ -18,12 +18,14 @@ config = { path = "../../lib/deprecated/config" } easyenv = { path = "../../lib/easyenv" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile yytimes) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.20" once_cell = "1.18.0" rand = "0.8.5" regex = "1.10.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # Async Runtime # NB: Not using Tokio here since everything is blocking so far. diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml index f7af132bb6..eb08a399a3 100644 --- a/crates/cli/lru-delete-models/Cargo.toml +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -17,10 +17,12 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } logging = { path = "../../lib/logging" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = { version = "4.4.7", features = ["derive"] } walkdir = "2.4.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index 99753086ef..d286897c20 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -18,7 +18,9 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } web_scrapers = { path = "../../lib/web_scrapers" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/actix_helpers/Cargo.toml b/crates/lib/actix_helpers/Cargo.toml index 89c0cbf5d9..4f57ff1785 100644 --- a/crates/lib/actix_helpers/Cargo.toml +++ b/crates/lib/actix_helpers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Actix actix-http = "3.4.0" actix-rt = "2.9.0" @@ -34,7 +37,6 @@ futures-util = "0.3.29" # Serialization serde = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index ee5435b17a..3d4c2d3f77 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal buckets = { path = "../../../_types/buckets" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" @@ -28,4 +31,3 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/google_drive_common/Cargo.toml b/crates/lib/api_clients/google_drive_common/Cargo.toml index d15ae569df..75e1d58c0e 100644 --- a/crates/lib/api_clients/google_drive_common/Cargo.toml +++ b/crates/lib/api_clients/google_drive_common/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } subprocess_common = { path = "../../subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" tempdir = "0.3.7" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/twitch_common/Cargo.toml b/crates/lib/api_clients/twitch_common/Cargo.toml index b377c34ee2..9d1fb2eaa8 100644 --- a/crates/lib/api_clients/twitch_common/Cargo.toml +++ b/crates/lib/api_clients/twitch_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } easyenv = { path = "../../easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" once_cell = "1.18.0" @@ -31,4 +34,3 @@ twitch_oauth2 = "0.6.0-rc.2" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/bootstrap/Cargo.toml b/crates/lib/bootstrap/Cargo.toml index 3baecb1333..e3cfd8376e 100644 --- a/crates/lib/bootstrap/Cargo.toml +++ b/crates/lib/bootstrap/Cargo.toml @@ -19,11 +19,13 @@ easyenv = { path = "../easyenv" } errors = { path = "../errors" } server_environment = { path = "../server_environment" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/caching/memory_caching/Cargo.toml b/crates/lib/caching/memory_caching/Cargo.toml index 06a8da7dfc..22b6ee7736 100644 --- a/crates/lib/caching/memory_caching/Cargo.toml +++ b/crates/lib/caching/memory_caching/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" # In-memory caching lru_time_cache = "0.11.11" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/caching/redis_caching/Cargo.toml b/crates/lib/caching/redis_caching/Cargo.toml index 9f31f5bdbd..a36bc100f1 100644 --- a/crates/lib/caching/redis_caching/Cargo.toml +++ b/crates/lib/caching/redis_caching/Cargo.toml @@ -16,6 +16,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Redis Database r2d2_redis = "0.14.0" @@ -23,7 +26,6 @@ r2d2_redis = "0.14.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml index f74421ec5a..080bd36abb 100644 --- a/crates/lib/collections/Cargo.toml +++ b/crates/lib/collections/Cargo.toml @@ -16,6 +16,8 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/concurrency/Cargo.toml b/crates/lib/concurrency/Cargo.toml index c9511ffd58..4d9e02d580 100644 --- a/crates/lib/concurrency/Cargo.toml +++ b/crates/lib/concurrency/Cargo.toml @@ -14,6 +14,9 @@ path = "src/lib.rs" [dependencies] # Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/cookies/Cargo.toml b/crates/lib/cookies/Cargo.toml index 8abda155b4..6bd0b45ec9 100644 --- a/crates/lib/cookies/Cargo.toml +++ b/crates/lib/cookies/Cargo.toml @@ -17,11 +17,13 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External hmac = "0.10.0" # jwt signing jwt = "0.13.0" sha2 = "0.9.5" # hmac signing -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/crockford/Cargo.toml b/crates/lib/crockford/Cargo.toml index 45fa6a9f04..526e91fb99 100644 --- a/crates/lib/crockford/Cargo.toml +++ b/crates/lib/crockford/Cargo.toml @@ -15,8 +15,14 @@ name = "crockford" path = "src/lib.rs" [dependencies] -rand = "0.8.5" +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +rand = "0.8.5" + [dev-dependencies] # None yet diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 7b69245835..6bb08d5c56 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,9 +13,15 @@ name = "datetimes" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External chrono = { version = "0.4.22", features = ["serde"] } once_cell = "1.18.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # (none) \ No newline at end of file diff --git a/crates/lib/deprecated/config/Cargo.toml b/crates/lib/deprecated/config/Cargo.toml index 6142b3221e..80e5232bb5 100644 --- a/crates/lib/deprecated/config/Cargo.toml +++ b/crates/lib/deprecated/config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" # Internal easyenv = { path = "../../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" url = "2.4.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/container_common/Cargo.toml b/crates/lib/deprecated/container_common/Cargo.toml index 7d0e9cb9dd..32fbc9f9f2 100644 --- a/crates/lib/deprecated/container_common/Cargo.toml +++ b/crates/lib/deprecated/container_common/Cargo.toml @@ -21,10 +21,12 @@ rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } log = "0.4.14" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Serialization serde = { version = "1.0.125", features = ["derive"] } toml = "0.5.8" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 4b7259c477..0a4e4e2159 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" errors = { path = "../../errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -39,7 +42,6 @@ language-tags = "0.3.2" # Parsing single tags richly serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] actix-rt = "2.9.0" diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index 8bfc0d1044..b3e5c1bfee 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -16,11 +16,13 @@ path = "src/lib.rs" # Internal storyteller_root = { path = "../storyteller_root" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index feb580c8c2..020ced75b9 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -10,6 +10,12 @@ path = "src/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Internal errors = { path = "../errors" } -lettre = "0.11.2" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External +lettre = "0.11.2" + diff --git a/crates/lib/errors/Cargo.toml b/crates/lib/errors/Cargo.toml index 6268214f7d..ffa8549b87 100644 --- a/crates/lib/errors/Cargo.toml +++ b/crates/lib/errors/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Consider the dependencies here wisely. # Ideally we have almost nothing as a dependency of this crate. anyhow = "1.0.75" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/files/archives/Cargo.toml b/crates/lib/files/archives/Cargo.toml index d260c780c6..4e02301f23 100644 --- a/crates/lib/files/archives/Cargo.toml +++ b/crates/lib/files/archives/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External zip = "0.6.6" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index bbd7dbf27f..19aae75a68 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -16,11 +16,14 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -log = "0.4.20" # External # KEEP THIS TO A MINIMUM OR NIL. +log = "0.4.20" + [dev-dependencies] # None yet diff --git a/crates/lib/files/hashing/Cargo.toml b/crates/lib/files/hashing/Cargo.toml index adb00e6168..62d47fa10b 100644 --- a/crates/lib/files/hashing/Cargo.toml +++ b/crates/lib/files/hashing/Cargo.toml @@ -17,12 +17,14 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes data-encoding = "2.3.2" md-5 = "0.9.1" # DO NOT USE FOR SERIOUS CRYPTO (we use this for gravatar) ring = "0.17.0-alpha.10" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml index 3327f009b6..ddfe507191 100644 --- a/crates/lib/files/images/Cargo.toml +++ b/crates/lib/files/images/Cargo.toml @@ -19,9 +19,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External image = "0.24.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/files/media/Cargo.toml b/crates/lib/files/media/Cargo.toml index 47d371d0b8..02df59e277 100644 --- a/crates/lib/files/media/Cargo.toml +++ b/crates/lib/files/media/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External log = "0.4.14" @@ -26,7 +29,5 @@ log = "0.4.14" # Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/ symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", rev = "e1a70095c4409478c1ded499fb8dd4afa540fb90", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [dev-dependencies] # None yet diff --git a/crates/lib/files/mimetypes/Cargo.toml b/crates/lib/files/mimetypes/Cargo.toml index 6fe34f2034..0e37ad2df4 100644 --- a/crates/lib/files/mimetypes/Cargo.toml +++ b/crates/lib/files/mimetypes/Cargo.toml @@ -14,9 +14,12 @@ name = "mimetypes" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External infer = "0.11" # Mimetype detection -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + [dev-dependencies] # None yet diff --git a/crates/lib/idempotency/Cargo.toml b/crates/lib/idempotency/Cargo.toml index 578404b0fb..1c927b73ab 100644 --- a/crates/lib/idempotency/Cargo.toml +++ b/crates/lib/idempotency/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # errors = { path = "../../lib/errors" } # (None yet) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External uuid = { version = "0.8.2", features = ["v4"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index 1e6767ea01..98ae1d11d1 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -22,9 +22,11 @@ container_common = { path = "../deprecated/container_common" } errors = { path = "../errors" } redis_common = { path = "../redis_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index bf708a7b84..779659367c 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -14,10 +14,16 @@ name = "logging" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.20" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 4b09061ac0..335aab10f3 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -23,6 +23,9 @@ errors = { path = "../errors" } reusable_types = { path = "../../_types/reusable_types" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -44,5 +47,4 @@ tokio = { version = "1.6.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/password/Cargo.toml b/crates/lib/password/Cargo.toml index f8063637b3..a158566eb1 100644 --- a/crates/lib/password/Cargo.toml +++ b/crates/lib/password/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } #hmac = "0.10.0" # jwt signing #jwt = "0.13.0" #sha2 = "0.9.5" # hmac signing diff --git a/crates/lib/redis_common/Cargo.toml b/crates/lib/redis_common/Cargo.toml index 28aa9fc780..a22cd5841c 100644 --- a/crates/lib/redis_common/Cargo.toml +++ b/crates/lib/redis_common/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal container_common = { path = "../deprecated/container_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" @@ -27,4 +30,3 @@ serde_json = "1.0.108" # TODO: Get rid of in favor of std::time time = "0.3.30" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/server_environment/Cargo.toml b/crates/lib/server_environment/Cargo.toml index 5ece74511a..2124b7857c 100644 --- a/crates/lib/server_environment/Cargo.toml +++ b/crates/lib/server_environment/Cargo.toml @@ -14,8 +14,14 @@ name = "server_environment" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] # None yet diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index 57245fc712..a26c50f6d5 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -16,8 +16,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/storyteller_root/Cargo.toml b/crates/lib/storyteller_root/Cargo.toml index 60143bba40..c32ec7493e 100644 --- a/crates/lib/storyteller_root/Cargo.toml +++ b/crates/lib/storyteller_root/Cargo.toml @@ -13,9 +13,15 @@ name = "storyteller_root" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] serial_test = "2.0.0" tempdir = "0.3.7" diff --git a/crates/lib/testing/Cargo.toml b/crates/lib/testing/Cargo.toml index de25b327df..e90fb429ad 100644 --- a/crates/lib/testing/Cargo.toml +++ b/crates/lib/testing/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 985c8e11fc..d6e4f622b9 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../deprecated/container_common" } user_input_common = { path = "../user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -29,4 +32,3 @@ unicode-segmentation = "1.10.1" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/url_config/Cargo.toml b/crates/lib/url_config/Cargo.toml index 91ee274c39..f458e4cb1a 100644 --- a/crates/lib/url_config/Cargo.toml +++ b/crates/lib/url_config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" http_server_common = { path = "../deprecated/http_server_common" } reusable_types = { path = "../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External actix-web = "4.4.0" anyhow = "1.0.75" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml index d737ce2074..f0a2c6db70 100644 --- a/crates/lib/user_input_common/Cargo.toml +++ b/crates/lib/user_input_common/Cargo.toml @@ -13,10 +13,11 @@ name = "user_input_common" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External once_cell = "1.18.0" pulldown-cmark = "0.9.3" # Markdown regex = "1.5.4" unicode-segmentation = "1.10.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index c9f8bced1e..ef1916e6e7 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" enums = { path = "../../_types/enums" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bytes = "1.3" chrono = { version = "0.4.22", features = ["serde"] } @@ -35,7 +38,6 @@ reqwest = { version = "0.11", features = ["json"] } # HTTP Client # Scraping rss = "2.0.2" scraper = "0.17.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index a1dac56f4d..59f30971d9 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } tokens = { path = "../../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -42,4 +45,3 @@ tokio = { version = "1.6.0", features = ["macros"] } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 42bcfa7769..96598161bc 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -33,6 +33,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } subprocess_common = { path = "../../../lib/subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -60,4 +63,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 965444fc3a..de0c02e9d6 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -41,6 +41,9 @@ testing = { path = "../../../lib/testing" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code # newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -82,4 +85,3 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index d0008b9eac..44b1bb910a 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -38,6 +38,9 @@ subprocess_common = { path = "../../../lib/subprocess_common" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -78,4 +81,3 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 0df22ea669..70deaeb076 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -82,6 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" +#TT2_AS_MEDIA_FILES="true" # ================================= # # # diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 64fe625e3a..7506d7f670 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 6ac55b3d1d..9fbdc2df76 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -27,6 +27,9 @@ memory_caching = { path = "../../../lib/caching/memory_caching" } mysql_queries = { path = "../../../lib/mysql_queries" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -58,4 +61,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 2d3d54d8d0..cfdafda722 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 84c844bd40..34db170d30 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -24,6 +24,9 @@ google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -50,4 +53,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index e9c1bb359a..1fda2a5a73 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -25,6 +25,9 @@ tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -81,7 +84,6 @@ toml = "0.5.8" # Text processing pulldown-cmark = "0.9.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 9623a9f9dc..4e123524e9 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -21,6 +21,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } url_config = { path = "../../../lib/url_config" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -51,7 +54,6 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt async-trait = "0.1.58" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] mockall = "0.11.3" diff --git a/crates/service/plugins/inference_workloads/Cargo.toml b/crates/service/plugins/inference_workloads/Cargo.toml index 6d1f817577..3b3bee9e34 100644 --- a/crates/service/plugins/inference_workloads/Cargo.toml +++ b/crates/service/plugins/inference_workloads/Cargo.toml @@ -14,5 +14,5 @@ path = "src/lib.rs" [dependencies] -# External +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index ca9835c9ae..86db221476 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -31,6 +31,9 @@ server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } user_input_common = { path = "../../../lib/user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -65,4 +68,3 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", r2d2_redis = "0.14.0" byteorder = "1.5.0" strum_macros = "0.25.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 5ed83e2aa9..671723c73b 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -20,6 +20,9 @@ easyenv = { path = "../../../lib/easyenv" } errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" @@ -45,4 +48,3 @@ actix-web = "4.4.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 5b2aea92a1..c57dddc4db 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -163,7 +163,6 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 2d29dcc45b..f492098015 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } redis_common = { path = "../../../lib/redis_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -89,9 +92,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" -# Temporary (just to ask for oauth credentials during dev) -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "2.1.3", features = ["mysql"] } From 90ec14b8d02b78be760b1b777d88f8703cfc42c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:27:10 -0500 Subject: [PATCH 34/58] vendored needs to refer to workplace-hack --- Cargo.lock | 1 + crates/vendor/limitation/Cargo.toml | 3 +++ crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 24d5e60379..c68180726b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3350,6 +3350,7 @@ version = "0.1.1" dependencies = [ "chrono", "futures 0.1.31", + "my-workspace-hack", "redis 0.13.0", "time 0.1.45", "tokio 0.1.22", diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 86fb33b371..7c4015b643 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -23,6 +23,9 @@ futures = "0.1.29" redis = "0.13.0" time = "0.1.42" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + [dev-dependencies] tokio = "0.1.22" version-sync = "0.8.1" diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index d0e3422320..088a3cf568 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -21,6 +21,8 @@ flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } tokio = { version = "1.1.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] From 6033289a36df50cd1916ff48f5d721e26dd0fbf6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:36:39 -0500 Subject: [PATCH 35/58] fix most tests and lints --- .../by_table/model_weights/weights_types.rs | 4 +-- .../src/seeding/model_weights.rs | 32 +++++++++---------- .../tacotron2_v2_early_fakeyou/process_job.rs | 3 -- .../upload_results.rs | 7 ++-- .../tts/vall_e_x/process_job_create_voice.rs | 8 ++--- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 905dc39e47..1b6bbcb132 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -7,7 +7,7 @@ use strum::EnumIter; use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, @@ -106,7 +106,7 @@ mod tests { #[test] fn test_all_variants() { let variants = WeightsType::all_variants(); - assert_eq!(variants.len(), 7); + assert_eq!(variants.len(), 8); assert!(variants.contains(&WeightsType::HifiganTacotron2)); assert!(variants.contains(&WeightsType::RvcV2)); assert!(variants.contains(&WeightsType::StableDiffusion15)); diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 2de742f766..e766086a0b 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -1,22 +1,20 @@ use std::vec; -use buckets::public::media_files::original_file; use log::info; +use sqlx::{MySql, Pool}; -use tokens::tokens::users::UserToken; -use sqlx::{ MySql, Pool }; use enums::by_table::model_weights::{ - weights_types::{WeightsType, self}, - weights_category::{WeightsCategory, self}, + weights_category::WeightsCategory, + weights_types::WeightsType, }; -use errors::{ anyhow, AnyhowResult }; +use enums::common::visibility::Visibility; +use errors::{anyhow, AnyhowResult}; +use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs}; +use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; use crate::seeding::users::HANASHI_USERNAME; -use mysql_queries::queries::model_weights::create_weight::{ create_weight, CreateModelWeightsArgs }; -use tokens::tokens::model_weights::ModelWeightToken; -use enums::common::visibility::Visibility; - -use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { let sd1_5_markdown_description = r#" @@ -326,8 +324,8 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062 let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken weights_type: weights_types, // replace with actual WeightsType - weights_category: weights_category, // replace with actual WeightsCategory - title: title, + weights_category, // replace with actual WeightsCategory + title, maybe_thumbnail_token: Some(thumbnail_token.to_string()), description_markdown: description, description_rendered_html: description_rendered_html.to_string(), @@ -342,11 +340,11 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062 private_bucket_hash: "bucket_hash1".to_string(), maybe_private_bucket_prefix: Some(private_bucket_prefix), maybe_private_bucket_extension: Some(private_bucket_extension), - cached_user_ratings_total_count: cached_user_ratings_total_count, - cached_user_ratings_positive_count: cached_user_ratings_positive_count, - cached_user_ratings_negative_count: cached_user_ratings_negative_count, + cached_user_ratings_total_count, + cached_user_ratings_positive_count, + cached_user_ratings_negative_count, maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32), - version: version, + version, mysql_pool: &mysql_pool, // replace with actual MySqlPool }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 0a3460beb4..c2e16d1e72 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -7,16 +7,13 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; -use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; -use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs index cd5df7ff23..87e9747a3d 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs @@ -1,11 +1,10 @@ use std::path::Path; -use anyhow::anyhow; +use anyhow::anyhow; use log::info; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use buckets::public::media_files::original_file::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; @@ -15,8 +14,8 @@ use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReport use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs}; -use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType; use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result; +use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::FileMetadata; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index 3c3547daf1..f83b5b764c 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -61,16 +61,14 @@ pub async fn process_create_voice( let creator_ip_address = &job.creator_ip_address; - let creator_user_token: UserToken; - - match &job.maybe_creator_user_token { + let creator_user_token = match &job.maybe_creator_user_token { Some(token) => { - creator_user_token = UserToken::new_from_str(token); + UserToken::new_from_str(token) }, None => { return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))); } - } + }; // STEP 1. SETUP A TEMP DIRECTORY let work_temp_dir = format!("/tmp/temp_zeroshot_create_voice_{}", job.id.0); From c811296bdcfe5534cc59600e35c55e3e870aa74f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 02:04:34 -0500 Subject: [PATCH 36/58] temporarily disable test --- .../src/queries/model_weights/tests.rs | 26 +++++++------------ execute_tests.sh | 6 +++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 917fdd98b3..3239e4173e 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -1,38 +1,32 @@ #[cfg(test)] mod tests { - use std::result; - use anyhow::Ok; - use chrono::offset; - use enums::by_table::model_weights::{ weights_types, weights_category }; - use sqlx::{ MySqlPool, Acquire }; use rand::Rng; - - use tokio; use serial_test::serial; - use container_common::anyhow_result::AnyhowResult; + use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; - use config::shared_constants::{ DEFAULT_MYSQL_CONNECTION_STRING }; + use tokio; + + use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; + use container_common::anyhow_result::AnyhowResult; // common tests use enums::by_table::model_weights::{ - weights_types::WeightsType, weights_category::WeightsCategory, + weights_types::WeightsType, }; use enums::common::visibility::Visibility; - use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken}; - use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; use crate::queries::model_weights::create_weight::create_weight; - use crate::queries::model_weights::get_weight::get_weight_by_token; - + use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; use crate::queries::model_weights::delete_weights::{ - delete_weights_as_user, delete_weights_as_mod, + delete_weights_as_user, undelete_weights_as_mod, undelete_weights_as_user, }; + use crate::queries::model_weights::get_weight::get_weight_by_token; use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; - use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username; diff --git a/execute_tests.sh b/execute_tests.sh index 319503eb31..ad5219c463 100755 --- a/execute_tests.sh +++ b/execute_tests.sh @@ -13,6 +13,7 @@ cargo test \ --exclude billing_component \ --exclude fakeyou_client \ --exclude inference-job \ + --exclude mysql_queries \ --exclude sqlite_queries # Single broken tests @@ -22,4 +23,9 @@ cargo test \ --skip util::audiowmark::tests::test_audiowmark \ --skip util::common_commands::audiowmark::tests::test_audiowmark +cargo test \ + --package mysql_queries \ + -- \ + --skip queries::model_weights::tests + echo 'Tests passed.' From 4a806e7d67c8c469f6dc8cde569906caf8345ab0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 02:16:50 -0500 Subject: [PATCH 37/58] codegen db --- ...c1837d645d05198d483564b39a2a8b8034df25e287a4.json | 12 ------------ ...07e053b0393a0e341caf652c18c7160dd0a59247a19a.json | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 .sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json create mode 100644 .sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json diff --git a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json b/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json deleted file mode 100644 index b6a7dbec85..0000000000 --- a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\n UPDATE model_weights\n SET\n title = ?,\n description_markdown = ?,\n maybe_thumbnail_token = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n version = version + 1\n WHERE token = ?\n LIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - }, - "hash": "4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4" -} diff --git a/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json new file mode 100644 index 0000000000..cc5ca52c8c --- /dev/null +++ b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\n UPDATE model_weights\n SET\n title = COALESCE(?, title),\n description_markdown = COALESCE(?, description_markdown),\n maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token),\n description_rendered_html = COALESCE(?, description_rendered_html),\n creator_set_visibility = COALESCE(?, creator_set_visibility),\n version = version + 1\n WHERE token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a" +} From ab1f81036f6b877b1cc7b2b6ba3b181dc4909669 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:00:42 -0500 Subject: [PATCH 38/58] update my-workspace-hack --- Cargo.lock | 3 +- crates/my-workspace-hack/Cargo.toml | 64 ++++++++++++++++------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c68180726b..1aaf8d3a68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3755,11 +3755,11 @@ dependencies = [ "generic-array", "getrandom 0.1.16", "getrandom 0.2.11", - "hashbrown 0.12.3", "hashbrown 0.14.2", "heck", "hmac 0.12.1", "hyper", + "indexmap", "lazy_static", "libc", "log", @@ -3807,6 +3807,7 @@ dependencies = [ "void", "winapi 0.3.9", "windows-sys 0.48.0", + "zip", ] [[package]] diff --git a/crates/my-workspace-hack/Cargo.toml b/crates/my-workspace-hack/Cargo.toml index 22f41a903d..dddf543e0b 100644 --- a/crates/my-workspace-hack/Cargo.toml +++ b/crates/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -40,12 +40,12 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } getrandom-c65f7effa3be6d31 = { package = "getrandom", version = "0.1.16", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } hyper = { version = "0.14.27", features = ["client", "http1", "http2", "runtime", "server"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } miniz_oxide = { version = "0.7.1", features = ["simd"] } @@ -67,19 +67,20 @@ sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } simd-adler32 = { version = "0.3.7" } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } -sqlx = { version = "0.7.2", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx = { version = "0.7.3", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } time = { version = "0.3.30", features = ["macros", "serde-well-known"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } twitch_oauth2 = { version = "0.6.1", features = ["reqwest_client"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } void = { version = "1.0.2" } +zip = { version = "0.6.6" } [build-dependencies] ahash = { version = "0.8.6" } @@ -88,11 +89,13 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } +deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0", features = ["serde"] } +flate2 = { version = "1.0.28" } flume = { version = "0.11.0", default-features = false, features = ["async"] } futures-channel = { version = "0.3.29", features = ["sink"] } futures-core = { version = "0.3.29" } @@ -102,13 +105,15 @@ futures-sink = { version = "0.3.29" } futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } heck = { version = "0.4.1", features = ["unicode"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } +miniz_oxide = { version = "0.7.1", features = ["simd"] } mio = { version = "0.8.9", features = ["net", "os-ext"] } native-tls = { version = "0.2.11", default-features = false, features = ["vendored"] } num-integer = { version = "0.1.45", features = ["i128"] } @@ -124,30 +129,33 @@ serde = { version = "1.0.193", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-macros = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-macros-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +simd-adler32 = { version = "0.3.7" } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-macros = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-macros-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.39", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +time = { version = "0.3.30", features = ["macros", "serde-well-known"] } time-macros = { version = "0.2.15", default-features = false, features = ["formatting", "parsing", "serde"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } +zip = { version = "0.6.6" } [target.x86_64-unknown-linux-gnu.dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-unknown-linux-gnu.build-dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } @@ -163,13 +171,13 @@ spin = { version = "0.9.8" } [target.x86_64-pc-windows-msvc.dependencies] spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } [target.x86_64-pc-windows-msvc.build-dependencies] socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } ### END HAKARI SECTION From b4afec5c2637e3af96b08509cced3e49b6557131 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:21:16 -0500 Subject: [PATCH 39/58] rename endpoint (internally) --- ...animation.rs => enqueue_face_animation.rs} | 52 +++++++++---------- .../http_server/endpoints/animation/mod.rs | 2 +- .../service/web/storyteller_web/src/routes.rs | 4 +- 3 files changed, 29 insertions(+), 29 deletions(-) rename crates/service/web/storyteller_web/src/http_server/endpoints/animation/{enqueue_lipsync_animation.rs => enqueue_face_animation.rs} (86%) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs similarity index 86% rename from crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs index 8d1e536dcc..f9c54eed05 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs @@ -37,7 +37,7 @@ const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; #[derive(Deserialize)] -pub struct EnqueueLipsyncAnimationRequest { +pub struct EnqueueFaceAnimationRequest { uuid_idempotency_token: String, audio_source: AudioSource, @@ -117,35 +117,35 @@ pub struct ImageSource { } #[derive(Serialize)] -pub struct EnqueueLipsyncAnimationSuccessResponse { +pub struct EnqueueFaceAnimationSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] -pub enum EnqueueLipsyncAnimationError { +pub enum EnqueueFaceAnimationError { BadInput(String), NotAuthorized, ServerError, RateLimited, } -impl ResponseError for EnqueueLipsyncAnimationError { +impl ResponseError for EnqueueFaceAnimationError { fn status_code(&self) -> StatusCode { match *self { - EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, - EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, - EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + EnqueueFaceAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueFaceAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueFaceAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueFaceAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(), - EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(), - EnqueueLipsyncAnimationError::ServerError => "server error".to_string(), - EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(), + EnqueueFaceAnimationError::BadInput(reason) => reason.to_string(), + EnqueueFaceAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueFaceAnimationError::ServerError => "server error".to_string(), + EnqueueFaceAnimationError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -153,16 +153,16 @@ impl ResponseError for EnqueueLipsyncAnimationError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl std::fmt::Display for EnqueueLipsyncAnimationError { +impl std::fmt::Display for EnqueueFaceAnimationError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) } } -pub async fn enqueue_lipsync_animation_handler( +pub async fn enqueue_face_animation_handler( http_request: HttpRequest, - request: web::Json, - server_state: web::Data>) -> Result + request: web::Json, + server_state: web::Data>) -> Result { let mut maybe_user_token : Option = None; @@ -171,7 +171,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); @@ -184,7 +184,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; if let Some(user_session) = maybe_user_session.as_ref() { @@ -215,14 +215,14 @@ pub async fn enqueue_lipsync_animation_handler( None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { if user.role.is_banned { - return Err(EnqueueLipsyncAnimationError::NotAuthorized); + return Err(EnqueueFaceAnimationError::NotAuthorized); } &server_state.redis_rate_limiters.logged_in }, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(EnqueueLipsyncAnimationError::RateLimited); + return Err(EnqueueFaceAnimationError::RateLimited); } // ==================== LOOK UP MODEL INFO ==================== // @@ -239,7 +239,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token { LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("audio source not fully specified".to_string())); } }; @@ -249,7 +249,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.image_source.maybe_media_upload_token { LipsyncAnimationImageSource::media_upload_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("image source not fully specified".to_string())); } }; @@ -329,7 +329,7 @@ pub async fn enqueue_lipsync_animation_handler( Ok((job_token, _id)) => job_token, Err(err) => { warn!("New generic inference job creation DB error: {:?}", err); - return Err(EnqueueLipsyncAnimationError::ServerError); + return Err(EnqueueFaceAnimationError::ServerError); } }; @@ -341,16 +341,16 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("error publishing event: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; - let response: EnqueueLipsyncAnimationSuccessResponse = EnqueueLipsyncAnimationSuccessResponse { + let response: EnqueueFaceAnimationSuccessResponse = EnqueueFaceAnimationSuccessResponse { success: true, inference_job_token: job_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?; + .map_err(|_e| EnqueueFaceAnimationError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs index 1102d9357d..dd4f619b5e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -1 +1 @@ -pub mod enqueue_lipsync_animation; \ No newline at end of file +pub mod enqueue_face_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 19dc95b7e9..eb67880dd5 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -11,7 +11,7 @@ use users_component::default_routes::add_suggested_api_v1_account_creation_and_s use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; -use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler; +use crate::http_server::endpoints::animation::enqueue_face_animation::enqueue_face_animation_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; @@ -215,7 +215,7 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> // ==================== Animations ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler) + .add_post("/v1/animation/face_animation/create", enqueue_face_animation_handler) .into_app(); // ==================== "Generic" Inference ==================== From 996f8ce05b1b13d98911b097caedd4f0c6a5260e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:38:46 -0500 Subject: [PATCH 40/58] inference-job : sad talker to use audio media files --- .../lipsync/sad_talker/download_audio_file.rs | 166 +++++++++++------- .../lipsync/sad_talker/process_job.rs | 18 +- 2 files changed, 114 insertions(+), 70 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 19c0d958c9..5e6c8c7d47 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; @@ -11,8 +12,10 @@ use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; use mysql_queries::queries::voice_conversion::results::get_voice_conversion_result_for_inference::get_voice_conversion_result_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -23,86 +26,47 @@ pub struct AudioFile { pub filesystem_path: PathBuf, } -pub async fn download_audio_file( - audio_source: &LipsyncAnimationAudioSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadAudioFileArgs<'a> { + pub audio_source: &'a LipsyncAnimationAudioSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result { let bucket_object_path; - match audio_source { + match args.audio_source { LipsyncAnimationAudioSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationAudioSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } - LipsyncAnimationAudioSource::T(tts_result_token) => { - // TODO(bt, 2023-09-08): Implement + LipsyncAnimationAudioSource::T(_tts_result_token) => { + // NB(bt,2023-12-08): This will likely never be implemented now that TTS is on media files. return Err(ProcessSingleJobError::NotYetImplemented) } LipsyncAnimationAudioSource::V(voice_conversion_result_token) => { - let voice_conversion_result = get_voice_conversion_result_for_inference( - voice_conversion_result_token, - false, - &mysql_pool, - ).await; - - let voice_conversion_result = match voice_conversion_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) - } - Err(e) => { - error!("could not query voice conversion result: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); - bucket_object_path = result_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_voice_conversion_result(voice_conversion_result_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("audio.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "audio file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -110,3 +74,83 @@ pub async fn download_audio_file( filesystem_path: downloaded_filesystem_path, }) } + + +async fn from_media_file(media_file_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file: {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} + +async fn from_voice_conversion_result(voice_conversion_result_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let voice_conversion_result = get_voice_conversion_result_for_inference( + voice_conversion_result_token, + false, + &args.mysql_pool, + ).await; + + let voice_conversion_result = match voice_conversion_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) + } + Err(e) => { + error!("could not query voice conversion result: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); + + Ok(result_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 14fd9a32e0..0d75a844bb 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -19,7 +19,7 @@ use tokens::tokens::users::UserToken; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; -use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; +use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs}; use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; @@ -97,15 +97,15 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result Date: Fri, 8 Dec 2023 04:45:28 -0500 Subject: [PATCH 41/58] inference-job - sadtalker to work with image media files --- .../lipsync/sad_talker/download_audio_file.rs | 3 +- .../lipsync/sad_talker/download_image_file.rs | 113 ++++++++++++------ .../lipsync/sad_talker/process_job.rs | 20 ++-- 3 files changed, 86 insertions(+), 50 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 5e6c8c7d47..4e6f8973e7 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -4,8 +4,8 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; use cloud_storage::bucket_client::BucketClient; @@ -75,7 +75,6 @@ pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result) -> Result { let media_file_token = MediaFileToken::new_from_str(media_file_token); let media_file_result = get_media_file( diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 6ac3b4d3e9..3d401f0a7a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -5,12 +5,15 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -21,59 +24,40 @@ pub struct ImageFile { pub filesystem_path: PathBuf, } -pub async fn download_image_file( - image_source: &LipsyncAnimationImageSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadImageFileArgs<'a> { + pub image_source: &'a LipsyncAnimationImageSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_image_file(args: DownloadImageFileArgs<'_>) -> Result { let bucket_object_path; - match image_source { + match args.image_source { LipsyncAnimationImageSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationImageSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("image.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("image.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "image file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -81,3 +65,56 @@ pub async fn download_image_file( filesystem_path: downloaded_filesystem_path, }) } + +async fn from_media_file(media_file_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file : {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file : {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 0d75a844bb..cb6d563156 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -20,7 +20,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs}; -use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; +use crate::job::job_types::lipsync::sad_talker::download_image_file::{download_image_file, DownloadImageFileArgs}; use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; @@ -107,17 +107,17 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result Date: Fri, 8 Dec 2023 05:15:13 -0500 Subject: [PATCH 42/58] env vars --- .env | 3 ++- .../job/inference_job/config/inference-job.development.env | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env b/.env index b205c3a6b3..a39f1b1d8d 100644 --- a/.env +++ b/.env @@ -75,4 +75,5 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt # Development key. Permissible to check in. NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" \ No newline at end of file + +#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 70deaeb076..c16128c8d6 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -82,7 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" -#TT2_AS_MEDIA_FILES="true" +TT2_AS_MEDIA_FILES="true" # ================================= # # # From 5971d95a8052bbd04a8f88262ae415c9bd320f1c Mon Sep 17 00:00:00 2001 From: kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:57:55 +0530 Subject: [PATCH 43/58] Rename favorites -> user_bookmarks, add new table --- ...ebbdb76045ad26aeb1c517a54c296bf31087.json} | 8 +- ...c6fa65ede80d9efcaa4996694ce32c12cc0c.json} | 6 +- ...0b5a04e33d7c6b9c38bef96f275b064a12386.json | 12 ++ ...04d73b9f4a649098fc0ba78cbc20cfde12437.json | 12 -- ...dfe33c94b61046c30940f55cdc73789f94041.json | 12 -- ...1efd0cfa6b20899151f9e7f1a122c1f7a70e.json} | 8 +- ...c352624b79bd4e097f57db5ad59f53fd66995.json | 12 ++ ...449362c074fc2397e23ecd175b6a0425b16b.json} | 8 +- .../down.sql | 5 + .../up.sql | 46 +++++++ .../favorites/favorite_entity_type.rs | 128 ------------------ .../enums/src/by_table/favorites/mod.rs | 1 - crates/_types/enums/src/by_table/mod.rs | 2 +- .../enums/src/by_table/user_bookmarks/mod.rs | 1 + .../user_bookmark_entity_type.rs | 128 ++++++++++++++++++ crates/_types/tokens/src/prefixes.rs | 4 +- crates/_types/tokens/src/tokens/mod.rs | 2 +- .../{favorites.rs => user_bookmarks.rs} | 6 +- .../favorites/favorite_entity_token.rs | 48 ------- .../src/queries/favorites/mod.rs | 7 - crates/lib/mysql_queries/src/queries/mod.rs | 2 +- .../create_user_bookmark.rs} | 22 +-- .../delete_user_bookmark.rs} | 16 +-- .../get_user_bookmark.rs} | 56 ++++---- .../list_user_bookmarks.rs} | 44 +++--- .../list_user_bookmarks_for_entity.rs} | 36 ++--- .../list_user_bookmarks_result.rs} | 40 +++--- .../src/queries/user_bookmarks/mod.rs | 7 + .../user_bookmark_entity_token.rs | 48 +++++++ .../list_favorites_for_entity_handler.rs | 122 ----------------- .../http_server/endpoints/favorites/mod.rs | 5 - .../src/http_server/endpoints/mod.rs | 2 +- .../create_user_bookmark_handler.rs} | 62 ++++----- .../delete_user_bookmark_handler.rs} | 52 +++---- .../list_user_bookmarks_for_entity_handler.rs | 122 +++++++++++++++++ ...ist_user_bookmarks_for_session_handler.rs} | 76 +++++------ .../list_user_bookmarks_for_user_handler.rs} | 74 +++++----- .../endpoints/user_bookmarks/mod.rs | 5 + .../service/web/storyteller_web/src/routes.rs | 22 +-- 39 files changed, 661 insertions(+), 608 deletions(-) rename .sqlx/{query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json => query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json} (76%) rename .sqlx/{query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json => query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json} (76%) create mode 100644 .sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json delete mode 100644 .sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json delete mode 100644 .sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json rename .sqlx/{query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json => query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json} (70%) create mode 100644 .sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json rename .sqlx/{query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json => query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json} (70%) create mode 100644 _sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql create mode 100644 _sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql delete mode 100644 crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs delete mode 100644 crates/_types/enums/src/by_table/favorites/mod.rs create mode 100644 crates/_types/enums/src/by_table/user_bookmarks/mod.rs create mode 100644 crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs rename crates/_types/tokens/src/tokens/{favorites.rs => user_bookmarks.rs} (59%) delete mode 100644 crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs delete mode 100644 crates/lib/mysql_queries/src/queries/favorites/mod.rs rename crates/lib/mysql_queries/src/queries/{favorites/create_favorite.rs => user_bookmarks/create_user_bookmark.rs} (67%) rename crates/lib/mysql_queries/src/queries/{favorites/delete_favorite.rs => user_bookmarks/delete_user_bookmark.rs} (61%) rename crates/lib/mysql_queries/src/queries/{favorites/get_favorite.rs => user_bookmarks/get_user_bookmark.rs} (51%) rename crates/lib/mysql_queries/src/queries/{favorites/list_user_favorites.rs => user_bookmarks/list_user_bookmarks.rs} (75%) rename crates/lib/mysql_queries/src/queries/{favorites/list_favorites_for_entity.rs => user_bookmarks/list_user_bookmarks_for_entity.rs} (67%) rename crates/lib/mysql_queries/src/queries/{favorites/list_user_favorites_result.rs => user_bookmarks/list_user_bookmarks_result.rs} (66%) create mode 100644 crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs create mode 100644 crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/create_favorite_handler.rs => user_bookmarks/create_user_bookmark_handler.rs} (52%) rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/delete_favorite_handler.rs => user_bookmarks/delete_user_bookmark_handler.rs} (58%) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/list_favorites_for_session_handler.rs => user_bookmarks/list_user_bookmarks_for_session_handler.rs} (51%) rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/list_favorites_for_user_handler.rs => user_bookmarks/list_user_bookmarks_for_user_handler.rs} (51%) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs diff --git a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json similarity index 76% rename from .sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json rename to .sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json index 0115c0bc84..047b14d7ff 100644 --- a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json +++ b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -120,5 +120,5 @@ true ] }, - "hash": "f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d" + "hash": "02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087" } diff --git a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json similarity index 76% rename from .sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json rename to .sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json index c4b6c41fdb..48546a186c 100644 --- a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json +++ b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -98,5 +98,5 @@ true ] }, - "hash": "99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16" + "hash": "1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c" } diff --git a/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json new file mode 100644 index 0000000000..c829a96d4d --- /dev/null +++ b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nUPDATE user_bookmarks\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386" +} diff --git a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json b/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json deleted file mode 100644 index fefdbaf494..0000000000 --- a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nUPDATE favorites\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437" -} diff --git a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json b/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json deleted file mode 100644 index 01e7177d85..0000000000 --- a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO favorites\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041" -} diff --git a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json similarity index 70% rename from .sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json rename to .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json index 2c4133a738..33334d1769 100644 --- a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json +++ b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038" + "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" } diff --git a/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json new file mode 100644 index 0000000000..d2dc52c79e --- /dev/null +++ b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO user_bookmarks\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + }, + "hash": "e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995" +} diff --git a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json similarity index 70% rename from .sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json rename to .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json index 4ebd7fa022..55d497beee 100644 --- a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json +++ b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4" + "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" } diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql new file mode 100644 index 0000000000..8b6d041461 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE user_bookmarks; \ No newline at end of file diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql new file mode 100644 index 0000000000..54ad3a50b0 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql @@ -0,0 +1,46 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Favorites that can be attached to just about anything. +-- This is a polymorphic table and can apply to a wide variety of entities. +CREATE TABLE user_bookmarks +( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- User making the favorite + user_token VARCHAR(32) NOT NULL, + + -- The type of entity the favorite applies to + entity_type VARCHAR(32) NOT NULL, + + -- The token of the entity the favorite applies to + entity_token VARCHAR(32) NOT NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY index_entity_type_entity_token (entity_type, entity_token), + KEY index_user_token_entity_token (user_token, entity_type), + KEY index_user_token (user_token), + KEY index_created_at (created_at), + KEY index_user_deleted_at (deleted_at) + +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; diff --git a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs b/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs deleted file mode 100644 index 5b96603ff1..0000000000 --- a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs +++ /dev/null @@ -1,128 +0,0 @@ -use serde::Deserialize; -use serde::Serialize; -#[cfg(test)] -use strum::EnumCount; -#[cfg(test)] -use strum::EnumIter; - -/// Used in the `favorites` table in a `VARCHAR(32)` field named `entity_type`. -#[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] -pub enum FavoriteEntityType { - /// User - #[serde(rename = "user")] - User, - - /// TTS model (architecture does not matter) - #[serde(rename = "tts_model")] - TtsModel, - - /// TTS result (architecture does not matter) - #[serde(rename = "tts_result")] - TtsResult, - - /// W2L template - #[serde(rename = "w2l_template")] - W2lTemplate, - - /// W2L result - #[serde(rename = "w2l_result")] - W2lResult, - - /// MediaFile - #[serde(rename = "media_file")] - MediaFile, - - /// VoiceConversionModel - #[serde(rename = "voice_conversion_model")] - VoiceConversionModel, - - /// ZsVoice - #[serde(rename = "zs_voice")] - ZsVoice, -} - -// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! -impl_enum_display_and_debug_using_to_str!(FavoriteEntityType); -impl_mysql_enum_coders!(FavoriteEntityType); - -/// NB: Legacy API for older code. -impl FavoriteEntityType { - pub fn to_str(&self) -> &'static str { - match self { - Self::User => "user", - Self::TtsModel => "tts_model", - Self::TtsResult => "tts_result", - Self::W2lTemplate => "w2l_template", - Self::W2lResult => "w2l_result", - Self::MediaFile => "media_file", - Self::VoiceConversionModel => "voice_conversion_model", - Self::ZsVoice => "zs_voice", - } - } - - pub fn from_str(value: &str) -> Result { - match value { - "user" => Ok(Self::User), - "tts_model" => Ok(Self::TtsModel), - "tts_result" => Ok(Self::TtsResult), - "w2l_template" => Ok(Self::W2lTemplate), - "w2l_result" => Ok(Self::W2lResult), - "media_file" => Ok(Self::MediaFile), - "voice_conversion_model" => Ok(Self::VoiceConversionModel), - "zs_voice" => Ok(Self::ZsVoice), - _ => Err(format!("invalid value: {:?}", value)), - } - } -} - -#[cfg(test)] -mod tests { - use crate::by_table::favorites::favorite_entity_type::FavoriteEntityType; - use crate::test_helpers::assert_serialization; - - mod serde { - use super::*; - - #[test] - fn test_serialization() { - assert_serialization(FavoriteEntityType::User, "user"); - assert_serialization(FavoriteEntityType::TtsModel, "tts_model"); - assert_serialization(FavoriteEntityType::TtsResult, "tts_result"); - assert_serialization(FavoriteEntityType::W2lTemplate, "w2l_template"); - assert_serialization(FavoriteEntityType::W2lResult, "w2l_result"); - assert_serialization(FavoriteEntityType::MediaFile, "media_file"); - assert_serialization(FavoriteEntityType::VoiceConversionModel, "voice_conversion_model"); - assert_serialization(FavoriteEntityType::ZsVoice, "zs_voice"); - } - } - - mod impl_methods { - use super::*; - - #[test] - fn test_to_str() { - assert_eq!(FavoriteEntityType::User.to_str(), "user"); - assert_eq!(FavoriteEntityType::TtsModel.to_str(), "tts_model"); - assert_eq!(FavoriteEntityType::TtsResult.to_str(), "tts_result"); - assert_eq!(FavoriteEntityType::W2lTemplate.to_str(), "w2l_template"); - assert_eq!(FavoriteEntityType::W2lResult.to_str(), "w2l_result"); - assert_eq!(FavoriteEntityType::MediaFile.to_str(), "media_file"); - assert_eq!(FavoriteEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); - assert_eq!(FavoriteEntityType::ZsVoice.to_str(), "zs_voice"); - } - - #[test] - fn test_from_str() { - assert_eq!(FavoriteEntityType::from_str("user").unwrap(), FavoriteEntityType::User); - assert_eq!(FavoriteEntityType::from_str("tts_model").unwrap(), FavoriteEntityType::TtsModel); - assert_eq!(FavoriteEntityType::from_str("tts_result").unwrap(), FavoriteEntityType::TtsResult); - assert_eq!(FavoriteEntityType::from_str("w2l_template").unwrap(), FavoriteEntityType::W2lTemplate); - assert_eq!(FavoriteEntityType::from_str("w2l_result").unwrap(), FavoriteEntityType::W2lResult); - assert_eq!(FavoriteEntityType::from_str("media_file").unwrap(), FavoriteEntityType::MediaFile); - assert_eq!(FavoriteEntityType::from_str("voice_conversion_model").unwrap(), FavoriteEntityType::VoiceConversionModel); - assert_eq!(FavoriteEntityType::from_str("zs_voice").unwrap(), FavoriteEntityType::ZsVoice); - assert!(FavoriteEntityType::from_str("foo").is_err()); - } - } -} diff --git a/crates/_types/enums/src/by_table/favorites/mod.rs b/crates/_types/enums/src/by_table/favorites/mod.rs deleted file mode 100644 index 18ac93ed9c..0000000000 --- a/crates/_types/enums/src/by_table/favorites/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod favorite_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/mod.rs b/crates/_types/enums/src/by_table/mod.rs index 91f8cb15da..97876b2094 100644 --- a/crates/_types/enums/src/by_table/mod.rs +++ b/crates/_types/enums/src/by_table/mod.rs @@ -10,6 +10,7 @@ pub mod media_uploads; pub mod model_categories; pub mod trending_model_analytics; pub mod tts_models; +pub mod user_bookmarks; pub mod user_ratings; pub mod voice_conversion_models; pub mod voice_conversion_results; @@ -19,4 +20,3 @@ pub mod model_weights; pub mod tts_render_tasks; pub mod web_rendition_targets; pub mod web_scraping_targets; -pub mod favorites; diff --git a/crates/_types/enums/src/by_table/user_bookmarks/mod.rs b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs new file mode 100644 index 0000000000..c3c539aad3 --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs @@ -0,0 +1 @@ +pub mod user_bookmark_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs new file mode 100644 index 0000000000..69bf47cf5e --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs @@ -0,0 +1,128 @@ +use serde::Deserialize; +use serde::Serialize; +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the `user_bookmarks` table in a `VARCHAR(32)` field named `entity_type`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum UserBookmarkEntityType { + /// User + #[serde(rename = "user")] + User, + + /// TTS model (architecture does not matter) + #[serde(rename = "tts_model")] + TtsModel, + + /// TTS result (architecture does not matter) + #[serde(rename = "tts_result")] + TtsResult, + + /// W2L template + #[serde(rename = "w2l_template")] + W2lTemplate, + + /// W2L result + #[serde(rename = "w2l_result")] + W2lResult, + + /// MediaFile + #[serde(rename = "media_file")] + MediaFile, + + /// VoiceConversionModel + #[serde(rename = "voice_conversion_model")] + VoiceConversionModel, + + /// ZsVoice + #[serde(rename = "zs_voice")] + ZsVoice, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(UserBookmarkEntityType); +impl_mysql_enum_coders!(UserBookmarkEntityType); + +/// NB: Legacy API for older code. +impl UserBookmarkEntityType { + pub fn to_str(&self) -> &'static str { + match self { + Self::User => "user", + Self::TtsModel => "tts_model", + Self::TtsResult => "tts_result", + Self::W2lTemplate => "w2l_template", + Self::W2lResult => "w2l_result", + Self::MediaFile => "media_file", + Self::VoiceConversionModel => "voice_conversion_model", + Self::ZsVoice => "zs_voice", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "user" => Ok(Self::User), + "tts_model" => Ok(Self::TtsModel), + "tts_result" => Ok(Self::TtsResult), + "w2l_template" => Ok(Self::W2lTemplate), + "w2l_result" => Ok(Self::W2lResult), + "media_file" => Ok(Self::MediaFile), + "voice_conversion_model" => Ok(Self::VoiceConversionModel), + "zs_voice" => Ok(Self::ZsVoice), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(UserBookmarkEntityType::User, "user"); + assert_serialization(UserBookmarkEntityType::TtsModel, "tts_model"); + assert_serialization(UserBookmarkEntityType::TtsResult, "tts_result"); + assert_serialization(UserBookmarkEntityType::W2lTemplate, "w2l_template"); + assert_serialization(UserBookmarkEntityType::W2lResult, "w2l_result"); + assert_serialization(UserBookmarkEntityType::MediaFile, "media_file"); + assert_serialization(UserBookmarkEntityType::VoiceConversionModel, "voice_conversion_model"); + assert_serialization(UserBookmarkEntityType::ZsVoice, "zs_voice"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(UserBookmarkEntityType::User.to_str(), "user"); + assert_eq!(UserBookmarkEntityType::TtsModel.to_str(), "tts_model"); + assert_eq!(UserBookmarkEntityType::TtsResult.to_str(), "tts_result"); + assert_eq!(UserBookmarkEntityType::W2lTemplate.to_str(), "w2l_template"); + assert_eq!(UserBookmarkEntityType::W2lResult.to_str(), "w2l_result"); + assert_eq!(UserBookmarkEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserBookmarkEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); + assert_eq!(UserBookmarkEntityType::ZsVoice.to_str(), "zs_voice"); + } + + #[test] + fn test_from_str() { + assert_eq!(UserBookmarkEntityType::from_str("user").unwrap(), UserBookmarkEntityType::User); + assert_eq!(UserBookmarkEntityType::from_str("tts_model").unwrap(), UserBookmarkEntityType::TtsModel); + assert_eq!(UserBookmarkEntityType::from_str("tts_result").unwrap(), UserBookmarkEntityType::TtsResult); + assert_eq!(UserBookmarkEntityType::from_str("w2l_template").unwrap(), UserBookmarkEntityType::W2lTemplate); + assert_eq!(UserBookmarkEntityType::from_str("w2l_result").unwrap(), UserBookmarkEntityType::W2lResult); + assert_eq!(UserBookmarkEntityType::from_str("media_file").unwrap(), UserBookmarkEntityType::MediaFile); + assert_eq!(UserBookmarkEntityType::from_str("voice_conversion_model").unwrap(), UserBookmarkEntityType::VoiceConversionModel); + assert_eq!(UserBookmarkEntityType::from_str("zs_voice").unwrap(), UserBookmarkEntityType::ZsVoice); + assert!(UserBookmarkEntityType::from_str("foo").is_err()); + } + } +} diff --git a/crates/_types/tokens/src/prefixes.rs b/crates/_types/tokens/src/prefixes.rs index 6e24a9802b..382492bae4 100644 --- a/crates/_types/tokens/src/prefixes.rs +++ b/crates/_types/tokens/src/prefixes.rs @@ -27,7 +27,6 @@ pub(crate) enum TokenPrefix { Comment, DownloadJob, EmailSenderJob, - Favorite, InferenceJob, MediaFile, MediaUpload, @@ -36,6 +35,7 @@ pub(crate) enum TokenPrefix { PasswordReset, TtsRenderTask, // NB: aichatbot / sqlite User, + UserBookmark, UserSession, VoiceConversionModel, VoiceConversionResult, @@ -94,7 +94,6 @@ impl PrefixGenerator for TokenPrefix { Self::DownloadJob => "jdown_", // NB: Previously "JGUP:" Self::EmailSenderJob => "email_job_", Self::InferenceJob => "jinf_", - Self::Favorite => "fav_", Self::MediaFile => "m_", Self::MediaUpload => "mu_", Self::ModelWeight => "weight_", @@ -102,6 +101,7 @@ impl PrefixGenerator for TokenPrefix { Self::PasswordReset => "pw_reset_", Self::TtsRenderTask => "tts_task_", Self::User => "user_", // NB: Previously "U:" + Self::UserBookmark => "ub_", Self::UserSession => "session_", Self::VoiceConversionModel => "vcm_", Self::VoiceConversionResult => "vcr_", diff --git a/crates/_types/tokens/src/tokens/mod.rs b/crates/_types/tokens/src/tokens/mod.rs index e75f732b55..b7c4eed592 100644 --- a/crates/_types/tokens/src/tokens/mod.rs +++ b/crates/_types/tokens/src/tokens/mod.rs @@ -34,4 +34,4 @@ pub mod w2l_templates; pub mod zs_voice_dataset_samples; pub mod zs_voice_datasets; pub mod zs_voices; -pub mod favorites; +pub mod user_bookmarks; diff --git a/crates/_types/tokens/src/tokens/favorites.rs b/crates/_types/tokens/src/tokens/user_bookmarks.rs similarity index 59% rename from crates/_types/tokens/src/tokens/favorites.rs rename to crates/_types/tokens/src/tokens/user_bookmarks.rs index 78ecb7e0b9..ac91dc079d 100644 --- a/crates/_types/tokens/src/tokens/favorites.rs +++ b/crates/_types/tokens/src/tokens/user_bookmarks.rs @@ -8,7 +8,7 @@ use crate::prefixes::TokenPrefix; /// The primary key for Audit Logs. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] -pub struct FavoriteToken(pub String); +pub struct UserBookmarkToken(pub String); -impl_string_token!(FavoriteToken); -impl_crockford_generator!(FavoriteToken, 32usize, TokenPrefix::Favorite, CrockfordLower); +impl_string_token!(UserBookmarkToken); +impl_crockford_generator!(UserBookmarkToken, 32usize, TokenPrefix::UserBookmark, CrockfordLower); diff --git a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs b/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs deleted file mode 100644 index 3a0007253f..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs +++ /dev/null @@ -1,48 +0,0 @@ -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use tokens::tokens::media_files::MediaFileToken; -use tokens::tokens::tts_models::TtsModelToken; -use tokens::tokens::tts_results::TtsResultToken; -use tokens::tokens::users::UserToken; -use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; -use tokens::tokens::w2l_results::W2lResultToken; -use tokens::tokens::w2l_templates::W2lTemplateToken; -use tokens::tokens::zs_voices::ZsVoiceToken; - -pub enum FavoriteEntityToken { - User(UserToken), - TtsModel(TtsModelToken), - TtsResult(TtsResultToken), - W2lTemplate(W2lTemplateToken), - W2lResult(W2lResultToken), - MediaFile(MediaFileToken), - VoiceConversionModel(VoiceConversionModelToken), - ZsVoice(ZsVoiceToken), -} - -impl FavoriteEntityToken { - pub fn from_entity_type_and_token(entity_type: FavoriteEntityType, token: &str) -> Self { - match entity_type { - FavoriteEntityType::User => Self::User(UserToken::new_from_str(token)), - FavoriteEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), - FavoriteEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), - FavoriteEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), - FavoriteEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), - FavoriteEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), - FavoriteEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), - FavoriteEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), - } - } - - pub fn get_composite_keys(&self) -> (FavoriteEntityType, &str) { - match self { - FavoriteEntityToken::User(user_token) => (FavoriteEntityType::User, user_token.as_str()), - FavoriteEntityToken::TtsModel(tts_model_token) => (FavoriteEntityType::TtsModel, tts_model_token.as_str()), - FavoriteEntityToken::TtsResult(tts_result_token) => (FavoriteEntityType::TtsResult, tts_result_token.as_str()), - FavoriteEntityToken::W2lTemplate(w2l_template_token) => (FavoriteEntityType::W2lTemplate, w2l_template_token.as_str()), - FavoriteEntityToken::W2lResult(w2l_result_token) => (FavoriteEntityType::W2lResult, w2l_result_token.as_str()), - FavoriteEntityToken::MediaFile(media_file_token) => (FavoriteEntityType::MediaFile, media_file_token.as_str()), - FavoriteEntityToken::VoiceConversionModel(voice_conversion_model_token) => (FavoriteEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), - FavoriteEntityToken::ZsVoice(zs_voice_token) => (FavoriteEntityType::ZsVoice, zs_voice_token.as_str()), - } - } -} diff --git a/crates/lib/mysql_queries/src/queries/favorites/mod.rs b/crates/lib/mysql_queries/src/queries/favorites/mod.rs deleted file mode 100644 index 8cc8a3c7c5..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod create_favorite; -pub mod delete_favorite; -pub mod favorite_entity_token; -pub mod get_favorite; -pub mod list_favorites_for_entity; -pub mod list_user_favorites; -pub mod list_user_favorites_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/mod.rs b/crates/lib/mysql_queries/src/queries/mod.rs index 1acd2d7a89..39499425b4 100644 --- a/crates/lib/mysql_queries/src/queries/mod.rs +++ b/crates/lib/mysql_queries/src/queries/mod.rs @@ -3,7 +3,6 @@ pub mod audit_logs; pub mod billing; pub mod comments; pub mod email_sender_jobs; -pub mod favorites; pub mod generic_download; pub mod generic_inference; pub mod generic_synthetic_ids; @@ -18,6 +17,7 @@ pub mod trending_model_analytics; pub mod tts; pub mod twitch; pub mod users; +pub mod user_bookmarks; pub mod vocoder; pub mod voice_clone_requests; pub mod voice_conversion; diff --git a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs index a6decf5fff..b897d8daaf 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs @@ -4,15 +4,15 @@ use anyhow::anyhow; use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct CreateFavoriteArgs<'e, 'c, E> +pub struct CreateUserBookmarkArgs<'e, 'c, E> where E: 'e + Executor<'c, Database = MySql> { - pub entity_token: &'e FavoriteEntityToken, + pub entity_token: &'e UserBookmarkEntityToken, pub user_token: &'e UserToken, @@ -23,19 +23,19 @@ pub struct CreateFavoriteArgs<'e, 'c, E> pub phantom: PhantomData<&'c E>, } -pub async fn create_favorite<'e, 'c : 'e, E>( - args: CreateFavoriteArgs<'e, 'c, E>, +pub async fn create_user_bookmark<'e, 'c : 'e, E>( + args: CreateUserBookmarkArgs<'e, 'c, E>, ) - -> AnyhowResult + -> AnyhowResult where E: 'e + Executor<'c, Database = MySql> { - let favorite_token = FavoriteToken::generate(); + let user_bookmark_token = UserBookmarkToken::generate(); let (entity_type, entity_token) = args.entity_token.get_composite_keys(); let query_result = sqlx::query!( r#" -INSERT INTO favorites +INSERT INTO user_bookmarks SET token = ?, user_token = ?, @@ -46,7 +46,7 @@ ON DUPLICATE KEY UPDATE deleted_at = NULL, version = version + 1 "#, - &favorite_token, + &user_bookmark_token, args.user_token, entity_type, entity_token, @@ -59,5 +59,5 @@ ON DUPLICATE KEY UPDATE Err(err) => return Err(anyhow!("Mysql error: {:?}", err)), }; - Ok(favorite_token) + Ok(user_bookmark_token) } diff --git a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs similarity index 61% rename from crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs index 0ec20c4e6a..025b6f51c7 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs @@ -1,22 +1,22 @@ use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; // NB: UserToken is only supplied as an optimistic form of validation -pub async fn delete_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - user_token: &'e UserToken, - mysql_executor: E +pub async fn delete_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + user_token: &'e UserToken, + mysql_executor: E ) - -> AnyhowResult<()> + -> AnyhowResult<()> where E: 'e + Executor<'c, Database = MySql> { sqlx::query!( r#" -UPDATE favorites +UPDATE user_bookmarks SET deleted_at = CURRENT_TIMESTAMP, version = version + 1 @@ -25,7 +25,7 @@ token = ? AND user_token = ? LIMIT 1 "#, - favorite_token, + user_bookmark_token, user_token ) .execute(mysql_executor) diff --git a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs similarity index 51% rename from crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs index 01d46404bc..efb6903070 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs @@ -2,15 +2,15 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use sqlx::{Executor, MySql}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, @@ -23,21 +23,21 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn get_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - mysql_executor: E +pub async fn get_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + mysql_executor: E ) - -> AnyhowResult> + -> AnyhowResult> where E: 'e + Executor<'c, Database = MySql> { let maybe_results = sqlx::query_as!( - RawFavorite, + RawUserBookmark, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, @@ -50,29 +50,29 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE f.token = ? "#, - favorite_token + user_bookmark_token ) .fetch_one(mysql_executor) .await; match maybe_results { - Ok(favorite) => Ok(Some(Favorite { - token: favorite.token, - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - user_token: favorite.user_token, - username: favorite.username, - user_display_name: favorite.user_display_name, - user_gravatar_hash: favorite.user_gravatar_hash, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - maybe_deleted_at: favorite.deleted_at, + Ok(user_bookmark) => Ok(Some(UserBookmark { + token: user_bookmark.token, + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + user_token: user_bookmark.user_token, + username: user_bookmark.username, + user_display_name: user_bookmark.user_display_name, + user_gravatar_hash: user_bookmark.user_gravatar_hash, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + maybe_deleted_at: user_bookmark.deleted_at, })), Err(err) => match err { sqlx::Error::RowNotFound => Ok(None), @@ -81,10 +81,10 @@ WHERE } } -pub struct RawFavorite { - pub token: FavoriteToken, +pub struct RawUserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs similarity index 75% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index f42c7da477..1e1896a750 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -2,24 +2,26 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; +use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; +use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; +use tokens::tokens::user_bookmarks::UserBookmarkToken; -use crate::queries::favorites::list_user_favorites_result::{RawUserFavoriteRecord, UserFavorite}; -pub async fn list_user_favorites( +pub async fn list_user_bookmarks( username: &str, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { /// TODO(bt,2023-11-21): Maybe this query can use a switch /// See: https://stackoverflow.com/questions/2761574/mysql-use-case-else-value-as-join-parameter let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -39,7 +41,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -65,28 +67,28 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub async fn list_user_favorites_by_entity_type( - username: &str, - entity_type: FavoriteEntityType, - mysql_pool: &MySqlPool -) -> AnyhowResult> { +pub async fn list_user_user_bookmarks_by_entity_type( + username: &str, + entity_type: UserBookmarkEntityType, + mysql_pool: &MySqlPool +) -> AnyhowResult> { let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -106,7 +108,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -134,12 +136,12 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs index e1080ac694..3df0f0812c 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs @@ -4,13 +4,13 @@ use log::warn; use sqlx::MySqlPool; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, pub user_token: UserToken, pub username: String, @@ -22,18 +22,18 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn list_favorites_for_entity( - favorite_entity_token: FavoriteEntityToken, +pub async fn list_user_bookmarks_for_entity( + user_bookmark_entity_token: UserBookmarkEntityToken, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { - let (entity_type, entity_token) = favorite_entity_token.get_composite_keys(); + let (entity_type, entity_token) = user_bookmark_entity_token.get_composite_keys(); let maybe_results= sqlx::query_as!( - RawFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, u.display_name as user_display_name, @@ -44,7 +44,7 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE @@ -64,18 +64,18 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub struct RawFavoriteRecord { - token: FavoriteToken, +pub struct RawUserBookmarkRecord { + token: UserBookmarkToken, user_token: UserToken, username: String, @@ -87,9 +87,9 @@ pub struct RawFavoriteRecord { deleted_at: Option>, } -impl RawFavoriteRecord { - pub fn into_public_type(self) -> Favorite { - Favorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs similarity index 66% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index c391a5d8e6..e56d1de9d8 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -1,18 +1,18 @@ use chrono::{DateTime, Utc}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_type::MediaFileType; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct UserFavorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, - /// Something descriptive about the favorited entity. + /// Something descriptive about the bookmarked entity. /// This might be TTS text, a username, etc. It depends on the entity type. pub maybe_entity_descriptive_text: Option, @@ -26,10 +26,10 @@ pub struct UserFavorite { pub maybe_deleted_at: Option>, } -pub struct RawUserFavoriteRecord { - pub (crate) token: FavoriteToken, +pub struct RawUserBookmarkRecord { + pub (crate) token: UserBookmarkToken, - pub (crate) entity_type: FavoriteEntityType, + pub (crate) entity_type: UserBookmarkEntityType, pub (crate) entity_token: String, pub (crate) user_token: UserToken, @@ -51,27 +51,27 @@ pub struct RawUserFavoriteRecord { pub (crate) maybe_descriptive_text_zs_voice_title: Option, } -impl RawUserFavoriteRecord { - pub fn into_public_type(self) -> UserFavorite { - UserFavorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, entity_type: self.entity_type, entity_token: self.entity_token, maybe_entity_descriptive_text: match self.entity_type { - FavoriteEntityType::User => self.maybe_descriptive_text_user_display_name, - FavoriteEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, - FavoriteEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, - FavoriteEntityType::W2lTemplate => None, - FavoriteEntityType::W2lResult => None, + UserBookmarkEntityType::User => self.maybe_descriptive_text_user_display_name, + UserBookmarkEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, + UserBookmarkEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, + UserBookmarkEntityType::W2lTemplate => None, + UserBookmarkEntityType::W2lResult => None, // TODO(bt,2023-11-21): Summary text needs to be better enriched. - FavoriteEntityType::MediaFile => self.maybe_media_file_type + UserBookmarkEntityType::MediaFile => self.maybe_media_file_type .and_then(|media_file_type| match media_file_type { MediaFileType::Audio => Some("audio media file".to_string()), MediaFileType::Image => Some("image media file".to_string()), MediaFileType::Video => Some("video media file".to_string()), }), - FavoriteEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, - FavoriteEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, + UserBookmarkEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, + UserBookmarkEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, }, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs new file mode 100644 index 0000000000..df6d7f415c --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs @@ -0,0 +1,7 @@ +pub mod delete_user_bookmark; +pub mod create_user_bookmark; +pub mod user_bookmark_entity_token; +pub mod get_user_bookmark; +pub mod list_user_bookmarks_for_entity; +pub mod list_user_bookmarks; +pub mod list_user_bookmarks_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs new file mode 100644 index 0000000000..5fbf8224ad --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs @@ -0,0 +1,48 @@ +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use tokens::tokens::zs_voices::ZsVoiceToken; + +pub enum UserBookmarkEntityToken { + User(UserToken), + TtsModel(TtsModelToken), + TtsResult(TtsResultToken), + W2lTemplate(W2lTemplateToken), + W2lResult(W2lResultToken), + MediaFile(MediaFileToken), + VoiceConversionModel(VoiceConversionModelToken), + ZsVoice(ZsVoiceToken), +} + +impl UserBookmarkEntityToken { + pub fn from_entity_type_and_token(entity_type: UserBookmarkEntityType, token: &str) -> Self { + match entity_type { + UserBookmarkEntityType::User => Self::User(UserToken::new_from_str(token)), + UserBookmarkEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), + UserBookmarkEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), + UserBookmarkEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), + UserBookmarkEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), + UserBookmarkEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), + UserBookmarkEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), + UserBookmarkEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), + } + } + + pub fn get_composite_keys(&self) -> (UserBookmarkEntityType, &str) { + match self { + UserBookmarkEntityToken::User(user_token) => (UserBookmarkEntityType::User, user_token.as_str()), + UserBookmarkEntityToken::TtsModel(tts_model_token) => (UserBookmarkEntityType::TtsModel, tts_model_token.as_str()), + UserBookmarkEntityToken::TtsResult(tts_result_token) => (UserBookmarkEntityType::TtsResult, tts_result_token.as_str()), + UserBookmarkEntityToken::W2lTemplate(w2l_template_token) => (UserBookmarkEntityType::W2lTemplate, w2l_template_token.as_str()), + UserBookmarkEntityToken::W2lResult(w2l_result_token) => (UserBookmarkEntityType::W2lResult, w2l_result_token.as_str()), + UserBookmarkEntityToken::MediaFile(media_file_token) => (UserBookmarkEntityType::MediaFile, media_file_token.as_str()), + UserBookmarkEntityToken::VoiceConversionModel(voice_conversion_model_token) => (UserBookmarkEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), + UserBookmarkEntityToken::ZsVoice(zs_voice_token) => (UserBookmarkEntityType::ZsVoice, zs_voice_token.as_str()), + } + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs deleted file mode 100644 index efbb773b84..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs +++ /dev/null @@ -1,122 +0,0 @@ -// NB: Incrementally getting rid of build warnings... -#![forbid(unused_imports)] -#![forbid(unused_mut)] -#![forbid(unused_variables)] - -use std::fmt; -use std::sync::Arc; - -use actix_web::{HttpRequest, HttpResponse, web}; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; -use actix_web::web::Path; -use chrono::{DateTime, Utc}; -use log::warn; - -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::list_favorites_for_entity::list_favorites_for_entity; -use tokens::tokens::favorites::FavoriteToken; - -use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::server_state::ServerState; - -/// For the URL PathInfo -#[derive(Deserialize)] -pub struct ListFavoritesPathInfo { - entity_type: FavoriteEntityType, - entity_token: String, -} - -#[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { - pub success: bool, - pub favorites: Vec, -} - -#[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, - - pub user: UserDetailsLight, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug)] -pub enum ListFavoritesError { - ServerError, -} - -impl ResponseError for ListFavoritesError { - fn status_code(&self) -> StatusCode { - match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, - } - } - - fn error_response(&self) -> HttpResponse { - let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - }; - - to_simple_json_error(&error_reason, self.status_code()) - } -} - -// NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -pub async fn list_favorites_for_entity_handler( - _http_request: HttpRequest, - path: Path, - server_state: web::Data> -) -> Result -{ - let entity_token = FavoriteEntityToken::from_entity_type_and_token( - path.entity_type, &path.entity_token); - - let query_results = list_favorites_for_entity( - entity_token, - &server_state.mysql_pool, - ).await; - - let favorites = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); - } - }; - - let response = ListFavoritesSuccessResponse { - success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - user: UserDetailsLight { - user_token: favorite.user_token.clone(), - username: favorite.username.to_string(), // NB: Cloned because of ref use for avatar below - display_name: favorite.user_display_name.clone(), - gravatar_hash: favorite.user_gravatar_hash.clone(), - default_avatar: DefaultAvatarInfo::from_username(&favorite.username), - }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - }) - .collect(), - }; - - let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs deleted file mode 100644 index 4529d2f18b..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod create_favorite_handler; -pub mod delete_favorite_handler; -pub mod list_favorites_for_entity_handler; -pub mod list_favorites_for_session_handler; -pub mod list_favorites_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index d4c32ebd1d..0a6ed9375d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -25,5 +25,5 @@ pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; pub mod w2l; -pub mod favorites; +pub mod user_bookmarks; pub mod weights; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs similarity index 52% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs index cb4aa5f5f4..b62fe2feeb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs @@ -11,46 +11,46 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::create_favorite::{create_favorite, CreateFavoriteArgs}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct CreateFavoriteRequest { +pub struct CreateUserBookmarkRequest { entity_token: String, - entity_type: FavoriteEntityType, + entity_type: UserBookmarkEntityType, } #[derive(Serialize)] -pub struct CreateFavoriteSuccessResponse { +pub struct CreateUserBookmarkSuccessResponse { pub success: bool, - pub favorite_token: FavoriteToken, + pub user_bookmark_token: UserBookmarkToken, } #[derive(Debug)] -pub enum CreateFavoriteError { +pub enum CreateUserBookmarkError { BadInput(String), NotAuthorized, ServerError, } -impl ResponseError for CreateFavoriteError { +impl ResponseError for CreateUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - CreateFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - CreateFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - CreateFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + CreateUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - CreateFavoriteError::BadInput(reason) => reason.to_string(), - CreateFavoriteError::NotAuthorized => "unauthorized".to_string(), - CreateFavoriteError::ServerError => "server error".to_string(), + CreateUserBookmarkError::BadInput(reason) => reason.to_string(), + CreateUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + CreateUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -58,24 +58,24 @@ impl ResponseError for CreateFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for CreateFavoriteError { +impl fmt::Display for CreateUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn create_favorite_handler( +pub async fn create_user_bookmark_handler( http_request: HttpRequest, - request: web::Json, + request: web::Json, server_state: web::Data>, -) -> Result +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -84,42 +84,42 @@ pub async fn create_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { warn!("not logged in"); - return Err(CreateFavoriteError::NotAuthorized); + return Err(CreateUserBookmarkError::NotAuthorized); } }; - let entity_token = FavoriteEntityToken::from_entity_type_and_token( + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( request.entity_type, &request.entity_token); - let query_result = create_favorite(CreateFavoriteArgs { + let query_result = create_user_bookmark(CreateUserBookmarkArgs { entity_token: &entity_token, user_token: &user_session.user_token_typed, mysql_executor: &mut *mysql_connection, phantom: Default::default(), }).await; - let favorite_token = match query_result { + let user_bookmark_token = match query_result { Ok(token) => token, Err(err) => { - warn!("error inserting favorite: {:?}", err); - return Err(CreateFavoriteError::ServerError); + warn!("error inserting user_bookmark: {:?}", err); + return Err(CreateUserBookmarkError::ServerError); } }; - let response = CreateFavoriteSuccessResponse { + let response = CreateUserBookmarkSuccessResponse { success: true, - favorite_token, + user_bookmark_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| CreateFavoriteError::ServerError)?; + .map_err(|_e| CreateUserBookmarkError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs similarity index 58% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs index be28d24534..d68fc29e75 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs @@ -12,8 +12,8 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use log::warn; -use mysql_queries::queries::favorites::delete_favorite::delete_favorite; -use tokens::tokens::favorites::FavoriteToken; +use mysql_queries::queries::user_bookmarks::delete_user_bookmark::delete_user_bookmark; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -21,40 +21,40 @@ use crate::server_state::ServerState; /// For the URL PathInfo #[derive(Deserialize)] -pub struct DeleteFavoritePathInfo { - favorite_token: FavoriteToken, +pub struct DeleteUserBookmarkPathInfo { + user_bookmark_token: UserBookmarkToken, } #[derive(Deserialize)] -pub struct DeleteFavoriteRequest { +pub struct DeleteUserBookmarkRequest { /// NB: this is only to disambiguate when a user is both a mod and an author. as_mod: Option, } #[derive(Debug)] -pub enum DeleteFavoriteError { +pub enum DeleteUserBookmarkError { BadInput(String), NotAuthorized, NotFound, ServerError, } -impl ResponseError for DeleteFavoriteError { +impl ResponseError for DeleteUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - DeleteFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - DeleteFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - DeleteFavoriteError::NotFound => StatusCode::NOT_FOUND, - DeleteFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + DeleteUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteUserBookmarkError::NotFound => StatusCode::NOT_FOUND, + DeleteUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - DeleteFavoriteError::BadInput(reason) => reason.to_string(), - DeleteFavoriteError::NotAuthorized => "unauthorized".to_string(), - DeleteFavoriteError::NotFound => "not found".to_string(), - DeleteFavoriteError::ServerError => "server error".to_string(), + DeleteUserBookmarkError::BadInput(reason) => reason.to_string(), + DeleteUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + DeleteUserBookmarkError::NotFound => "not found".to_string(), + DeleteUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -62,24 +62,24 @@ impl ResponseError for DeleteFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for DeleteFavoriteError { +impl fmt::Display for DeleteUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn delete_favorite_handler( +pub async fn delete_user_bookmark_handler( http_request: HttpRequest, - path: Path, - _request: web::Json, + path: Path, + _request: web::Json, server_state: web::Data> -) -> Result { +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -88,18 +88,18 @@ pub async fn delete_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { - return Err(DeleteFavoriteError::NotAuthorized); + return Err(DeleteUserBookmarkError::NotAuthorized); } }; - let query_result = delete_favorite( - &path.favorite_token, + let query_result = delete_user_bookmark( + &path.user_bookmark_token, &user_session.user_token_typed, &mut *mysql_connection ).await; @@ -108,7 +108,7 @@ pub async fn delete_favorite_handler( Ok(_) => {}, Err(err) => { warn!("Update tts mod approval status DB error: {:?}", err); - return Err(DeleteFavoriteError::ServerError); + return Err(DeleteUserBookmarkError::ServerError); } }; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs new file mode 100644 index 0000000000..0fa28887fd --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs @@ -0,0 +1,122 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks_for_entity::list_user_bookmarks_for_entity; +use tokens::tokens::user_bookmarks::UserBookmarkToken; + +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct ListUserBookmarkPathInfo { + entity_type: UserBookmarkEntityType, + entity_token: String, +} + +#[derive(Serialize)] +pub struct ListUserBookmarksSuccessResponse { + pub success: bool, + pub user_bookmarks: Vec, +} + +#[derive(Serialize)] +pub struct UserBookmark { + pub token: UserBookmarkToken, + + pub user: UserDetailsLight, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ListUserBookmarksError { + ServerError, +} + +impl ResponseError for ListUserBookmarksError { + fn status_code(&self) -> StatusCode { + match *self { + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListUserBookmarksError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListUserBookmarksError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_user_bookmarks_for_entity_handler( + _http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( + path.entity_type, &path.entity_token); + + let query_results = list_user_bookmarks_for_entity( + entity_token, + &server_state.mysql_pool, + ).await; + + let user_bookmarks = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListUserBookmarksError::ServerError); + } + }; + + let response = ListUserBookmarksSuccessResponse { + success: true, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + user: UserDetailsLight { + user_token: user_bookmark.user_token.clone(), + username: user_bookmark.username.to_string(), // NB: Cloned because of ref use for avatar below + display_name: user_bookmark.user_display_name.clone(), + gravatar_hash: user_bookmark.user_gravatar_hash.clone(), + default_avatar: DefaultAvatarInfo::from_username(&user_bookmark.username), + }, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + }) + .collect(), + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| ListUserBookmarksError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs index cdc11f8550..dc1f17d90c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs @@ -13,38 +13,38 @@ use actix_web::web::Query; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -55,23 +55,23 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, NotAuthorizedError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - ListFavoritesError::NotAuthorizedError => StatusCode::UNAUTHORIZED, + ListUserBookmarksError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::NotAuthorizedError => StatusCode::UNAUTHORIZED, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - ListFavoritesError::NotAuthorizedError => "not authorized".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), + ListUserBookmarksError::NotAuthorizedError => "not authorized".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -79,61 +79,61 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_session_handler( +pub async fn list_user_bookmarks_for_session_handler( http_request: HttpRequest, - query: Query, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let user_session = server_state.session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); - ListFavoritesError::ServerError + ListUserBookmarksError::ServerError })? - .ok_or(ListFavoritesError::NotAuthorizedError)?; + .ok_or(ListUserBookmarksError::NotAuthorizedError)?; let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&user_session.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&user_session.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, maybe_summary_text: None, maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs index 071b1fcb0e..ccaf8b61f4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs @@ -13,43 +13,43 @@ use actix_web::web::{Path, Query}; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesPathInfo { +pub struct ListUserBookmarksPathInfo { username: String, } #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -60,20 +60,20 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -81,56 +81,56 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_user_handler( +pub async fn list_user_bookmarks_for_user_handler( _http_request: HttpRequest, - path: Path, - query: Query, + path: Path, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&path.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&path.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - maybe_summary_text: favorite.maybe_entity_descriptive_text, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + maybe_summary_text: user_bookmark.maybe_entity_descriptive_text, // TODO(bt,2023-11-21): Thumbnails need proper support. We should build them as a // first-class system before handling the backfill here. maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs new file mode 100644 index 0000000000..11881f5bb3 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs @@ -0,0 +1,5 @@ +pub mod create_user_bookmark_handler; +pub mod delete_user_bookmark_handler; +pub mod list_user_bookmarks_for_entity_handler; +pub mod list_user_bookmarks_for_session_handler; +pub mod list_user_bookmarks_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index eb67880dd5..9dcdce3e35 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -28,11 +28,11 @@ use crate::http_server::endpoints::comments::list_comments_handler::list_comment use crate::http_server::endpoints::download_job::enqueue_generic_download::enqueue_generic_download_handler; use crate::http_server::endpoints::download_job::get_generic_upload_job_status::get_generic_download_job_status_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; -use crate::http_server::endpoints::favorites::create_favorite_handler::create_favorite_handler; -use crate::http_server::endpoints::favorites::delete_favorite_handler::delete_favorite_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_entity_handler::list_favorites_for_entity_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_session_handler::list_favorites_for_session_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_user_handler::list_favorites_for_user_handler; +use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler; use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; @@ -202,14 +202,14 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> .add_post("/v1/comments/delete/{comment_token}", delete_comment_handler) .into_app(); - // ==================== Favorites ==================== + // ==================== User Bookmarks ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/favorites/create", create_favorite_handler) - .add_post("/v1/favorites/delete/{favorite_token}", delete_favorite_handler) - .add_get("/v1/favorites/list/session", list_favorites_for_session_handler) - .add_get("/v1/favorites/list/user/{username}", list_favorites_for_user_handler) - .add_get("/v1/favorites/list/entity/{entity_type}/{entity_token}", list_favorites_for_entity_handler) + .add_post("/v1/user_bookmarks/create", create_user_bookmark_handler) + .add_post("/v1/user_bookmarks/delete/{user_bookmark_token}", delete_user_bookmark_handler) + .add_get("/v1/user_bookmarks/list/session", list_user_bookmarks_for_session_handler) + .add_get("/v1/user_bookmarks/list/user/{username}", list_user_bookmarks_for_user_handler) + .add_get("/v1/user_bookmarks/list/entity/{entity_type}/{entity_token}", list_user_bookmarks_for_entity_handler) .into_app(); // ==================== Animations ==================== From e78cdf85e30ead4cca5b6cbe92404493679f1a71 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 11 Dec 2023 14:40:14 -0500 Subject: [PATCH 44/58] Upload weights front end side done, need to write the download code then run inference depending on the model. --- crates/lib/deprecated/config/src/bad_urls.rs | 5 + .../drain_multipart_weights_request.rs | 100 ----- .../endpoints/weights/upload_weight.rs | 385 +++++++----------- 3 files changed, 158 insertions(+), 332 deletions(-) delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs diff --git a/crates/lib/deprecated/config/src/bad_urls.rs b/crates/lib/deprecated/config/src/bad_urls.rs index 13be46019b..280ed1d3bb 100644 --- a/crates/lib/deprecated/config/src/bad_urls.rs +++ b/crates/lib/deprecated/config/src/bad_urls.rs @@ -3,11 +3,16 @@ use crate::is_bad_download_url::is_bad_download_url; /// Reports if the URL is inappropriate for downloading as a TTS model, vocoder, etc. +/// DEPRECATE: pub fn is_bad_tts_model_download_url(url: &str) -> anyhow::Result { // TODO: Just use this. is_bad_download_url(url) } +pub fn is_bad_model_weights_download_url(url: &str) -> anyhow::Result { + is_bad_download_url(url) +} + #[cfg(test)] mod tests { use crate::bad_urls::is_bad_tts_model_download_url; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs deleted file mode 100644 index c87a9d82c5..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs +++ /dev/null @@ -1,100 +0,0 @@ -use actix_multipart::Multipart; -use actix_web::web::BytesMut; -use futures::TryStreamExt; -use log::warn; - -use errors::AnyhowResult; -use tokens::tokens::weights::ModelWeightToken; - -use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; - -pub struct WeightUploadRequest { - pub uuid_idempotency_token: Option, - pub model_weights_token: Option, - pub file_name: Option, - pub file_bytes: Option, - pub media_source: MediaSource, -} - -/// Where the frontend tells us the file came from. -#[derive(Eq, PartialEq, Clone, Copy)] -pub enum MediaSource { - Unknown, - UserFile, - /// Eg. the web audio API in Javascript. - UserDeviceApi, -} - -/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. -pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { - let mut uuid_idempotency_token = None; - let mut model_weight_token = None; - let mut file_bytes = None; - let mut file_name = None; - let mut media_source = None; - - while let Ok(Some(mut field)) = multipart_payload.try_next().await { - let mut field_name = None; - let mut field_filename = None; - - if let content_disposition = field.content_disposition() { - field_name = content_disposition.get_name() - .map(|s| s.to_string()); - field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. - .map(|s| s.to_string()); - } - - match field_name.as_deref() { - Some("uuid_idempotency_token") => { - uuid_idempotency_token = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading idempotency token: {:}", e); - e - })?; - }, - Some("dataset_token") => { - model_weight_token = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading dataset token: {:}", e); - e - })?; - }, - Some("file") => { - file_name = field_filename.clone(); - file_bytes = checked_read_multipart_bytes(&mut field).await - .map_err(|e| { - warn!("Error reading audio upload: {:}", e); - e - })?; - }, - Some("source") => { - media_source = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading source: {:}", e); - e - })?; - }, - _ => continue, - } - } - - let media_source = match media_source.as_deref() { - Some("device") => MediaSource::UserDeviceApi, - Some("file") => MediaSource::UserFile, - _ => { - if file_name.as_deref() == Some("blob") { - MediaSource::UserDeviceApi - } else { - MediaSource::Unknown - } - }, - }; - - Ok(UploadSampleRequest { - uuid_idempotency_token, - dataset_token, - file_name, - file_bytes, - media_source, - }) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs index 2d66aa7684..8d2aa5fc5c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs @@ -2,10 +2,10 @@ use std::collections::HashSet; use std::sync::Arc; use actix_multipart::Multipart; -use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::{ HttpRequest, HttpResponse, web }; use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType; -use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use mysql_queries::queries::model_weights::create_weight::{ self, create_weight }; use once_cell::sync::Lazy; use tokens::tokens::model_weights::ModelWeightToken; @@ -13,259 +13,180 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; -use log::{error, info, warn}; +use log::{ error, info, warn }; use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; -use enums::common::visibility::Visibility; + use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; use http_server_common::request::get_request_ip::get_request_ip; use media::decode_basic_audio_info::decode_basic_audio_bytes_info; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; -use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; +use mysql_queries::queries::media_uploads::insert_media_upload::{ Args, insert_media_upload }; -use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{ + drain_multipart_request, + MediaSource, +}; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; use utoipa::ToSchema; +use config::bad_urls::is_bad_model_weights_download_url; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::common::visibility::Visibility; +use utoipa::ToSchema; + +#[derive(Deserialize, ToSchema)] +pub struct UploadModelWeightsRequest { + idempotency_token: String, + title: String, + download_url: String, + model_weights_type: WeightsType, + model_weights_category: WeightsCategory, + creator_set_visibility: Option, +} + +/// Tell the frontend how to deal with the download queue. +#[derive(Serialize, ToSchema)] +pub enum DownloadJobType { + /// Modern shared download type (everything will use this in the + /// future, and this endpoint will die.) + #[serde(rename = "generic")] + Generic, +} + +#[derive(Serialize, ToSchema)] +pub struct UploadModelWeightsSuccessResponse { + pub success: bool, + /// This is how frontend clients can request the job execution status. + pub job_token: String, + /// This is a transitional field to tell the frontend how to process the job checking. + pub job_type: DownloadJobType, +} + +#[derive(Debug)] +pub enum UploadModelWeightsError { + BadInput(String), + MustBeLoggedIn, + ServerError, + RateLimited, +} + +impl ResponseError for UploadModelWeightsError { + fn status_code(&self) -> StatusCode { + match *self { + UploadModelWeightsError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadModelWeightsError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadModelWeightsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadModelWeightsError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + UploadModelWeightsError::BadInput(reason) => reason.to_string(), + UploadModelWeightsError::MustBeLoggedIn => "user must be logged in".to_string(), + UploadModelWeightsError::ServerError => "server error".to_string(), + UploadModelWeightsError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + pub async fn upload_weights_handler( http_request: HttpRequest, server_state: web::Data>, - mut multipart_payload: Multipart, - ) -> Result { - - let response = handle_weight_upload( - &http_request, - &server_state, - multipart_payload, - &ALLOWED_MIME_TYPES).await?; - - // send a body with the required payload along side the multipart upload - - let weight_upload_token = response.to_model_token(); - - let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { - success: true, - upload_token: weight_upload_token, - }; - - let body = serde_json::to_string(&response) - .map_err(|e| UploadError::ServerError)?; - - return Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)); - } - - pub enum SuccessCase { - WeightAlreadyUploaded { - existing_upload_token: ModelWeightToken, - }, - WeightSuccessfullyUploaded { - upload_token: ModelWeightToken, + mut multipart_payload: Multipart +) -> Result { + // Rate limiter check + if + let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request( + &http_request + ) + { + return Err(UploadModelWeightsError::RateLimited); } - } - - impl SuccessCase { - pub fn to_model_token(self) -> ModelWeightToken { - match self { - SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, - SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, - } - } - } - - pub async fn handle_weight_upload( - http_request: &HttpRequest, - server_state: &web::Data>, - mut multipart_payload: Multipart, - allowed_mimetypes: &HashSet<&'static str>, - ) -> Result { - - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - error!("MySql pool error: {:?}", err); - UploadError::ServerError - })?; - - // ==================== READ SESSION ==================== // - - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) - .await + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { - error!("Session checker error: {:?}", e); - UploadError::ServerError + warn!("Session checker error: {:?}", e); + UploadModelWeightsError::ServerError })?; - - // ==================== RATE LIMIT ==================== // - - let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UploadModelWeightsError::MustBeLoggedIn); } - &server_state.redis_rate_limiters.logged_in - }, }; - - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); + + if let Err(reason) = validate_idempotency_token(&request.idempotency_token) { + return Err(UploadModelWeightsError::BadInput(reason)); } - - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); + + if let Err(reason) = validate_model_title(&request.title) { + return Err(UploadModelWeightsError::BadInput(reason)); } - - // ==================== READ MULTIPART REQUEST ==================== // - - let upload_weights_request = drain_multipart_request(multipart_payload) - .await - .map_err(|e| { - // TODO: Error handling could be nicer. - UploadError::BadInput("bad request".to_string()) - })?; - - let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token - .ok_or(UploadError::BadInput("no uuid".to_string()))?; - - let maybe_existing_upload = - get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) - .await; - - match maybe_existing_upload { - Err(err) => { - error!("Error checking for previous upload: {:?}", err); - return Err(UploadError::ServerError); - } - Ok(Some(upload)) => { - // File already uploaded and frontend didn't protect us. - return Ok(SuccessCase::WeightAlreadyUploaded { - existing_upload_token: upload.token, - }); - } - Ok(None) => { - // Proceed. - } + + match is_bad_model_weights_download_url(&download_url) { + Ok(false) => {} // Ok case + Ok(true) => { + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } + Err(err) => { + warn!("Error parsing url: {:?}", err); + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } } - - if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { - return Err(UploadError::BadInput(reason)); - } - - let creator_set_visibility = maybe_user_session - .as_ref() - .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. - .unwrap_or(Visibility::default()); - - let ip_address = get_request_ip(&http_request); - - let maybe_user_token = maybe_user_session - .map(|session| session.get_strongly_typed_user_token()); - - let maybe_file_size_bytes = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| bytes.len()); - - info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); - - let maybe_mimetype = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) - .flatten(); - - - let bytes = match upload_weights_request.file_bytes { - None => return Err(UploadError::BadInput("missing file contents".to_string())), - Some(bytes) => bytes, + + let supported_model_weights_type = request.model_weights_type; + + let job_token; + let download_job_type; + + let (download_job_token, _record_id) = insert_generic_download_job( + InsertGenericDownloadJobArgs { + uuid_idempotency_token: &uuid, + download_type: GenericDownloadType::Vits, + download_url: &download_url, + title: &title, + creator_user_token: &user_session.user_token, + creator_ip_address: &ip_address, + creator_set_visibility: visibility, // TODO: Creator set preference + mysql_pool: &server_state.mysql_pool, + } + ).await.map_err(|err| { + warn!("New generic download creation DB error (for generic model): {:?}", err); + UploadModelWeightsError::ServerError + })?; + + job_token = download_job_token.to_string(); + download_job_type = DownloadJobType::Generic; + + let response = UploadModelWeightsSuccessResponse { + success: true, + job_token: job_token.to_string(), + job_type: download_job_type, }; - - let hash = sha256_hash_bytes(&bytes) - .map_err(|io_error| { - error!("Problem hashing bytes: {:?}", io_error); - UploadError::ServerError - })?; - - let file_size_bytes = bytes.len(); - - let mut maybe_duration_millis = None; - let mut maybe_codec_name = None; - let mut media_upload_type = None; - - if let Some(mimetype) = maybe_mimetype.as_deref() { - - if !allowed_mimetypes.contains(mimetype) { - // NB: Don't let our error message inject malicious strings - let filtered_mimetype = mimetype - .chars() - .filter(|c| c.is_ascii()) - .filter(|c| c.is_alphanumeric() || *c == '/') - .collect::(); - return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); - } - media_upload_type = match mimetype { - "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), - _ => None, - }; + + let body = serde_json::to_string(&response).map_err(|_e| UploadModelWeightsError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} + +fn validate_idempotency_token(token: &str) -> Result<(), String> { + if token.len() != 36 { + return Err("idempotency token should be 36 characters".to_string()); } - - // This is fine for now we just don't expose the path! - let public_upload_path = WeightUploadOriginalFilePath::generate_new(); - - info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); - - // need to upload based on visibility - server_state.public_bucket_client.upload_file_with_content_type( - public_upload_path.get_full_object_path_str(), - bytes.as_ref(), - mime_type) - .await - .map_err(|e| { - warn!("Upload media bytes to bucket error: {:?}", e); - UploadError::ServerError - })?; - - let token = create_weight(CreateModelWeightsArgs { - token: &ModelWeightToken::generate(), - weights_type: WeightsType::LoRA, // to do parse multi part for this - weights_category: WeightsCategory::ImageGeneration, - title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: &ThumbnailToken::generate(), - description_markdown: "", - description_rendered_html: "", - creator_user_token: maybe_user_token.as_str(), - creator_ip_address: &ip_address, - creator_set_visibility, - maybe_last_update_user_token: "", - original_download_url: "", - original_filename: upload_weights_request.file_name.as_deref(), - file_size_bytes: file_size_bytes as i32, - file_checksum_sha2: hash, - private_bucket_hash: private_bucket_hash, - maybe_private_bucket_prefix: None, - maybe_private_bucket_extension: None, - cached_user_ratings_total_count: 0, - cached_user_ratings_positive_count: 0, - cached_user_ratings_negative_count: 0, - maybe_cached_user_ratings_ratio: None, - version: 0, - mysql_pool: &server_state.mysql_pool, - }).await - .map_err(|e| { - error!("Error creating weights upload: {:?}", e); - UploadError::ServerError - })?; - - info!("new weights upload id: {} token: {:?}", record_id, &token); - - Ok(SuccessCase::WeightSuccessfullyUploaded { - upload_token: token, - }) - } - \ No newline at end of file + + Ok(()) +} From c09adc3729ea697a032db6172b330e8d78c3fa8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:08:50 +0000 Subject: [PATCH 45/58] Bump hyper-tls from 0.5.0 to 0.6.0 Bumps [hyper-tls](https://github.com/hyperium/hyper-tls) from 0.5.0 to 0.6.0. - [Commits](https://github.com/hyperium/hyper-tls/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: hyper-tls dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 172 ++++++++++++++---- .../newrelic-telemetry-sdk-rust/Cargo.toml | 2 +- 2 files changed, 141 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1aaf8d3a68..9182c02682 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http", + "http 0.2.11", "httparse", "httpdate", "itoa 1.0.9", @@ -176,7 +176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "http", + "http 0.2.11", "regex", "serde", "tracing", @@ -614,8 +614,8 @@ dependencies = [ "hex", "hmac 0.12.1", "http-types", - "hyper", - "hyper-tls", + "hyper 0.14.27", + "hyper-tls 0.5.0", "serde", "serde_json", "serde_path_to_error", @@ -676,7 +676,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ - "http", + "http 0.2.11", "log", "native-tls", "serde", @@ -1961,7 +1961,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "mimetypes", @@ -2151,7 +2151,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "images", "jobs_common", "lfu", @@ -2636,7 +2636,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap", "slab", "tokio 1.34.0", @@ -2796,6 +2796,17 @@ dependencies = [ "itoa 1.0.9", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa 1.0.9", +] + [[package]] name = "http-body" version = "0.4.5" @@ -2803,7 +2814,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes 1.5.0", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes 1.5.0", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2823,7 +2857,7 @@ dependencies = [ "async-channel", "base64 0.13.1", "futures-lite", - "http", + "http 0.2.11", "infer 0.2.3", "pin-project-lite", "rand 0.7.3", @@ -2887,8 +2921,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa 1.0.9", @@ -2900,6 +2934,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "pin-project-lite", + "tokio 1.34.0", + "want", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2907,12 +2957,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.5.0", - "hyper", + "hyper 0.14.27", "native-tls", "tokio 1.34.0", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes 1.5.0", + "http-body-util", + "hyper 1.0.1", + "hyper-util", + "native-tls", + "tokio 1.34.0", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.0.1", + "pin-project-lite", + "socket2 0.5.5", + "tokio 1.34.0", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -3057,7 +3143,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "images", "jobs_common", "lfu", @@ -3758,7 +3844,7 @@ dependencies = [ "hashbrown 0.14.2", "heck", "hmac 0.12.1", - "hyper", + "hyper 0.14.27", "indexmap", "lazy_static", "libc", @@ -3897,8 +3983,8 @@ dependencies = [ "env_logger", "flate2", "futures 0.3.29", - "hyper", - "hyper-tls", + "hyper 0.14.27", + "hyper-tls 0.6.0", "log", "my-workspace-hack", "rand 0.8.5", @@ -4905,10 +4991,10 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", - "hyper-tls", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -5066,7 +5152,7 @@ dependencies = [ "futures 0.3.29", "hex", "hmac 0.12.1", - "http", + "http 0.2.11", "log", "maybe-async", "md5", @@ -6027,7 +6113,7 @@ dependencies = [ "hmac 0.10.1", "hostname", "http_server_common", - "hyper", + "hyper 0.14.27", "infer 0.5.0", "jwt", "language-tags", @@ -6867,6 +6953,28 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio 1.34.0", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -6927,7 +7035,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -6958,7 +7066,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "lfu", "log", @@ -7003,7 +7111,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.11", "httparse", "log", "rand 0.8.5", @@ -7033,7 +7141,7 @@ dependencies = [ "futures 0.3.29", "futures-util", "hostname", - "hyper", + "hyper 0.14.27", "lazy_static", "log", "magic-crypt", @@ -7075,7 +7183,7 @@ dependencies = [ "chrono", "displaydoc", "futures 0.3.29", - "http", + "http 0.2.11", "once_cell", "reqwest", "serde", @@ -7115,7 +7223,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "displaydoc", - "http", + "http 0.2.11", "once_cell", "rand 0.8.5", "reqwest", @@ -7437,7 +7545,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -7466,7 +7574,7 @@ dependencies = [ "filesys", "google_drive_common", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -7686,7 +7794,7 @@ dependencies = [ "futures-util", "hostname", "http_server_common", - "hyper", + "hyper 0.14.27", "lazy_static", "limitation", "log", diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 088a3cf568..cf1336f3f3 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0.57", features = ["derive"] } serde_json = "1.0.108" hyper = { version = "0.14", features = ["http1"], optional = true } -hyper-tls = { version = "0.5.0", optional = true } +hyper-tls = { version = "0.6.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } tokio = { version = "1.1.0", features = ["macros"], optional = true } From d3be9a3a6b8d258d48557b1b4198e52634aedf9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:09:11 +0000 Subject: [PATCH 46/58] Bump tokio from 0.1.22 to 1.34.0 Bumps [tokio](https://github.com/tokio-rs/tokio) from 0.1.22 to 1.34.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-0.1.22...tokio-1.34.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Cargo.lock | 233 ++++-------------- crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_upload_media_file/Cargo.toml | 2 +- crates/cli/elasticsearch-cli/Cargo.toml | 2 +- crates/cli/scrape_test/Cargo.toml | 2 +- .../lib/api_clients/cloud_storage/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- .../service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- .../service/job/tts_download_job/Cargo.toml | 2 +- .../service/job/tts_inference_job/Cargo.toml | 2 +- .../service/job/w2l_download_job/Cargo.toml | 2 +- .../service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/limitation/Cargo.toml | 2 +- .../newrelic-telemetry-sdk-rust/Cargo.toml | 4 +- 22 files changed, 71 insertions(+), 206 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1aaf8d3a68..67129df838 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -39,7 +39,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -115,7 +115,7 @@ dependencies = [ "rand 0.8.5", "sha1 0.10.6", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", "zstd 0.12.4", @@ -153,7 +153,7 @@ dependencies = [ "serde_json", "serde_plain", "tempfile", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -190,7 +190,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "actix-macros", "futures-core", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -206,7 +206,7 @@ dependencies = [ "futures-util", "mio 0.8.9", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -285,7 +285,7 @@ dependencies = [ "bytestring", "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -465,7 +465,7 @@ dependencies = [ "serde_json", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -600,7 +600,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -624,7 +624,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -810,7 +810,7 @@ dependencies = [ "serde_derive", "serde_json", "sqlx", - "tokio 1.34.0", + "tokio", "toml", "url_config", ] @@ -1162,7 +1162,7 @@ dependencies = [ "log", "my-workspace-hack", "rust-s3", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1218,7 +1218,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -1386,17 +1386,6 @@ dependencies = [ "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1404,25 +1393,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.15", + "crossbeam-epoch", "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -1436,17 +1410,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1791,7 +1754,7 @@ dependencies = [ "sqlx", "storyteller_root", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1812,7 +1775,7 @@ dependencies = [ "mysql_queries", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1976,7 +1939,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", - "tokio 1.34.0", + "tokio", "zip", ] @@ -2022,7 +1985,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2099,7 +2062,7 @@ dependencies = [ "storyteller_root", "strum", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2174,7 +2137,7 @@ dependencies = [ "tempdir", "testing", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -2639,7 +2602,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -2894,7 +2857,7 @@ dependencies = [ "itoa 1.0.9", "pin-project-lite", "socket2 0.4.10", - "tokio 1.34.0", + "tokio", "tower-service", "tracing", "want", @@ -2909,7 +2872,7 @@ dependencies = [ "bytes 1.5.0", "hyper", "native-tls", - "tokio 1.34.0", + "tokio", "tokio-native-tls", ] @@ -3079,7 +3042,7 @@ dependencies = [ "subprocess_common", "tempdir", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -3288,7 +3251,7 @@ dependencies = [ "once_cell", "quoted_printable", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "url 2.5.0", ] @@ -3353,7 +3316,7 @@ dependencies = [ "my-workspace-hack", "redis 0.13.0", "time 0.1.45", - "tokio 0.1.22", + "tokio", "version-sync", ] @@ -3556,15 +3519,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.7.1" @@ -3797,7 +3751,7 @@ dependencies = [ "syn 2.0.39", "time 0.3.30", "time-macros", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", "twitch_oauth2", @@ -3834,7 +3788,7 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", "toml", ] @@ -3904,7 +3858,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "tokio 1.34.0", + "tokio", "uuid 0.8.2", ] @@ -4733,7 +4687,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-deque 0.8.3", + "crossbeam-deque", "crossbeam-utils 0.8.16", ] @@ -4798,7 +4752,7 @@ dependencies = [ "pin-project-lite", "ryu", "sha1 0.6.1", - "tokio 1.34.0", + "tokio", "tokio-util", "url 2.5.0", ] @@ -4921,7 +4875,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "system-configuration", - "tokio 1.34.0", + "tokio", "tokio-native-tls", "tokio-util", "tower-service", @@ -5079,7 +5033,7 @@ dependencies = [ "sha2 0.10.8", "thiserror", "time 0.3.30", - "tokio 1.34.0", + "tokio", "tokio-stream", "url 2.5.0", ] @@ -5251,7 +5205,7 @@ dependencies = [ "clap 2.34.0", "errors", "my-workspace-hack", - "tokio 1.34.0", + "tokio", "web_scrapers", ] @@ -5767,7 +5721,7 @@ dependencies = [ "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue 0.3.8", + "crossbeam-queue", "dotenvy", "either", "event-listener", @@ -5793,7 +5747,7 @@ dependencies = [ "smallvec 1.11.2", "sqlformat", "thiserror", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", "url 2.5.0", @@ -5836,7 +5790,7 @@ dependencies = [ "sqlx-sqlite", "syn 1.0.109", "tempfile", - "tokio 1.34.0", + "tokio", "url 2.5.0", ] @@ -6066,7 +6020,7 @@ dependencies = [ "symphonia", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "toml", "tts_common", "twitch_common", @@ -6583,30 +6537,6 @@ dependencies = [ "utoipa", ] -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.34.0" @@ -6637,16 +6567,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - [[package]] name = "tokio-executor" version = "0.1.10" @@ -6657,17 +6577,6 @@ dependencies = [ "futures 0.1.31", ] -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - [[package]] name = "tokio-io" version = "0.1.13" @@ -6697,7 +6606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6726,7 +6635,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "webpki", ] @@ -6738,7 +6647,7 @@ checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6765,35 +6674,6 @@ dependencies = [ "tokio-reactor", ] -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue 0.2.3", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - [[package]] name = "tokio-tungstenite" version = "0.15.0" @@ -6804,28 +6684,13 @@ dependencies = [ "log", "pin-project", "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "tokio-rustls", "tungstenite", "webpki", "webpki-roots 0.21.1", ] -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - [[package]] name = "tokio-uds" version = "0.2.7" @@ -6854,7 +6719,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -6939,7 +6804,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6973,7 +6838,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -7054,7 +6919,7 @@ dependencies = [ "sqlx", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "tokio-tungstenite", "toml", "tts_common", @@ -7449,7 +7314,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7478,7 +7343,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7707,7 +7572,7 @@ dependencies = [ "sha2 0.9.9", "sqlx", "time 0.3.30", - "tokio 1.34.0", + "tokio", "toml", "twitch_api2", "twitch_common", diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index ae63bd60b6..a771241cf5 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -37,7 +37,7 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index ff572d4512..47d58a89c0 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -34,7 +34,7 @@ log = "0.4.14" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index c8f0841c69..7361ad74b6 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -34,7 +34,7 @@ strum = { version = "0.25", features = ["derive"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index d286897c20..36284d1f33 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -23,4 +23,4 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index 3d4c2d3f77..2cd5d20198 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -30,4 +30,4 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 335aab10f3..228c472322 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -43,7 +43,7 @@ serde_json = "1.0.108" toml = "0.5.8" serial_test = "2.0.0" -tokio = { version = "1.6.0", features = ["full"] } +tokio = { version = "1.34.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 59f30971d9..ed1487b8a3 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -39,7 +39,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Serialization serde = "1.0.125" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 96598161bc..e97068915d 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -53,7 +53,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index de0c02e9d6..f72b7f7fba 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -66,7 +66,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 44b1bb910a..9f8205ca14 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -62,7 +62,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 7506d7f670..07bfd1b465 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -45,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 9fbdc2df76..cbfdd24de1 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -51,7 +51,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index cfdafda722..7819055ed0 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -45,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 34db170d30..ac801c96bd 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -43,7 +43,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 1fda2a5a73..58a1486caf 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -43,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 4e123524e9..869eee7790 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -59,4 +59,4 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", mockall = "0.11.3" # NB: Tokio version matches `async-stripe` transitive dep; features are for testing -tokio = { version = "1.13.0", features = ["rt", "macros"] } +tokio = { version = "1.34.0", features = ["rt", "macros"] } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 671723c73b..4ea1672720 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -32,7 +32,7 @@ hostname = "0.3.1" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures-util = "0.3.29" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index c57dddc4db..c8d81cef9a 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -95,7 +95,7 @@ native-tls = { version = "0.2.8", features = ["vendored"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index f492098015..20c89b006b 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -43,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 7c4015b643..51f1626635 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -27,5 +27,5 @@ time = "0.1.42" my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] -tokio = "0.1.22" +tokio = "1.34.0" version-sync = "0.8.1" diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 088a3cf568..df26faf565 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -19,7 +19,7 @@ hyper = { version = "0.14", features = ["http1"], optional = true } hyper-tls = { version = "0.5.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } -tokio = { version = "1.1.0", features = ["macros"], optional = true } +tokio = { version = "1.34.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } # Workspace hack (faster compile times) @@ -30,4 +30,4 @@ rand = "0.8.5" env_logger = "0.10.1" futures = { version = "0.3.1" } hyper = { version = "0.14", features = ["http1", "server"] } -tokio = { version = "1.1.0", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } From 1cde23c08087ceecedfa4b9840e896ab4298dd27 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Dec 2023 20:38:50 -0500 Subject: [PATCH 47/58] Seed media files --- ...5c07196ab520cd8c63d30689b6021fa4b4ec1.json | 12 - ...106ac6bea906979c89355f805abb9650a7588.json | 12 + crates/cli/dev_database_seed/src/main.rs | 2 + .../cli/dev_database_seed/src/seeding/mod.rs | 5 +- .../src/seeding/seed_media_files.rs | 210 ++++++++++++++++++ crates/cli/dev_upload_media_file/src/main.rs | 1 + .../insert_media_file_from_cli_tool.rs | 8 +- 7 files changed, 235 insertions(+), 15 deletions(-) delete mode 100644 .sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json create mode 100644 .sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json create mode 100644 crates/cli/dev_database_seed/src/seeding/seed_media_files.rs diff --git a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json b/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json deleted file mode 100644 index fb10d9d281..0000000000 --- a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - }, - "hash": "030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1" -} diff --git a/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json new file mode 100644 index 0000000000..6a0e91bef8 --- /dev/null +++ b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n maybe_origin_filename = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + }, + "hash": "8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588" +} diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 917a8622a0..558478772e 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -8,6 +8,7 @@ use errors::AnyhowResult; use crate::bucket_clients::get_bucket_clients; use crate::cli_args::parse_cli_args; use crate::seeding::model_weights::seed_weights; +use crate::seeding::seed_media_files::seed_media_files; use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; use crate::seeding::voice_conversion::seed_voice_conversion; @@ -52,6 +53,7 @@ pub async fn main() -> AnyhowResult<()> { } seed_user_accounts(&pool).await?; + seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?; seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?; seed_voice_conversion(&pool).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index cb462d17c9..c1100805b6 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -1,5 +1,6 @@ +pub mod model_weights; +pub mod seed_media_files; pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; -pub mod zero_shot_tts; -pub mod model_weights; \ No newline at end of file +pub mod zero_shot_tts; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs new file mode 100644 index 0000000000..5bca8981e2 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -0,0 +1,210 @@ +use std::path::Path; + +use log::{info, warn}; +use sqlx::{MySql, Pool}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use filesys::file_read_bytes::file_read_bytes; +use filesys::file_size::file_size; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs}; +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { + info!("Seeding media files..."); + + // NB: This is idempotent and will only install the media files once. + // Delete the records if you want to re-seed. + let media_files = [ + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"), + // Audio + (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), + (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), + (HANASHI_USER_TOKEN, "m_sq2vhrrmmbr079afk21543aa2jg53g", MediaFileType::Audio, "media/audio/voice-samples/brandon2.wav"), + (HANASHI_USER_TOKEN, "m_s61a8g3jnddr3xdg3gctsjxxk9jwja", MediaFileType::Audio, "media/audio/voice-samples/elon.wav"), + (HANASHI_USER_TOKEN, "m_e59fh75g76pvebx4qe4r4aymd93q30", MediaFileType::Audio, "media/audio/voice-samples/goku.wav"), + (HANASHI_USER_TOKEN, "m_ttq0ym8ht4v0z87zqz27fhadp0743c", MediaFileType::Audio, "media/audio/voice-samples/hillary.wav"), + (HANASHI_USER_TOKEN, "m_300bv6f141hv553fjf287p5gb76k37", MediaFileType::Audio, "media/audio/voice-samples/mario_impersonator.wav"), + (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), + (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), + (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + for (user_token, media_file_token, media_file_type, subdirectory_path) in media_files { + let user_token = UserToken::new_from_str(user_token); + let media_file_token = MediaFileToken::new_from_str(media_file_token); + + let mut media_file_path = seed_tool_data_root.clone(); + media_file_path.push(subdirectory_path); + + let result = seed_model( + &mysql_pool, + &media_file_token, + &user_token, + media_file_type, + &media_file_path, + maybe_bucket_clients, + ).await; + + match result { + Ok(_) => info!("Seeded {:?}", media_file_path), + Err(err) => warn!(r#" + Could not seed media file {} , {:?} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the media file on a previous invocation!) + "#, media_file_token, subdirectory_path, err), + } + } + + Ok(()) +} + +async fn seed_model( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, + user_token: &UserToken, + media_file_type: MediaFileType, + media_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients>, +) -> AnyhowResult<()> { + info!("Seeding Media file {:?} ...", media_file_path); + + let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut maybe_bucket_prefix = None; + let mut maybe_bucket_extension = None; + + if media_file_exists(mysql_pool, media_file_token).await? { + info!("Media file already seeded: {:?}", media_file_token); + return Ok(()) + } + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(media_file_path, bucket_clients).await?; + bucket_hash = bucket_details.bucket_hash; + maybe_bucket_prefix = bucket_details.maybe_bucket_prefix; + maybe_bucket_extension = bucket_details.maybe_bucket_extension; + } + + let maybe_mime_type = get_mimetype_for_file(media_file_path)?; + let file_size_bytes = file_size(media_file_path)?; + let sha256_checksum = sha256_hash_file(media_file_path)?; + + let maybe_filename = media_file_path + .file_name() + .map(|name| name.to_str()) + .flatten(); + + insert_media_file_from_cli_tool(InsertArgs { + pool: &mysql_pool, + media_file_type, + maybe_mime_type, + file_size_bytes, + sha256_checksum: &sha256_checksum, + maybe_origin_filename: maybe_filename, + maybe_creator_user_token: Some(user_token), + creator_set_visibility: Visibility::Public, + public_bucket_directory_hash: &bucket_hash, + maybe_public_bucket_prefix: maybe_bucket_prefix.as_deref(), + maybe_public_bucket_extension: maybe_bucket_extension.as_deref(), + }).await?; + + Ok(()) +} + +async fn media_file_exists( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, +) -> AnyhowResult { + + const CAN_SEE_DELETED : bool = true; + + let maybe_file = get_media_file( + &media_file_token, + CAN_SEE_DELETED, + mysql_pool + ).await?; + + Ok(maybe_file.is_some()) +} + +struct BucketDetails { + bucket_hash: String, + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +async fn seed_file_to_bucket( + media_file_path: &Path, + bucket_clients: &BucketClients, +) -> AnyhowResult { + + info!("Uploading media file {:?} ...", media_file_path); + + let maybe_bucket_prefix = Some("fakeyou_"); + + let maybe_bucket_extension = media_file_path + .extension() + .map(|extension| extension.to_str()) + .flatten(); + + let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension); + + let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf()); + + info!("Reading media file: {:?}", media_file_path); + + let bytes = file_read_bytes(media_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + info!("Uploading media file to bucket path: {:?}", bucket_path); + + let _r = bucket_clients.public.upload_file_with_content_type( + &bucket_path, + bytes.as_ref(), + mimetype + ).await?; + + Ok(BucketDetails { + bucket_hash: bucket_location.get_object_hash().to_string(), + maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()), + maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()), + }) +} + +// Temporary "test" to generate random tokens. +// #[cfg(test)] +// mod tests { +// use tokens::tokens::media_files::MediaFileToken; +// +// #[test] +// fn test() { +// // Print some randomly sampled tokens +// for _i in 0..50 { +// let token = MediaFileToken::generate(); +// println!("Token: {}", token); +// } +// // Deliberately fail the test to get the output +// assert_eq!(1, 2); +// } +// } diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 7a72bacf56..694780ffb9 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -123,6 +123,7 @@ pub async fn main() -> AnyhowResult<()> { maybe_mime_type, file_size_bytes: 0, // TODO sha256_checksum: "", // TODO + maybe_origin_filename: None, maybe_creator_user_token: None, creator_set_visibility, public_bucket_directory_hash: public_upload_path.get_object_hash(), diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index 43a543db7d..fb180eba35 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -19,7 +19,9 @@ pub struct InsertArgs<'a> { pub file_size_bytes: u64, pub sha256_checksum: &'a str, - pub maybe_creator_user_token: Option, + pub maybe_origin_filename: Option<&'a str>, + + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_set_visibility: Visibility, pub public_bucket_directory_hash: &'a str, @@ -58,6 +60,8 @@ SET file_size_bytes = ?, checksum_sha2 = ?, + maybe_origin_filename = ?, + public_bucket_directory_hash = ?, maybe_public_bucket_prefix = ?, maybe_public_bucket_extension = ?, @@ -77,6 +81,8 @@ SET args.file_size_bytes, args.sha256_checksum, + args.maybe_origin_filename, + args.public_bucket_directory_hash, args.maybe_public_bucket_prefix, args.maybe_public_bucket_extension, From 30d6bca9b2456faa8c05963a6437226adde225fb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Dec 2023 21:53:31 -0500 Subject: [PATCH 48/58] seed video files + fix idempotency --- .../src/seeding/seed_media_files.rs | 30 ++++++++++++++----- crates/cli/dev_upload_media_file/src/main.rs | 1 + .../src/queries/media_files/get_media_file.rs | 2 +- .../insert_media_file_from_cli_tool.rs | 8 ++++- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs index 5bca8981e2..15f8959ef6 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -28,14 +28,6 @@ pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Op // NB: This is idempotent and will only install the media files once. // Delete the records if you want to re-seed. let media_files = [ - // Images - (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"), - (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"), - (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"), - (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"), - (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"), - (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"), - (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"), // Audio (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), @@ -47,6 +39,27 @@ pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Op (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/image/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/image/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/image/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/image/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/image/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/image/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/image/face-animator-inputs/zelda.jpg"), + // Video + (HANASHI_USER_TOKEN, "m_gedarcfdkxx3zv8zz2wmwsnc1r8jbp", MediaFileType::Video, "media/video/storyteller-studio-renders/girl-rotating-room.mp4"), + (HANASHI_USER_TOKEN, "m_3s74yyd4jwrjdp17yx1694n29jv5gr", MediaFileType::Video, "media/video/storyteller-studio-renders/hanashi-run.mp4"), + (HANASHI_USER_TOKEN, "m_p6d64gm1wawz25bmdahfded3366b9f", MediaFileType::Video, "media/video/storyteller-studio-renders/mocap-kiss.mp4"), + (HANASHI_USER_TOKEN, "m_szp2c77z6je965k44vc22xn9432a88", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-girl.mp4"), + (HANASHI_USER_TOKEN, "m_mdwspz81zymw9hvmb1d3jm830t25xe", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-m.mp4"), + (HANASHI_USER_TOKEN, "m_prcs6xkm8phs5y4a2dfwemwjy6n6z3", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-girl.mp4"), + (HANASHI_USER_TOKEN, "m_5j889dzw08vn9pbrscx1sncff43ekp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-jww.mp4"), + (HANASHI_USER_TOKEN, "m_fwm2w6wbssez274ayz6x8ta0h1a9qp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pascal.mp4"), + (HANASHI_USER_TOKEN, "m_yqqkrt2srep6a35xg78114jn9epwg1", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pop.mp4"), + (HANASHI_USER_TOKEN, "m_m4r043kez14pm6vaamsecpz8fafh7g", MediaFileType::Video, "media/video/video-style-transfer/man-explaining.mp4"), + (HANASHI_USER_TOKEN, "m_81gq3hygqvrahg6ncxbtcpjap4f9nv", MediaFileType::Video, "media/video/video-style-transfer/woman-gradient-background.mp4"), + (HANASHI_USER_TOKEN, "m_dp9ykj3yasjx69g2fx2bpd26wwjz0f", MediaFileType::Video, "media/video/video-style-transfer/woman-red-hair.mp4"), ]; let seed_tool_data_root = get_seed_tool_data_root(); @@ -117,6 +130,7 @@ async fn seed_model( insert_media_file_from_cli_tool(InsertArgs { pool: &mysql_pool, + maybe_use_apriori_media_token: Some(media_file_token), media_file_type, maybe_mime_type, file_size_bytes, diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 694780ffb9..4112093b85 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -119,6 +119,7 @@ pub async fn main() -> AnyhowResult<()> { let (media_file_token, _id) = insert_media_file_from_cli_tool(InsertArgs { pool: &pool, + maybe_use_apriori_media_token: None, media_file_type, maybe_mime_type, file_size_bytes: 0, // TODO diff --git a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs index ec94000555..8dc215e82f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs @@ -13,7 +13,7 @@ use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct MediaFile { pub token: MediaFileToken, diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index fb180eba35..e829ba53fd 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -14,6 +14,10 @@ use tokens::tokens::users::UserToken; pub struct InsertArgs<'a> { pub pool: &'a MySqlPool, + /// If supplied, use this media token rather than generating a new one. + /// This is good to support idempotency in development and testing. + pub maybe_use_apriori_media_token: Option<&'a MediaFileToken>, + pub media_file_type: MediaFileType, pub maybe_mime_type: Option<&'a str>, pub file_size_bytes: u64, @@ -35,7 +39,9 @@ pub async fn insert_media_file_from_cli_tool( args: InsertArgs<'_> ) -> AnyhowResult<(MediaFileToken, u64)> { - let media_file_token = MediaFileToken::generate(); + let media_file_token = args.maybe_use_apriori_media_token + .map(|token| token.clone()) + .unwrap_or_else(|| MediaFileToken::generate()); let mut transaction = args.pool.begin().await?; From 795eb7a141851fa63a18c739d5b369309608475d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 02:15:06 -0500 Subject: [PATCH 49/58] spacing nit --- .../inference_job/config/inference-job.development.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index c16128c8d6..a8427bea58 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -18,11 +18,11 @@ REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" # should rename my folder # abs path to the folder that contains the inference code need to get that container in the repo need basepath? relative to everyones env VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" -#VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" -VALL_E_X_INFERENCE_COMMAND = "python3 main.py" -VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .venv/bin/activate" +#VALL_E_X_INFERENCE_ROOT_DIRECTORY="/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" +VALL_E_X_INFERENCE_COMMAND="python3 main.py" +VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .venv/bin/activate" -# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE = "" # fill this out for dev? +# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev? #VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" #VALL_E_X_INFERENCE_COMMAND="python3 main.py" #VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate" From 07750fa122614d81f948a2ce655d90c4427259cb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 02:55:23 -0500 Subject: [PATCH 50/58] refactor bucket modules --- crates/_types/buckets/src/private/mod.rs | 3 ++- .../zs_voices/bucket_directory.rs} | 4 ++-- .../zs_voices/bucket_file_path.rs} | 15 ++++++--------- .../_types/buckets/src/private/zs_voices/mod.rs | 2 ++ .../{directory.rs => bucket_directory.rs} | 3 +-- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../_types/buckets/src/public/media_files/mod.rs | 4 ++-- .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../buckets/src/public/media_uploads/mod.rs | 4 ++-- crates/_types/buckets/src/public/mod.rs | 1 - .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../src/public/voice_conversion_results/mod.rs | 4 ++-- .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 5 +++-- .../buckets/src/public/weight_uploads/mod.rs | 4 ++-- crates/_types/buckets/src/public/zs_voices/mod.rs | 2 -- .../src/seeding/model_weights.rs | 6 ++---- .../src/seeding/seed_media_files.rs | 2 +- .../src/seeding/zero_shot_tts.rs | 2 +- crates/cli/dev_upload_media_file/src/main.rs | 2 +- .../queries/media_uploads/insert_media_upload.rs | 2 +- .../queries/user_bookmarks/list_user_bookmarks.rs | 5 ++--- .../insert_dataset_sample_and_media_file.rs | 2 +- .../lipsync/sad_talker/download_audio_file.rs | 6 +++--- .../lipsync/sad_talker/download_image_file.rs | 4 ++-- .../job_types/lipsync/sad_talker/process_job.rs | 2 +- .../tacotron2_v2_early_fakeyou/upload_results.rs | 2 +- .../tts/vall_e_x/process_job_create_voice.rs | 6 +++--- .../tts/vall_e_x/process_job_inference_voice.rs | 6 +++--- .../src/job/job_types/vc/rvc_v2/process_job.rs | 4 ++-- .../job/job_types/vc/so_vits_svc/process_job.rs | 4 ++-- .../inference_job/get_inference_job_status.rs | 4 ++-- .../endpoints/media_files/get_media_file.rs | 2 +- .../media_uploads/common/handle_upload.rs | 2 +- .../endpoints/tts/get_tts_inference_job_status.rs | 2 +- .../voice_dataset_samples/upload_zs_sample.rs | 2 +- 38 files changed, 65 insertions(+), 71 deletions(-) rename crates/_types/buckets/src/{public/zs_voices/directory.rs => private/zs_voices/bucket_directory.rs} (96%) rename crates/_types/buckets/src/{public/zs_voices/file.rs => private/zs_voices/bucket_file_path.rs} (88%) create mode 100644 crates/_types/buckets/src/private/zs_voices/mod.rs rename crates/_types/buckets/src/public/media_files/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/media_files/{original_file.rs => bucket_file_path.rs} (96%) rename crates/_types/buckets/src/public/media_uploads/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/media_uploads/{original_file.rs => bucket_file_path.rs} (94%) rename crates/_types/buckets/src/public/voice_conversion_results/{directory.rs => bucket_directory.rs} (95%) rename crates/_types/buckets/src/public/voice_conversion_results/{original_file.rs => bucket_file_path.rs} (94%) rename crates/_types/buckets/src/public/weight_uploads/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/weight_uploads/{original_file.rs => bucket_file_path.rs} (94%) delete mode 100644 crates/_types/buckets/src/public/zs_voices/mod.rs diff --git a/crates/_types/buckets/src/private/mod.rs b/crates/_types/buckets/src/private/mod.rs index 27089e4df1..d3090254af 100644 --- a/crates/_types/buckets/src/private/mod.rs +++ b/crates/_types/buckets/src/private/mod.rs @@ -1 +1,2 @@ -pub mod private_path; \ No newline at end of file +pub mod private_path; +pub mod zs_voices; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/directory.rs b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/zs_voices/directory.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_directory.rs index 1cfc740769..9d4ecee304 100644 --- a/crates/_types/buckets/src/public/zs_voices/directory.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs @@ -1,6 +1,6 @@ use crockford::crockford_entropy_lower; -use crate::private::private_path::PrivatePath; +use crate::private::private_path::PrivatePath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. @@ -79,7 +79,7 @@ impl ZeroShotVoiceEmbeddingBucketDirectory { #[cfg(test)] mod tests { - use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; + use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/zs_voices/file.rs b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs similarity index 88% rename from crates/_types/buckets/src/public/zs_voices/file.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs index 6428936c7c..47948912ec 100644 --- a/crates/_types/buckets/src/public/zs_voices/file.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs @@ -3,11 +3,10 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::private::private_path::PrivatePath; -use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; +use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; // TODO: Generate these from a macro. - /// The original user upload file. /// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] @@ -74,11 +73,9 @@ impl ZeroShotVoiceEmbeddingBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; - mod with_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; #[test] pub fn generate_new_entropy() { @@ -97,8 +94,8 @@ mod tests { } mod without_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; use super::*; @@ -135,7 +132,7 @@ mod tests { #[test] pub fn get_object_hash() { let hash = "abcdefghijk"; - let file = MediaFileBucketPath::from_object_hash(hash, None, None); + let file = ZeroShotVoiceEmbeddingBucketPath::from_object_hash(hash, ModelCategory::Tts, ModelType::VallEx, 1); assert_eq!(file.get_object_hash(), hash); } } diff --git a/crates/_types/buckets/src/private/zs_voices/mod.rs b/crates/_types/buckets/src/private/zs_voices/mod.rs new file mode 100644 index 0000000000..29c183a5c6 --- /dev/null +++ b/crates/_types/buckets/src/private/zs_voices/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_files/directory.rs b/crates/_types/buckets/src/public/media_files/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/directory.rs rename to crates/_types/buckets/src/public/media_files/bucket_directory.rs index e0083da128..855332bcf3 100644 --- a/crates/_types/buckets/src/public/media_files/directory.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_directory.rs @@ -4,7 +4,6 @@ use crate::public::public_path::PublicPath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. - // TODO: Use a central path registry for quick reference const MEDIA_FILE_DIRECTORY: &str = "/media"; @@ -47,7 +46,7 @@ impl MediaFileBucketDirectory { #[cfg(test)] mod tests { - use crate::public::media_files::directory::MediaFileBucketDirectory; + use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_files/original_file.rs b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/original_file.rs rename to crates/_types/buckets/src/public/media_files/bucket_file_path.rs index efd0fd9458..e503819440 100644 --- a/crates/_types/buckets/src/public/media_files/original_file.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_files::directory::MediaFileBucketDirectory; +use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "file.bin"; @@ -79,7 +79,7 @@ impl MediaFileBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; + use crate::public::media_files::bucket_file_path::MediaFileBucketPath; mod with_prefix_and_extension { use super::*; diff --git a/crates/_types/buckets/src/public/media_files/mod.rs b/crates/_types/buckets/src/public/media_files/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_files/mod.rs +++ b/crates/_types/buckets/src/public/media_files/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_uploads/directory.rs b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_uploads/directory.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_directory.rs index aa4ffc0470..77ff4ca662 100644 --- a/crates/_types/buckets/src/public/media_uploads/directory.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs @@ -47,7 +47,7 @@ impl MediaUploadDirectory { #[cfg(test)] mod tests { - use crate::public::media_uploads::directory::MediaUploadDirectory; + use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/original_file.rs b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/media_uploads/original_file.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs index 7f0a40a6b7..6867ff6e5a 100644 --- a/crates/_types/buckets/src/public/media_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; @@ -61,7 +61,7 @@ impl MediaUploadOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + use crate::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/mod.rs b/crates/_types/buckets/src/public/media_uploads/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_uploads/mod.rs +++ b/crates/_types/buckets/src/public/media_uploads/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index dc35c40ed4..fcceb68d9b 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,5 +2,4 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod zs_voices; pub mod weight_uploads; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs similarity index 95% rename from crates/_types/buckets/src/public/voice_conversion_results/directory.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs index c853cf2d2f..4f726b2c14 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs @@ -40,7 +40,7 @@ impl VoiceConversionResultDirectory { #[cfg(test)] mod tests { - use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; + use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; #[test] pub fn get_directory_path_str() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/voice_conversion_results/original_file.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs index 9e256e89d2..9163363e49 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::public::public_path::PublicPath; -use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; +use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; const ORIGINAL_FILE_PREFIX : &str = "fakeyou_"; const ORIGINAL_FILE_SUFFIX : &str = ".wav"; @@ -65,7 +65,7 @@ impl VoiceConversionResultOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; + use crate::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/directory.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/weight_uploads/directory.rs rename to crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs index fcedda62dd..1f3e016fb1 100644 --- a/crates/_types/buckets/src/public/weight_uploads/directory.rs +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs @@ -48,7 +48,7 @@ impl WeightUploadDirectory { #[cfg(test)] mod tests { - use crate::public::weight_uploads::directory::WeightUploadDirectory; + use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/weight_uploads/original_file.rs rename to crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs index 42f203f849..69ca626486 100644 --- a/crates/_types/buckets/src/public/weight_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::weight_uploads::directory::WeightUploadDirectory; use crate::public::public_path::PublicPath; +use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; @@ -56,7 +56,8 @@ impl WeightUploadOriginalFilePath { #[cfg(test)] mod tests { use std::path::PathBuf; - use crate::public::weight_uploads::original_file::WeightUploadOriginalFilePath; + + use crate::public::weight_uploads::bucket_file_path::WeightUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/weight_uploads/mod.rs b/crates/_types/buckets/src/public/weight_uploads/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/weight_uploads/mod.rs +++ b/crates/_types/buckets/src/public/weight_uploads/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/mod.rs b/crates/_types/buckets/src/public/zs_voices/mod.rs deleted file mode 100644 index a5c0d4bfab..0000000000 --- a/crates/_types/buckets/src/public/zs_voices/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod directory; -pub mod file; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index e766086a0b..8860d323c7 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -16,7 +16,7 @@ use tokens::tokens::users::UserToken; use crate::seeding::users::HANASHI_USERNAME; -pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { +pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserToken) -> AnyhowResult<()> { let sd1_5_markdown_description = r#" # Dragonfruit AI Models Update and Workflow @@ -81,9 +81,7 @@ pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: User *Thank you for your continued support and feedback!* "#; -let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; - - let user_token = user_token; + let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; // create a loop that loops from 1 to 100 for i in 1..=100 { diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs index 15f8959ef6..1e3f2c3c80 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -3,7 +3,7 @@ use std::path::Path; use log::{info, warn}; use sqlx::{MySql, Pool}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use errors::AnyhowResult; diff --git a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs index d01c359d13..196c36b6ef 100644 --- a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs @@ -1,7 +1,7 @@ use log::info; use sqlx::{MySql, Pool}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 4112093b85..3928071c5d 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -3,7 +3,7 @@ use std::time::Duration; use log::info; use sqlx::mysql::MySqlPoolOptions; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; use enums::by_table::media_files::media_file_type::MediaFileType; diff --git a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs index 043a3b81f3..667e3751f1 100644 --- a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index 1e1896a750..232c7344a6 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -4,10 +4,9 @@ use sqlx::MySqlPool; use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; -use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; -use tokens::tokens::user_bookmarks::UserBookmarkToken; +use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; +use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; pub async fn list_user_bookmarks( username: &str, diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs index c12bf4e62a..0abda7cd4b 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use sqlx::MySqlPool; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 4e6f8973e7..e90e3fc887 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -5,9 +5,9 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 3d401f0a7a..8285a78c5a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -5,8 +5,8 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index cb6d563156..57852d2b4b 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -4,7 +4,7 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs index 87e9747a3d..e2a7547f42 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use log::info; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index f83b5b764c..e210b20259 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -5,9 +5,9 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; -use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; +use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; +use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index 20359f1e0d..76f4940796 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -5,9 +5,9 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; -use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; +use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; +use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 74496b6abc..158f304a2b 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -5,8 +5,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index f5d7fa1b89..e02716e323 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -4,8 +4,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 96be5ed7d7..ea3e6f8f15 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -9,8 +9,8 @@ use chrono::{DateTime, Utc}; use log::error; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::common::job_status_plus::JobStatusPlus; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs index 021d4db258..b18f8af8da 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::warn; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use mysql_queries::queries::media_files::get_media_file::get_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 792af74603..2d79cab2e7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -5,7 +5,7 @@ use actix_multipart::Multipart; use actix_web::{HttpRequest, web}; use log::{error, info, warn}; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index be43c33d8a..20dab038b7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::{error, log}; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index b257ce9c14..004d303f44 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -6,7 +6,7 @@ use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use hyper::StatusCode; use log::{error, info, warn}; use once_cell::sync::Lazy; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType::MediaUpload; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; From d9fe4e56a0556426ad041f35c07dae8007499ef8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 08:00:32 +0000 Subject: [PATCH 51/58] Bump version-sync from 0.8.1 to 0.9.5 Bumps [version-sync](https://github.com/mgeisler/version-sync) from 0.8.1 to 0.9.5. - [Release notes](https://github.com/mgeisler/version-sync/releases) - [Commits](https://github.com/mgeisler/version-sync/compare/0.8.1...0.9.5) --- updated-dependencies: - dependency-name: version-sync dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 460 +++++++++++++--------------- crates/vendor/limitation/Cargo.toml | 2 +- 2 files changed, 209 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fe281f68c..cfd5d64312 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,7 +78,7 @@ dependencies = [ "log", "mime", "mime_guess", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", ] @@ -110,7 +110,7 @@ dependencies = [ "language-tags", "local-channel", "mime", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "rand 0.8.5", "sha1 0.10.6", @@ -127,7 +127,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -164,8 +164,8 @@ checksum = "0a0a77f836d869f700e5b47ac7c3c8b9c8bc82e4aec861954c6198abee3ebd4d" dependencies = [ "darling 0.20.3", "parse-size", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -268,7 +268,7 @@ dependencies = [ "smallvec 1.11.2", "socket2 0.5.5", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -296,8 +296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -307,8 +307,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -416,8 +416,8 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9afe97decc070be1864d96ab118383abed1c58fc09696663666049431110c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -633,8 +633,8 @@ version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -681,7 +681,7 @@ dependencies = [ "native-tls", "serde", "serde_json", - "url 2.5.0", + "url", ] [[package]] @@ -715,7 +715,7 @@ dependencies = [ "serde", "thiserror", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -811,7 +811,7 @@ dependencies = [ "serde_json", "sqlx", "tokio", - "toml", + "toml 0.5.11", "url_config", ] @@ -1142,8 +1142,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1252,7 +1252,7 @@ dependencies = [ "anyhow", "easyenv", "my-workspace-hack", - "url 2.5.0", + "url", ] [[package]] @@ -1278,7 +1278,7 @@ dependencies = [ "rand 0.8.5", "serde", "tempfile", - "toml", + "toml 0.5.11", "uuid 0.8.2", ] @@ -1294,7 +1294,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "time 0.3.30", "version_check", ] @@ -1485,7 +1485,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1527,8 +1527,8 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1541,8 +1541,8 @@ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1555,8 +1555,8 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 2.0.39", ] @@ -1568,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1579,7 +1579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core 0.14.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1590,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1693,8 +1693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling 0.14.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -1715,8 +1715,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1788,8 +1788,8 @@ dependencies = [ "byteorder", "diesel_derives", "mysqlclient-sys", - "percent-encoding 2.3.1", - "url 2.5.0", + "percent-encoding", + "url", ] [[package]] @@ -1799,8 +1799,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ "diesel_table_macro_syntax", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1875,8 +1875,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -2030,13 +2030,13 @@ dependencies = [ "bytes 1.5.0", "dyn-clone", "lazy_static", - "percent-encoding 2.3.1", + "percent-encoding", "reqwest", "rustc_version 0.2.3", "serde", "serde_json", "serde_with", - "url 2.5.0", + "url", "void", ] @@ -2177,7 +2177,7 @@ dependencies = [ "sqlx-core", "sqlx-mysql", "strum", - "toml", + "toml 0.5.11", "utoipa", ] @@ -2347,7 +2347,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", ] [[package]] @@ -2475,8 +2475,8 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -2743,8 +2743,8 @@ dependencies = [ "log", "mac", "markup5ever", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -2828,7 +2828,7 @@ dependencies = [ "serde_json", "serde_qs 0.8.5", "serde_urlencoded", - "url 2.5.0", + "url", ] [[package]] @@ -2909,7 +2909,7 @@ dependencies = [ "http 1.0.0", "http-body 1.0.0", "pin-project-lite", - "tokio 1.34.0", + "tokio", "want", ] @@ -2937,7 +2937,7 @@ dependencies = [ "hyper 1.0.1", "hyper-util", "native-tls", - "tokio 1.34.0", + "tokio", "tokio-native-tls", "tower-service", ] @@ -2956,7 +2956,7 @@ dependencies = [ "hyper 1.0.1", "pin-project-lite", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "tower", "tower-service", "tracing", @@ -2999,17 +2999,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -3176,15 +3165,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.10.5" @@ -3330,7 +3310,7 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna 0.5.0", + "idna", "mime", "native-tls", "nom", @@ -3338,7 +3318,7 @@ dependencies = [ "quoted_printable", "socket2 0.5.5", "tokio", - "url 2.5.0", + "url", ] [[package]] @@ -3539,20 +3519,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "maybe-async" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3758,8 +3732,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3843,7 +3817,7 @@ dependencies = [ "twitch_oauth2", "unicode-bidi", "unicode-normalization", - "url 2.5.0", + "url", "void", "winapi 0.3.9", "windows-sys 0.48.0", @@ -3875,7 +3849,7 @@ dependencies = [ "sqlx", "tokens", "tokio", - "toml", + "toml 0.5.11", ] [[package]] @@ -4147,8 +4121,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4298,12 +4272,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -4367,8 +4335,8 @@ checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4405,8 +4373,8 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4517,8 +4485,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", "version_check", ] @@ -4529,20 +4497,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "version_check", ] -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid", -] - [[package]] name = "proc-macro2" version = "1.0.69" @@ -4561,17 +4520,6 @@ dependencies = [ "cc", ] -[[package]] -name = "pulldown-cmark" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1b74cc784b038a9921fd1a48310cc2e238101aa8ae0b94201e2d85121dd68b5" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4613,22 +4561,13 @@ dependencies = [ "memchr", ] -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.69", + "proc-macro2", ] [[package]] @@ -4797,7 +4736,7 @@ dependencies = [ "dtoa 0.4.8", "futures 0.1.31", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", "tokio-codec", "tokio-executor", @@ -4805,7 +4744,7 @@ dependencies = [ "tokio-sync", "tokio-tcp", "tokio-uds", - "url 2.5.0", + "url", ] [[package]] @@ -4818,9 +4757,9 @@ dependencies = [ "combine 4.6.6", "dtoa 0.4.8", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", - "url 2.5.0", + "url", ] [[package]] @@ -4834,13 +4773,13 @@ dependencies = [ "combine 4.6.6", "futures-util", "itoa 1.0.9", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "ryu", "sha1 0.6.1", "tokio", "tokio-util", - "url 2.5.0", + "url", ] [[package]] @@ -4955,7 +4894,7 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "serde", "serde_json", @@ -4965,7 +4904,7 @@ dependencies = [ "tokio-native-tls", "tokio-util", "tower-service", - "url 2.5.0", + "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", @@ -4982,7 +4921,7 @@ dependencies = [ "serde", "serde_derive", "sqlx", - "toml", + "toml 0.5.11", ] [[package]] @@ -5063,8 +5002,8 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rust-embed-utils", "shellexpand", "syn 2.0.39", @@ -5111,7 +5050,7 @@ dependencies = [ "maybe-async", "md5", "minidom", - "percent-encoding 2.3.1", + "percent-encoding", "quick-xml 0.26.0", "reqwest", "serde", @@ -5121,7 +5060,7 @@ dependencies = [ "time 0.3.30", "tokio", "tokio-stream", - "url 2.5.0", + "url", ] [[package]] @@ -5380,7 +5319,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0", + "semver-parser", ] [[package]] @@ -5395,12 +5334,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "semver-parser" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46e1121e8180c12ff69a742aabc4f310542b6ccb69f1691689ac17fdf8618aa" - [[package]] name = "serde" version = "1.0.193" @@ -5416,8 +5349,8 @@ version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5457,7 +5390,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5468,7 +5401,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cac3f1e2ca2fe333923a1ae72caca910b98ed0630bb35ef6f8c8517d6e81afa" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5479,11 +5412,20 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5513,8 +5455,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5538,8 +5480,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5692,8 +5634,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5824,7 +5766,7 @@ dependencies = [ "native-tls", "once_cell", "paste", - "percent-encoding 2.3.1", + "percent-encoding", "rustls 0.21.9", "rustls-pemfile", "serde", @@ -5836,7 +5778,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "url 2.5.0", + "url", "webpki-roots 0.25.3", ] @@ -5846,8 +5788,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "sqlx-core", "sqlx-macros-core", "syn 1.0.109", @@ -5865,8 +5807,8 @@ dependencies = [ "heck", "hex", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "serde", "serde_json", "sha2 0.10.8", @@ -5877,7 +5819,7 @@ dependencies = [ "syn 1.0.109", "tempfile", "tokio", - "url 2.5.0", + "url", ] [[package]] @@ -5909,7 +5851,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "rand 0.8.5", "rsa", "serde", @@ -5979,11 +5921,11 @@ dependencies = [ "futures-util", "libsqlite3-sys", "log", - "percent-encoding 2.3.1", + "percent-encoding", "serde", "sqlx-core", "tracing", - "url 2.5.0", + "url", "urlencoding", ] @@ -6086,7 +6028,7 @@ dependencies = [ "native-tls", "once_cell", "pin-project-lite", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_caching", @@ -6107,7 +6049,7 @@ dependencies = [ "time 0.3.30", "tokens", "tokio", - "toml", + "toml 0.5.11", "tts_common", "twitch_common", "twitch_oauth2", @@ -6150,8 +6092,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", ] [[package]] @@ -6193,8 +6135,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustversion", "syn 2.0.39", ] @@ -6390,25 +6332,14 @@ dependencies = [ "symphonia-metadata", ] -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid", -] - [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6418,8 +6349,8 @@ version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6525,8 +6456,8 @@ version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6619,7 +6550,7 @@ dependencies = [ "sqlx", "sqlx-core", "strum", - "toml", + "toml 0.5.11", "utoipa", ] @@ -6680,8 +6611,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6818,6 +6749,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -6828,7 +6793,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tower-layer", "tower-service", "tracing", @@ -6864,8 +6829,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6984,7 +6949,7 @@ dependencies = [ "rustls-native-certs", "sha-1", "thiserror", - "url 2.5.0", + "url", "utf-8", "webpki", ] @@ -7014,7 +6979,7 @@ dependencies = [ "my-workspace-hack", "mysql_queries", "native-tls", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_common", @@ -7029,7 +6994,7 @@ dependencies = [ "tokens", "tokio", "tokio-tungstenite", - "toml", + "toml 0.5.11", "tts_common", "twitch_api2", "twitch_common", @@ -7059,7 +7024,7 @@ dependencies = [ "thiserror", "twitch_oauth2", "typed-builder", - "url 2.5.0", + "url", "version_check", ] @@ -7095,7 +7060,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "url 2.5.0", + "url", ] [[package]] @@ -7104,8 +7069,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -7157,12 +7122,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -7190,17 +7149,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - [[package]] name = "url" version = "2.5.0" @@ -7208,8 +7156,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", - "percent-encoding 2.3.1", + "idna", + "percent-encoding", "serde", ] @@ -7236,7 +7184,7 @@ version = "0.0.1" dependencies = [ "my-workspace-hack", "once_cell", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "regex", "unicode-segmentation", ] @@ -7282,7 +7230,7 @@ dependencies = [ "strum_macros", "time 0.3.30", "tokens", - "toml", + "toml 0.5.11", "user_input_common", ] @@ -7317,8 +7265,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "regex", "syn 2.0.39", ] @@ -7368,18 +7316,17 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version-sync" -version = "0.8.1" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "844f3d3a2467f15cb999f5af7775f6e108ac546d4f42365832ed4c755404f806" +checksum = "835169da0173ea373ddf5987632aac1f918967fbbe58195e304342282efa6089" dependencies = [ - "itertools 0.8.2", - "proc-macro2 0.4.30", - "pulldown-cmark 0.4.1", + "proc-macro2", + "pulldown-cmark", "regex", - "semver-parser 0.9.0", - "syn 0.15.44", - "toml", - "url 1.7.2", + "semver 1.0.20", + "syn 2.0.39", + "toml 0.7.8", + "url", ] [[package]] @@ -7516,8 +7463,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-shared", ] @@ -7540,7 +7487,7 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ - "quote 1.0.33", + "quote", "wasm-bindgen-macro-support", ] @@ -7550,8 +7497,8 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7681,7 +7628,7 @@ dependencies = [ "sqlx", "time 0.3.30", "tokio", - "toml", + "toml 0.5.11", "twitch_api2", "twitch_common", "twitch_oauth2", @@ -7884,6 +7831,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -7919,8 +7875,8 @@ version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 51f1626635..d13d4d9d2d 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -28,4 +28,4 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] tokio = "1.34.0" -version-sync = "0.8.1" +version-sync = "0.9.5" From f9ff9b8c9ffd6ec520418d970bfd73ce5308e661 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 11:57:42 -0500 Subject: [PATCH 52/58] Kill newrelic. This is painful to keep upgrading Keeping the vendor directory because we may introduce later. We've already done some upkeep. --- Cargo.lock | 189 +++--------------- Cargo.toml | 3 +- .../service/job/email_sender_job/Cargo.toml | 3 - crates/service/job/inference_job/Cargo.toml | 3 - .../job/inference_job/src/job_dependencies.rs | 5 - crates/service/job/inference_job/src/main.rs | 9 - .../service/job/tts_inference_job/Cargo.toml | 3 - .../src/job_steps/job_args.rs | 5 - .../src/job_steps/process_single_job.rs | 47 +---- .../service/job/tts_inference_job/src/main.rs | 88 +++----- 10 files changed, 64 insertions(+), 291 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfd5d64312..25191d742f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http 0.2.11", + "http", "httparse", "httpdate", "itoa 1.0.9", @@ -176,7 +176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "http 0.2.11", + "http", "regex", "serde", "tracing", @@ -614,8 +614,8 @@ dependencies = [ "hex", "hmac 0.12.1", "http-types", - "hyper 0.14.27", - "hyper-tls 0.5.0", + "hyper", + "hyper-tls", "serde", "serde_json", "serde_path_to_error", @@ -676,7 +676,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ - "http 0.2.11", + "http", "log", "native-tls", "serde", @@ -1924,7 +1924,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "mimetypes", @@ -2114,7 +2114,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "images", "jobs_common", "lfu", @@ -2599,7 +2599,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", + "http", "indexmap", "slab", "tokio", @@ -2759,17 +2759,6 @@ dependencies = [ "itoa 1.0.9", ] -[[package]] -name = "http" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" -dependencies = [ - "bytes 1.5.0", - "fnv", - "itoa 1.0.9", -] - [[package]] name = "http-body" version = "0.4.5" @@ -2777,30 +2766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http 0.2.11", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" -dependencies = [ - "bytes 1.5.0", - "http 1.0.0", -] - -[[package]] -name = "http-body-util" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" -dependencies = [ - "bytes 1.5.0", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", + "http", "pin-project-lite", ] @@ -2820,7 +2786,7 @@ dependencies = [ "async-channel", "base64 0.13.1", "futures-lite", - "http 0.2.11", + "http", "infer 0.2.3", "pin-project-lite", "rand 0.7.3", @@ -2884,8 +2850,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", - "http-body 0.4.5", + "http", + "http-body", "httparse", "httpdate", "itoa 1.0.9", @@ -2897,22 +2863,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" -dependencies = [ - "bytes 1.5.0", - "futures-channel", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "pin-project-lite", - "tokio", - "want", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -2920,48 +2870,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.5.0", - "hyper 0.14.27", + "hyper", "native-tls", "tokio", "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes 1.5.0", - "http-body-util", - "hyper 1.0.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" -dependencies = [ - "bytes 1.5.0", - "futures-channel", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "hyper 1.0.1", - "pin-project-lite", - "socket2 0.5.5", - "tokio", - "tower", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.58" @@ -3095,7 +3009,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "images", "jobs_common", "lfu", @@ -3105,7 +3019,6 @@ dependencies = [ "mimetypes", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "once_cell", "r2d2_redis", "redis_common", @@ -3772,7 +3685,7 @@ dependencies = [ "hashbrown 0.14.2", "heck", "hmac 0.12.1", - "hyper 0.14.27", + "hyper", "indexmap", "lazy_static", "libc", @@ -3903,25 +3816,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" -[[package]] -name = "newrelic-telemetry" -version = "0.1.0" -dependencies = [ - "anyhow", - "env_logger", - "flate2", - "futures 0.3.29", - "hyper 0.14.27", - "hyper-tls 0.6.0", - "log", - "my-workspace-hack", - "rand 0.8.5", - "serde", - "serde_json", - "tokio", - "uuid 0.8.2", -] - [[package]] name = "nix" version = "0.26.4" @@ -4884,10 +4778,10 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", - "http-body 0.4.5", - "hyper 0.14.27", - "hyper-tls 0.5.0", + "http", + "http-body", + "hyper", + "hyper-tls", "ipnet", "js-sys", "log", @@ -5045,7 +4939,7 @@ dependencies = [ "futures 0.3.29", "hex", "hmac 0.12.1", - "http 0.2.11", + "http", "log", "maybe-async", "md5", @@ -6009,7 +5903,7 @@ dependencies = [ "hmac 0.10.1", "hostname", "http_server_common", - "hyper 0.14.27", + "hyper", "infer 0.5.0", "jwt", "language-tags", @@ -6783,28 +6677,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -6865,7 +6737,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -6896,14 +6768,13 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "lfu", "log", "memory_caching", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "r2d2_redis", "serde", "serde_derive", @@ -6941,7 +6812,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http 0.2.11", + "http", "httparse", "log", "rand 0.8.5", @@ -6971,7 +6842,7 @@ dependencies = [ "futures 0.3.29", "futures-util", "hostname", - "hyper 0.14.27", + "hyper", "lazy_static", "log", "magic-crypt", @@ -7013,7 +6884,7 @@ dependencies = [ "chrono", "displaydoc", "futures 0.3.29", - "http 0.2.11", + "http", "once_cell", "reqwest", "serde", @@ -7053,7 +6924,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "displaydoc", - "http 0.2.11", + "http", "once_cell", "rand 0.8.5", "reqwest", @@ -7357,7 +7228,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -7386,7 +7257,7 @@ dependencies = [ "filesys", "google_drive_common", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -7606,7 +7477,7 @@ dependencies = [ "futures-util", "hostname", "http_server_common", - "hyper 0.14.27", + "hyper", "lazy_static", "limitation", "log", diff --git a/Cargo.toml b/Cargo.toml index 7649065a0d..1ea5d8fefc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,8 @@ members = [ # Vendored code "crates/vendor/limitation", - "crates/vendor/newrelic-telemetry-sdk-rust", + # NB(bt,2023-12-12): Newrelic keeps breaking with upgrades. Kill for now. + #"crates/vendor/newrelic-telemetry-sdk-rust", # cargo-hakari workspace hack (compilation speedup hack) "crates/my-workspace-hack", diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index f72b7f7fba..733388d420 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -44,9 +44,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -# newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 9f8205ca14..199d70f522 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -41,9 +41,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 504d2fb08d..79d8d703c1 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -22,7 +22,6 @@ use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmen use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; -use newrelic_telemetry::Client as NewRelicClient; use crate::job_specific_dependencies::JobSpecificDependencies; use crate::util::scoped_execution::ScopedExecution; @@ -135,10 +134,6 @@ pub struct ClientDependencies { pub job_progress_reporter: Box, pub firehose_publisher: FirehosePublisher, - - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, } pub struct FileSystemDetails { diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 7779535799..186e87c9f4 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -44,7 +44,6 @@ use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use memory_caching::ttl_key_counter::TtlKeyCounter; use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use newrelic_telemetry::ClientBuilder; use crate::http_server::run_http_server::CreateServerArgs; use crate::http_server::run_http_server::launch_http_server; @@ -149,12 +148,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -237,8 +230,6 @@ async fn main() -> AnyhowResult<()> { clients: ClientDependencies { job_progress_reporter, firehose_publisher, - newrelic_client, - newrelic_disabled, }, job: JobSystemDependencies { system: JobSystemControls { diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index cbfdd24de1..2d1f6fa732 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -30,9 +30,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs index 433c04771f..3b4296a0d3 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs @@ -10,7 +10,6 @@ use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::TtsModelForInferenceRecord; -use newrelic_telemetry::Client as NewRelicClient; use crate::caching::cache_miss_strategizer_multi::SyncMultiCacheMissStrategizer; use crate::caching::virtual_lfu_cache::SyncVirtualLfuCache; @@ -38,10 +37,6 @@ pub struct JobArgs { pub job_stats: JobStats, - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, - pub worker_details: JobWorkerDetails, // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index bb9cd522a8..1767a674e2 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -6,14 +6,13 @@ use std::fs::File; use std::io::Read; use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; use container_common::anyhow_result::AnyhowResult; -use container_common::token::random_uuid::generate_random_uuid; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; @@ -24,7 +23,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_done use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceRecord}; use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; -use newrelic_telemetry::Span; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; @@ -56,34 +54,13 @@ pub async fn process_single_job( job_args: &JobArgs, job: &AvailableTtsInferenceJob, cached_model_record: &TtsModelForInferenceRecord, -) -> Result<(Span, Span), ProcessSingleJobError> { +) -> Result<(), ProcessSingleJobError> { // NB: Hack to allow cache bypass for debug requests // This will let us iterate faster with model changes. let mut maybe_queried_model = None; let mut model_record = cached_model_record; - let start = Instant::now(); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - let maybe_user_token = job.maybe_creator_user_token.as_deref().unwrap_or(""); - - let mut job_iteration_span = Span::new(&span_id, &trace_id, get_timestamp_millis()) - .name("single job execution") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - - let created_at_timestamp = (job.created_at.timestamp() as u64) * 1000; - - let mut since_creation_span = Span::new(&span_id, &trace_id, created_at_timestamp) - .name("job since creation") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - let mut job_progress_reporter = job_args .job_progress_reporter .new_tts_inference(&job.inference_job_token) @@ -100,15 +77,7 @@ pub async fn process_single_job( if !lock_acquired { warn!("Could not acquire job lock for: {:?}", &job.id); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "failure"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "failure"); - job_iteration_span.set_duration(duration); - - return Ok((since_creation_span, job_iteration_span)); + return Ok(()); } info!("Lock acquired for job: {}", job.inference_job_token); @@ -448,15 +417,7 @@ pub async fn process_single_job( info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", job.id, id, &inference_result_token); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "success"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "success"); - job_iteration_span.set_duration(duration); - - Ok((since_creation_span, job_iteration_span)) + Ok(()) } fn get_timestamp_millis() -> u64 { diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index 9738caa13c..db471482ac 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -45,8 +45,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::list_available_tts_inferenc use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; -use newrelic_telemetry::ClientBuilder; -use newrelic_telemetry::Span; use crate::caching::cache_miss_strategizer::CacheMissStrategizer; use crate::caching::cache_miss_strategizer::CacheMissStrategy; @@ -253,12 +251,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("TTS_MODEL_RECORD_CACHE_MILLIS", 300_000)?, // Five minutes ); - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -301,8 +293,6 @@ async fn main() -> AnyhowResult<()> { tts_sidecar_health_check_client, }, job_stats: JobStats::new(), - newrelic_client, - newrelic_disabled, worker_details: JobWorkerDetails { is_on_prem, worker_hostname: server_hostname.clone(), @@ -344,8 +334,6 @@ async fn main_loop(job_args: JobArgs) { let mut noop_logger = NoOpLogger::new(job_args.no_op_logger_millis as i64); - let mut span_batch = Vec::new(); - let mut sort_by_priority = true; let mut sort_by_priority_count = 0; @@ -411,23 +399,11 @@ async fn main_loop(job_args: JobArgs) { needs_health_check_at_start = false; } - let mut spans = match batch_result { - Ok(spans) => spans, - Err(e) => { - warn!("Error running job batch: {:?}", e); - std::thread::sleep(Duration::from_millis(error_timeout_millis)); - error_timeout_millis += INCREASE_TIMEOUT_MILLIS; - continue; - } - }; - - if !job_args.newrelic_disabled { - span_batch.append(&mut spans); - - if span_batch.len() > 50 { - let spans_to_send = span_batch.split_off(0).into(); - job_args.newrelic_client.send_spans(spans_to_send).await; - } + if let Err(e) = batch_result { + warn!("Error running job batch: {:?}", e); + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; } error_timeout_millis = START_TIMEOUT_MILLIS; // reset @@ -441,7 +417,7 @@ async fn process_jobs( inferencer: &JobArgs, jobs: Vec, needs_health_check_at_start: bool, -) -> AnyhowResult> { +) -> AnyhowResult<()> { if needs_health_check_at_start { maybe_block_on_sidecar_health_check(&inferencer.http_clients.tts_sidecar_health_check_client).await; @@ -449,8 +425,6 @@ async fn process_jobs( let mut maybe_sidecar_health_issue = false; - let mut batch_spans = Vec::new(); - for job in jobs.into_iter() { let model_state_result = ModelState::query_model_and_check_filesystem( &job, @@ -555,44 +529,38 @@ async fn process_jobs( } let result = process_single_job(inferencer, &job, &model_state.model_record).await; - match result { - Ok((span1, span2)) => { - batch_spans.push(span1); - batch_spans.push(span2); - }, - Err(e) => { - warn!("Failure to process job: {:?}", e); + if let Err(e) = result { + warn!("Failure to process job: {:?}", e); - record_failure_and_maybe_slow_down(&inferencer.job_stats); + record_failure_and_maybe_slow_down(&inferencer.job_stats); - maybe_sidecar_health_issue = true; + maybe_sidecar_health_issue = true; - let failure_reason = "failure processing job"; - let internal_debugging_failure_reason = format!("job error: {:?}", e); + let failure_reason = "failure processing job"; + let internal_debugging_failure_reason = format!("job error: {:?}", e); - let _r = mark_tts_inference_job_failure( - &inferencer.mysql_pool, - &job, - failure_reason, - &internal_debugging_failure_reason, - inferencer.job_max_attempts, - &inferencer.get_worker_name(), - ).await; + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + &internal_debugging_failure_reason, + inferencer.job_max_attempts, + &inferencer.get_worker_name(), + ).await; - match e { - ProcessSingleJobError::Other(_) => {} // No-op - ProcessSingleJobError::FilesystemFull => { - // TODO: Refactor - we should stop processing all of these jobs as we'll lose out - // on this entire batch by attempting to clear the filesystem. This should be handled - // in the calling code. - delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; - } + match e { + ProcessSingleJobError::Other(_) => {} // No-op + ProcessSingleJobError::FilesystemFull => { + // TODO: Refactor - we should stop processing all of these jobs as we'll lose out + // on this entire batch by attempting to clear the filesystem. This should be handled + // in the calling code. + delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; } } } } - Ok(batch_spans) + Ok(()) } fn record_failure_and_maybe_slow_down(job_stats: &JobStats) { From e5a85c752821c75f610f0dc2bd794dbc4974edfd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 18:56:02 -0500 Subject: [PATCH 53/58] fix failing doctest --- crates/vendor/limitation/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs index 649b30eb46..b40cf6b1fe 100644 --- a/crates/vendor/limitation/src/lib.rs +++ b/crates/vendor/limitation/src/lib.rs @@ -42,9 +42,12 @@ //! // The key to count and track //! let key = "10.0.0.5"; //! +//! // TODO(bt,2023-12-12): This doctest is so far behind in `futures` versions and needs to be updated. +//! // Tokio should be executing the future and evaluating the result. //! // Start and run a Tokio runtime to drive the Future to completion -//! tokio::run( -//! limiter +//! //let mut rt = tokio::runtime::Runtime::new().unwrap(); +//! //rt.block_on(...)//! +//! let _r = limiter //! // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` //! // containing a Status if the limit has been exceeded //! .count(key) @@ -57,8 +60,9 @@ //! .and_then(|status| { //! println!("ok: {:?}", status); //! Ok(()) -//! }), -//! ); +//! }) +//! .wait(); +//! //); //! # Ok::<(), limitation::Error>(()) //! ``` //! From e375c8090c658bd52ee6d8cd73b0078bdf457343 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 19:21:42 -0500 Subject: [PATCH 54/58] fix IDE handling (code compiled and tested, but IDE unhappy) --- .../billing/src/stripe/traits/internal_session_cache_purge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index d125c53d7f..df5bae0773 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; /// Allows us to purge session caches that might keep stale premium plan/billing data around -#[cfg_attr(test, automock)] +#[cfg_attr(test, mockall::automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. From 196be25fb5a8ccbb660b3f11c85d14d724d7cf59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 11:04:13 -0500 Subject: [PATCH 55/58] voice takedown requests --- _sql/actions/voice_conversion_models.sql | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 8520aa609e..9fd7b4ee20 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -344,6 +344,36 @@ where token IN ( 'vcm_zmxv2k1q9jd8' ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'takedown request' +where token IN ( + 'vcm_3mzcjkf78x7m', + 'vcm_7k2t1wnyqxsc', + 'vcm_w2edpe6q2nya', + 'vcm_vz9cg294xa29', + 'vcm_0z1ys58589k7', + 'vcm_4x8a30h1r26z', + 'vcm_0vf9prxsww60', + 'vcm_yxbadkvqhg16', + 'vcm_p0ezrc6wvzak', + 'vcm_6r5pbtexyvtp', + 'vcm_pb0ewrf7ych5', + 'vcm_7frb2xk0cxk3', + 'vcm_x129c9s2af06', + 'vcm_e0snndfd59b6', + 'vcm_nb8zb0djqwqw', + 'vcm_2erz5hbh8tpf', + 'vcm_e9wn2ccqe3bk', + 'vcm_f8jbxdbjgcjb', + 'vcm_x02h7z3jqxb0', + 'vcm_xy97xcgnk9y6', + 'vcm_a47wtfbsjrfc', + 'vcm_kygenhbg5mgj', + 'vcm_s8gndv19yt7p' +); + From 3766758cfb33ac0cedf93ced058c117130419883 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 13:33:48 -0500 Subject: [PATCH 56/58] Rate limiter support for AI streamers --- .../config/storyteller-web.development.env | 3 + .../src/configs/app_startup/mod.rs | 2 + .../app_startup/redis_rate_limiters.rs | 102 ++++++++++++++++++ .../src/configs/app_startup/username_set.rs | 102 ++++++++++++++++++ .../web/storyteller_web/src/configs/mod.rs | 1 + .../enqueue_infer_tts_handler.rs | 19 ++++ .../service/web/storyteller_web/src/main.rs | 67 +----------- .../web/storyteller_web/src/server_state.rs | 7 ++ 8 files changed, 238 insertions(+), 65 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/mod.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs diff --git a/crates/service/web/storyteller_web/config/storyteller-web.development.env b/crates/service/web/storyteller_web/config/storyteller-web.development.env index 01ffcc593e..c890b5ef3d 100644 --- a/crates/service/web/storyteller_web/config/storyteller-web.development.env +++ b/crates/service/web/storyteller_web/config/storyteller-web.development.env @@ -5,6 +5,9 @@ COOKIE_SECURE=false COOKIE_DOMAIN=.jungle.horse WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse +# Test the rate limiter bypass (obviously these are dummy values) +AI_STREAMER_USERNAMES="PewDiePie,xQc" + # Stripe configs # All secrets are development and okay to check in STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs new file mode 100644 index 0000000000..824be656d0 --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs @@ -0,0 +1,2 @@ +pub mod redis_rate_limiters; +pub mod username_set; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs new file mode 100644 index 0000000000..1dc2b4d5cd --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -0,0 +1,102 @@ +use std::time::Duration; + +use log::info; + +use config::common_env::CommonEnv; +use errors::AnyhowResult; +use limitation::Limiter; +use crate::configs::app_startup::username_set::UsernameSet; + +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::server_state::RedisRateLimiters; + + +/// Build the various rate limiters +pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult { + // TODO(bt,2023-12-13): Kill CommonEnv. + + info!("Setting up Redis rate limiters..."); + + let logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) + }; + + let logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) + }; + + let api_high_priority_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) + }; + + // This is for the following users: + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let api_ai_streamers_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_AI_STREAMERS_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_ai_streamers", limiter_enabled) + }; + + let ai_streamer_usernames = + UsernameSet::from_comma_separated(&easyenv::get_env_string_or_default("AI_STREAMER_USERNAMES", "")); + + info!("AI Streamers that can bypass the normal rate limiter ({}): {:?}", + ai_streamer_usernames.len(), + ai_streamer_usernames.list_names()); + + let model_upload_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) + }; + + Ok(RedisRateLimiters { + logged_out: logged_out_redis_rate_limiter, + logged_in: logged_in_redis_rate_limiter, + api_high_priority: api_high_priority_redis_rate_limiter, + api_ai_streamers: api_ai_streamers_redis_rate_limiter, + api_ai_streamer_username_set: ai_streamer_usernames, + model_upload: model_upload_rate_limiter, + }) +} diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs new file mode 100644 index 0000000000..f9858a201b --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -0,0 +1,102 @@ +use std::collections::{BTreeSet, HashSet}; + +#[derive(Clone)] +pub struct UsernameSet { + /// "usernames" in our service are always lowercase. + /// "display names" are case sensitive. + usernames: HashSet, +} + +impl UsernameSet { + pub fn from_comma_separated(comma_separated_list: &str) -> Self { + let usernames = comma_separated_list.split(",") + .into_iter() + .map(|item| item.trim()) + .map(|item| item.to_lowercase()) + .filter(|item| !item.is_empty()) + .collect::>(); + Self { + usernames + } + } + + pub fn len(&self) -> usize { + self.usernames.len() + } + + pub fn list_names(&self) -> BTreeSet<&str> { + // NB: BTreeSet for sorting, which is good for logging at app startup + self.usernames.iter().map(|item| item.as_str()).collect::<_>() + } + + pub fn username_is_in_set(&self, username: &str) -> bool { + // We lowercase the input username just in case a display name is passed + let username = username.to_lowercase(); + self.usernames.contains(&username) + } +} + +#[cfg(test)] +mod tests { + use crate::configs::app_startup::username_set::UsernameSet; + fn build_set() -> UsernameSet { + UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") + } + #[test] + fn test_deduplication() { + let set = build_set(); + assert_eq!(set.len(), 3); + } + + #[test] + fn test_in_set() { + let set = build_set(); + assert!(set.username_is_in_set("bob")); + assert!(set.username_is_in_set("alice")); + assert!(set.username_is_in_set("john")); + } + + #[test] + fn test_in_set_case_insensitive() { + let set = build_set(); + + assert!(set.username_is_in_set("Bob")); + assert!(set.username_is_in_set("BOB")); + + assert!(set.username_is_in_set("AlIcE")); + assert!(set.username_is_in_set("ALICE")); + + assert!(set.username_is_in_set("jOHN")); + assert!(set.username_is_in_set("JOHN")); + } + + #[test] + fn test_not_in_set() { + let set = build_set(); + + // NB: The build_set() function might have attempted to introduce these, + // but the struct constructor should filter blank spaces out + assert!(!set.username_is_in_set("")); + assert!(!set.username_is_in_set(" ")); + + // Obviously not in set + assert!(!set.username_is_in_set("mary")); + assert!(!set.username_is_in_set("bill")); + + // These usernames are in the set, but not with padding + assert!(!set.username_is_in_set(" alice ")); + assert!(!set.username_is_in_set("bob ")); + assert!(!set.username_is_in_set(" john")); + } + + #[test] + fn test_username_return() { + let set = build_set(); + let usernames = set.list_names(); + assert_eq!(usernames.len(), 3); + assert!(usernames.contains(&"alice")); + assert!(usernames.contains(&"bob")); + assert!(usernames.contains(&"john")); + assert!(!usernames.contains(&"mary")); + } +} diff --git a/crates/service/web/storyteller_web/src/configs/mod.rs b/crates/service/web/storyteller_web/src/configs/mod.rs index 0a735a5b5c..806e1e3209 100644 --- a/crates/service/web/storyteller_web/src/configs/mod.rs +++ b/crates/service/web/storyteller_web/src/configs/mod.rs @@ -1,2 +1,3 @@ +pub mod app_startup; pub mod plans; pub mod static_api_tokens; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index b01dd37a5d..cb417f5906 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -242,7 +242,12 @@ pub async fn enqueue_infer_tts_handler( // ==================== RATE LIMIT ==================== // + // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { @@ -250,6 +255,15 @@ pub async fn enqueue_infer_tts_handler( warn!("User is not authorized to use TTS model because they are banned."); return Err(InferTtsError::NotAuthorized); } + + if server_state + .redis_rate_limiters + .api_ai_streamer_username_set + .username_is_in_set(&user.user.username) + { + is_ai_streamer = true; + } + &server_state.redis_rate_limiters.logged_in }, }; @@ -264,6 +278,11 @@ pub async fn enqueue_infer_tts_handler( rate_limiter = &server_state.redis_rate_limiters.api_high_priority; } + if is_ai_streamer { + info!("Using AI streamer rate limiter"); + rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; + } + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index e77ed85093..57f09a82c8 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -75,6 +75,7 @@ use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStr use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; @@ -173,65 +174,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, ); - info!("Setting up Redis rate limiters..."); - - // Old env vars: - // - // "LIMITER_ENABLED" - // "LIMITER_MAX_REQUESTS" - // "LIMITER_WINDOW_SECONDS" - // - let logged_out_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) - }; - - let logged_in_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) - }; - - let api_high_priority_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) - }; - - let model_upload_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) - }; info!("Connecting to elasticsearch..."); @@ -466,12 +408,7 @@ async fn main() -> AnyhowResult<()> { elasticsearch, redis_pool, redis_ttl_cache, - redis_rate_limiters: RedisRateLimiters { - logged_out: logged_out_redis_rate_limiter, - logged_in: logged_in_redis_rate_limiter, - api_high_priority: api_high_priority_redis_rate_limiter, - model_upload: model_upload_rate_limiter, - }, + redis_rate_limiters: configure_redis_rate_limiters(&common_env)?, firehose_publisher, badge_granter, avt_cookie_manager, diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index 41ddadf1d3..a8f86133e2 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -20,6 +20,7 @@ use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; +use crate::configs::app_startup::username_set::UsernameSet; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo; @@ -140,6 +141,12 @@ pub struct RedisRateLimiters { /// constructions) pub api_high_priority: RedisRateLimiter, + /// API rate limit for AI streamers + pub api_ai_streamers: RedisRateLimiter, + + /// Usernames of AI streamers + pub api_ai_streamer_username_set: UsernameSet, + /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, } From 81dd377599b08f62d0f7a6d483ebb1238d6c512c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 15:27:34 -0500 Subject: [PATCH 57/58] clean up and test rate limiter choice code --- .../src/configs/app_startup/username_set.rs | 1 + .../enqueue_infer_tts_handler.rs | 148 +++++++++++++----- 2 files changed, 114 insertions(+), 35 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs index f9858a201b..0f48cb5350 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -42,6 +42,7 @@ mod tests { fn build_set() -> UsernameSet { UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") } + #[test] fn test_deduplication() { let set = build_set(); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index cb417f5906..2001562038 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -32,6 +32,7 @@ use tts_common::priority::{FAKEYOU_DEFAULT_VALID_API_TOKEN_PRIORITY_LEVEL, FAKEY use user_input_common::check_for_slurs::contains_slurs; use users_component::utils::user_session_extended::UserSessionExtended; +use crate::configs::app_startup::username_set::UsernameSet; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; use crate::configs::plans::plan::Plan; use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; @@ -240,49 +241,44 @@ pub async fn enqueue_infer_tts_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + warn!("User is not authorized to use TTS model because they are banned."); + return Err(InferTtsError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // - // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { - // For AI live streamers - // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 - let mut is_ai_streamer = false; + let maybe_username = maybe_user_session + .as_ref() + .map(|session| session.user.username.as_str()); - let mut rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - warn!("User is not authorized to use TTS model because they are banned."); - return Err(InferTtsError::NotAuthorized); - } + // FIXME(bt,2023-12-13): These boolean flags are a bit dangerous. + let rate_limiter_type = get_rate_limiter_type( + maybe_username, + &server_state.redis_rate_limiters.api_ai_streamer_username_set, + is_investor, + is_from_api, + use_high_priority_rate_limiter + ); - if server_state - .redis_rate_limiters - .api_ai_streamer_username_set - .username_is_in_set(&user.user.username) - { - is_ai_streamer = true; - } - - &server_state.redis_rate_limiters.logged_in + let rate_limiter = match rate_limiter_type { + RateLimiterType::LoggedOut => &server_state.redis_rate_limiters.logged_out, + RateLimiterType::LoggedIn => &server_state.redis_rate_limiters.logged_in, + RateLimiterType::ApiHighPriority => { + info!("Using API high priority rate limiter"); + &server_state.redis_rate_limiters.api_high_priority + }, + RateLimiterType::ApiAiStreamer => { + info!("Using AI streamer rate limiter"); + &server_state.redis_rate_limiters.api_ai_streamers }, }; - // TODO/TEMP - if is_investor || is_from_api { - rate_limiter = &server_state.redis_rate_limiters.logged_in; - } - - // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters - if use_high_priority_rate_limiter { - rate_limiter = &server_state.redis_rate_limiters.api_high_priority; - } - - if is_ai_streamer { - info!("Using AI streamer rate limiter"); - rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; - } - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } @@ -495,6 +491,54 @@ pub async fn enqueue_infer_tts_handler( .body(body)) } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RateLimiterType { + LoggedOut, + LoggedIn, + ApiHighPriority, + ApiAiStreamer, +} + +// FIXME(bt,2023-12-13): The boolean flags and disjoint decision making are kind of terrible. +fn get_rate_limiter_type( + maybe_username: Option<&str>, + ai_streamer_username_set: &UsernameSet, + is_investor: bool, + is_from_api: bool, + use_high_priority_rate_limiter: bool, +) -> RateLimiterType { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + + let mut rate_limiter = match maybe_username { + None => RateLimiterType::LoggedOut, + Some(username) => { + if ai_streamer_username_set.username_is_in_set(username) { + is_ai_streamer = true; + } + + RateLimiterType::LoggedIn + }, + }; + + // TODO/TEMP + if is_investor || is_from_api { + rate_limiter = RateLimiterType::LoggedIn; + } + + // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters + if use_high_priority_rate_limiter { + rate_limiter = RateLimiterType::ApiHighPriority; + } + + if is_ai_streamer { + rate_limiter = RateLimiterType::ApiAiStreamer; + } + + rate_limiter +} + pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> { if text.len() < 3 { return Err("text is too short".to_string()); @@ -561,3 +605,37 @@ fn random_troll_text() -> &'static str { let random = texts.choose(&mut rand::thread_rng()); random.map(|r| *r).unwrap_or(texts[0]) } + + +#[cfg(test)] +mod tests { + + mod rate_limiter_tests { + use crate::configs::app_startup::username_set::UsernameSet; + use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::{get_rate_limiter_type, RateLimiterType}; + + fn build_username_set() -> UsernameSet { + UsernameSet::from_comma_separated("ai_streamer") + } + + #[test] + fn test_logged_out() { + let rate_limiter = get_rate_limiter_type(None, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedOut); + } + + #[test] + fn test_logged_in() { + let username = Some("not_ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedIn); + } + + #[test] + fn test_ai_streamer() { + let username = Some("ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::ApiAiStreamer); + } + } +} From 9ab8e5221190980666a1510fdf699171149bfcba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 16:58:37 -0500 Subject: [PATCH 58/58] more rate limiter fixes --- .../app_startup/redis_rate_limiters.rs | 28 +++++++++++++++++++ .../animation/enqueue_face_animation.rs | 16 +++++++---- .../media_uploads/common/handle_upload.rs | 21 +++++++------- .../enqueue_voice_conversion_inference.rs | 15 ++++++---- .../inference/enqueue_tts_request.rs | 17 ++++++----- .../voice_dataset_samples/upload_zs_sample.rs | 18 +++++++----- .../voice_designer/voices/create_voice.rs | 16 +++++++---- .../w2l/enqueue_infer_w2l_with_uploads.rs | 15 ++++++---- .../web/storyteller_web/src/server_state.rs | 6 ++++ 9 files changed, 103 insertions(+), 49 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs index 1dc2b4d5cd..3aa3df32cf 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -91,6 +91,32 @@ pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult AnyhowResult &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueFaceAnimationError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 2d79cab2e7..7fe2872d84 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -66,26 +66,25 @@ pub async fn handle_upload( UploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(UploadError::RateLimited); } - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - // ==================== READ MULTIPART REQUEST ==================== // let upload_media_request = drain_multipart_request(multipart_payload) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index 22131729c0..7be7cec00d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -205,17 +205,20 @@ pub async fn enqueue_voice_conversion_inference_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _session) => &server_state.redis_rate_limiters.logged_in }; // TODO/TEMP diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 41adb52569..7e09f40782 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,7 +1,6 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] -use utoipa::ToSchema; use std::fmt::Debug; use std::sync::Arc; @@ -12,6 +11,7 @@ use actix_web::http::StatusCode; use log::warn; use serde::Deserialize; use serde::Serialize; +use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; @@ -153,16 +153,19 @@ pub async fn enqueue_tts_request( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueTTSRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueTTSRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index 004d303f44..e89e18eca9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -26,6 +26,7 @@ use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; @@ -111,16 +112,19 @@ pub async fn upload_zs_sample_handler( UploadSampleError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadSampleError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadSampleError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _user) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs index b47de2c732..6654843007 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs @@ -30,6 +30,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -138,16 +139,19 @@ pub async fn create_voice_handler( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueCreateVoiceRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueCreateVoiceRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index eee83b0dcc..8eea038050 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -102,16 +102,19 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_session { + if user.is_banned { + return Err(InferW2lWithUploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(InferW2lWithUploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index a8f86133e2..c7ba601993 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -149,6 +149,12 @@ pub struct RedisRateLimiters { /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_out: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_in: RedisRateLimiter, } /// In-memory caches of several types.