From d9d166e500d95bdb3bc4fd13d63f453676338731 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 28 Nov 2023 17:27:32 -0500 Subject: [PATCH 001/136] Run routes using open api spec --- Cargo.lock | 2 +- .../service/web/storyteller_web/src/routes.rs | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 28ce8804f0..3f9e9f4c93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3792,7 +3792,7 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", "toml", ] diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9b6f920061..489d346d79 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -182,6 +182,9 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> app = add_subscription_routes(app); /* /v1/subscriptions/... */ app = add_voice_designer_routes(app); /* /v1/voice_designer */ + if server_environment == ServerEnvironment::Development { + app = add_weights_routes(app); /* /v1/stubs/... */ + } // ==================== Comments ==================== let mut app = RouteBuilder::from_app(app) @@ -1216,3 +1219,34 @@ fn add_voice_designer_routes (app:App)-> App ) } + +// ==================== Weights ROUTES ==================== +fn add_weights_routes(app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = () + > +{ + app.service( + web + ::scope("/v1/weights") + .route("/upload", web::post().to(upload_weights_handler)) + .service( + web + ::resource("/weight/{weight_token}") + .route(web::get().to(get_weight_details_handler)) + .route(web::post().to(update_weight_metadata_handler)) + .route(web::delete().to(delete_weight_handler)) + ) + .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + .route( + "/by_user_and_type/{username}/{type}", + web::get().to(get_weights_by_user_and_type_handler) + ) + ) +} From 0c456d957c4d26c98b4f904e5abb90e595bf2b50 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 15:46:43 -0500 Subject: [PATCH 002/136] fix build errors --- .../job_loop/determine_dependency_status.rs | 51 ++++++++++++++----- .../src/job/job_loop/main_loop.rs | 3 +- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs index a42aff7b6e..c89ccd8228 100644 --- a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs +++ b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs @@ -109,7 +109,13 @@ fn get_model_token_and_path(job_dependencies: &JobDependencies, maybe_model: &Ma async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { let model = match (job.inference_category, job.maybe_model_token.as_deref()) { (InferenceCategory::TextToSpeech, Some(token)) => { - let maybe_model = job_dependencies.job.info.caches.tts_model_record_cache.copy_without_bump_if_unexpired(token.to_string())?; + let maybe_model = job_dependencies + .job + .info + .caches + .tts_model_record_cache + .copy_without_bump_if_unexpired(token.to_string()) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; match maybe_model { Some(model) => MaybeInferenceModel::TtsModel(model), @@ -117,25 +123,36 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie let maybe_tts_model = get_tts_model_for_inference_improved( &job_dependencies.db.mysql_pool, token) .await - .map_err(|err| - match err { - TtsModelForInferenceError::ModelDeleted => ProcessSingleJobError::ModelDeleted, - _ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)) - })?; + .map_err(|err| match err { + TtsModelForInferenceError::ModelDeleted => ProcessSingleJobError::ModelDeleted, + _ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)) + })?; + match maybe_tts_model { + None => MaybeInferenceModel::None, Some(model) => { let token = token.to_string(); - let _r = job_dependencies.job.info.caches.tts_model_record_cache.store_copy(&token, &model)?; + let _r = job_dependencies + .job + .info + .caches + .tts_model_record_cache + .store_copy(&token, &model) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; MaybeInferenceModel::TtsModel(model) }, - - None => MaybeInferenceModel::None, } } } } (InferenceCategory::VoiceConversion, Some(token)) => { - let maybe_model = job_dependencies.job.info.caches.vc_model_record_cache.copy_without_bump_if_unexpired(token.to_string())?; + let maybe_model = job_dependencies + .job + .info + .caches + .vc_model_record_cache + .copy_without_bump_if_unexpired(token.to_string()) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; match maybe_model { Some(model) => MaybeInferenceModel::VcModel(model), @@ -143,16 +160,22 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie let maybe_vc_model = get_voice_conversion_model_for_inference( &job_dependencies.db.mysql_pool, token) .await - .map_err(|err| anyhow!("database error: {:?}", err))?; + .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; match maybe_vc_model { + None => MaybeInferenceModel::None, Some(model) => { let token = token.to_string(); - let _r = job_dependencies.job.info.caches.vc_model_record_cache.store_copy(&token, &model)?; + let _r = job_dependencies + .job + .info + .caches + .vc_model_record_cache + .store_copy(&token, &model) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + MaybeInferenceModel::VcModel(model) }, - - None => MaybeInferenceModel::None, } } } diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 8add384db2..8da6cc00c7 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -125,7 +125,7 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { - warn!("Failure to process job: {} - {:?}",job.token, err); + warn!("Failure to process job: {:?} - {:?}",job.inference_job_token, err); let _r = handle_error(&job_dependencies, &job, err).await?; } } @@ -268,6 +268,7 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere ProcessSingleJobError::NotYetImplemented => {} ProcessSingleJobError::FaceDetectionFailure => {} ProcessSingleJobError::JobSystemMisconfiguration(_) => {} + ProcessSingleJobError::ModelDeleted => {} } Ok(()) From 65d6f178bca8e00287072915c05844a2c2afb2ab Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 17:07:18 -0500 Subject: [PATCH 003/136] WIP - weights endpoints need to be tested Update weights endpoints had to be fixed. should be ready to unit test the 3 endpoints before using the list weights --- .../src/queries/model_weights/get_weight.rs | 4 +- .../queries/model_weights/update_weight.rs | 72 +++----- .../src/http_server/endpoints/mod.rs | 3 +- .../endpoints/weights/create_weight.rs | 0 .../endpoints/weights/delete_weight.rs | 166 +++++++++++++++++ .../endpoints/weights/get_weight.rs | 156 ++++++++++++++++ .../endpoints/weights/list_weights_by_user.rs | 0 .../src/http_server/endpoints/weights/mod.rs | 3 + .../endpoints/weights/update_weight.rs | 169 ++++++++++++++++++ .../service/web/storyteller_web/src/routes.rs | 20 ++- 10 files changed, 535 insertions(+), 58 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs index 1b29dfceed..050f4de251 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs @@ -157,7 +157,7 @@ async fn select_include_deleted( "#, weight_token.as_str() ) - .fetch_one(mysql_connection).await + .fetch_one(&mut **mysql_connection).await } async fn select_without_deleted( @@ -207,7 +207,7 @@ async fn select_without_deleted( "#, weight_token.as_str() ) - .fetch_one(mysql_connection).await + .fetch_one(&mut **mysql_connection).await } // RawWeight is the struct that is returned from the database in raw form. diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index ba84497b52..21430a06e5 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -1,56 +1,51 @@ use anyhow::anyhow; use sqlx::MySqlPool; -use log::info; -use sqlx::mysql::MySqlPoolOptions; - use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::model_weights::ModelWeightToken; -use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; - pub struct UpdateWeightArgs<'a> { pub weight_token: &'a ModelWeightToken, pub title: Option<&'a str>, pub maybe_thumbnail_token: Option<&'a str>, - pub description_markdown: &'a str, - pub description_rendered_html: &'a str, - pub creator_set_visibility: &'a Visibility, + pub description_markdown: Option<&'a str>, + pub description_rendered_html: Option<&'a str>, + pub creator_set_visibility: Option<&'a Visibility>, pub mysql_pool: &'a MySqlPool, } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { let mut transaction = args.mysql_pool.begin().await?; - let query_result = sqlx - ::query!( - r#" - UPDATE model_weights - SET - title = ?, - description_markdown = ?, - maybe_thumbnail_token = ?, - description_rendered_html = ?, - creator_set_visibility = ?, - version = version + 1 - WHERE token = ? - LIMIT 1 - "#, - args.title, - args.description_markdown, - args.maybe_thumbnail_token, - args.description_rendered_html, - args.creator_set_visibility.to_str(), - args.weight_token.as_str() - ) - .execute(args.mysql_pool).await; + let query_result = sqlx::query!( + r#" + UPDATE model_weights + SET + title = COALESCE(?, title), + description_markdown = COALESCE(?, description_markdown), + maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token), + description_rendered_html = COALESCE(?, description_rendered_html), + creator_set_visibility = COALESCE(?, creator_set_visibility), + version = version + 1 + WHERE token = ? + LIMIT 1 + "#, + args.title.as_deref().unwrap_or(Some("")), + args.description_markdown.as_deref().unwrap_or(Some("")), + args.maybe_thumbnail_token.as_deref().unwrap_or(Some("")), + args.description_rendered_html.as_deref().unwrap_or(Some("")), + args.creator_set_visibility.as_deref().unwrap_or(Some("")), + args.weight_token.as_str() + ) + .execute(args.mysql_pool).await; transaction.commit().await?; match query_result { Ok(_) => Ok(()), Err(err) => { + transaction.rollback().await?; Err(anyhow!("weights update error: {:?}", err)) } } @@ -59,21 +54,4 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { #[cfg(test)] mod tests { - use tokio; - // Template - use sqlx::mysql::MySqlPoolOptions; - - use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING}; - use errors::AnyhowResult; - #[tokio::test] - async fn test_update_weights() -> AnyhowResult<()> { - let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; - - let pool = MySqlPoolOptions::new() - .max_connections(3) - .connect(&db_connection_string) - .await?; - - Ok(()) - } } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 81af5ee4b2..d4c32ebd1d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -25,4 +25,5 @@ pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; pub mod w2l; -pub mod favorites; \ No newline at end of file +pub mod favorites; +pub mod weights; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs new file mode 100644 index 0000000000..91dfcbe94c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -0,0 +1,166 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, log, warn}; +use enums::by_table::audit_logs::audit_log_entity_action::AuditLogEntityAction::Delete; + +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::delete_weights::{delete_weights_as_user,delete_weights_as_mod,undelete_weights_as_user,undelete_weights_as_mod}; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; + +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + + +#[derive(Deserialize)] +pub struct DeleteWeightRequest { + set_delete: bool, + as_mod: bool, + weight_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DeleteWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteWeightError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteWeightError::NotFound => StatusCode::NOT_FOUND, + DeleteWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn delete_weight_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> + ) -> Result{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(DeleteWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to delete this weight: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + + let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(path.as_mod)); + + let query_result = if path.set_delete { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_weights_as_mod( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + } else { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to undelete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update weight mod approval status DB error: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + Ok(simple_json_success()) + } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs new file mode 100644 index 0000000000..9ca0ed207e --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -0,0 +1,156 @@ +use std::fmt; +use std::sync::Arc; +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +use tokens::tokens::model_weights::ModelWeightToken; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use enums::by_table::model_weights::weights_types::WeightsType; +use enums::by_table::model_weights::weights_category::WeightsCategory; + +#[derive(Serialize, Clone)] +pub struct GetWeightResponse { + success: bool, + weight_token: ModelWeightToken, + title: String, + weights_type: WeightsType, + weights_category: WeightsCategory, + maybe_thumbnail_token: Option, + description_markdown: String, + description_rendered_html: String, + creator_set_visibility: Visibility, + + file_size_bytes: i32, + file_checksum_sha2: String, + + cached_user_ratings_negative_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_total_count: u32, + + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + version: i32, + created_at: DateTime, +} + +#[derive(Deserialize)] +pub struct GetWeightPathInfo { + weight_token: String, +} + +#[derive(Debug)] +pub enum GetWeightError { + NotAuthorized, + NotFound, + ServerError, +} + +impl fmt::Display for GetWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for GetWeightError { + fn status_code(&self) -> StatusCode { + match *self { + GetWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + GetWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetWeightError::NotFound => StatusCode::NOT_FOUND, + } + } +} + +pub async fn get_weight_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(GetWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(GetWeightError::ServerError); + } + }; + + // if the weight is private, only the creator can view it + let is_private = weight.creator_set_visibility == Visibility::Private; + + if is_private && creator_user_token.as_str() != &user_session.user_token { + warn!("user is not allowed to view this weight: {}", user_session.user_token); + return Err(GetWeightError::NotAuthorized); + } + + + let response = GetWeightResponse { + success: true, + weight_token: weight.token, + title: weight.title, + weights_type: weight.weights_type, + weights_category: weight.weights_category, + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + creator_set_visibility: weight.creator_set_visibility, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, + version: weight.version, + created_at: weight.created_at + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetWeightError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs new file mode 100644 index 0000000000..afc73c47a8 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -0,0 +1,3 @@ +pub mod get_weight; +pub mod delete_weight; +pub mod update_weight; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs new file mode 100644 index 0000000000..5b6c301f53 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -0,0 +1,169 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, warn}; + +use enums::common::visibility::Visibility; +use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; +use mysql_queries::queries::model_weights::update_weight::{update_weights, UpdateWeightArgs}; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::server_state::ServerState; + +/// TODO will eventually be polymorphic +#[derive(Deserialize)] +pub struct UpdateWeightRequest { + pub weight_token: String, + pub title: Option, + pub description_markdown: Option, + pub description_rendered_html: Option, + pub weight_type: Option, + pub weight_category: Option, +} + +#[derive(Serialize)] +pub struct UpdateWeightResponse { + pub success: bool, +} + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct UpdateWeightPathInfo { + weight_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum UpdateWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for UpdateWeightError { + fn status_code(&self) -> StatusCode { + match *self { + UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateWeightError::NotFound => StatusCode::NOT_FOUND, + UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UpdateWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn update_weight_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result + { + let my_sql_pool = &server_state.mysql_pool; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UpdateWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + + // TODO wouldn't we want to instead use a function that will query the DB for the user and determine if they are a mod? + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(UpdateWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to edit this weight: {}", user_session.user_token); + return Err(UpdateWeightError::NotAuthorized); + } + + let mut weight_type = None; + let mut weight_category = None; + + if let Some(wt) = request.weight_type.as_deref() { + weight_type = Some(wt.to_string()); + } + + if let Some(wc) = request.weight_category.as_deref() { + weight_category = Some(wc.to_string()); + } + + let mut maybe_mod_user_token = None; + + if is_mod { + maybe_mod_user_token = Some(user_session.user_token.clone()); + } + + let query_result = update_weights( + UpdateWeightArgs { + weight_token: &ModelWeightToken::new(weight_token.clone()), + mysql_pool: &server_state.mysql_pool, + title:request.title, + maybe_thumbnail_token: request.thumbnail_token, + description_markdown: request.description_markdown, + description_rendered_html: request.description_rendered_html, + creator_set_visibility: request.creator_set_visibility, + mysql_pool:my_sql_pool, + } + ).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update Weight DB error: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + }; + + Ok(simple_json_success()) + } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 489d346d79..db590ab3da 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -149,6 +149,10 @@ use crate::http_server::endpoints::w2l::list_user_w2l_templates::list_user_w2l_t use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::weights::get_weight::get_weight_handler; +use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; +use crate::http_server::endpoints::weights::update_weight::update_weight_handler; + pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where B: MessageBody, @@ -1235,18 +1239,18 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - .route("/upload", web::post().to(upload_weights_handler)) + // .route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") - .route(web::get().to(get_weight_details_handler)) - .route(web::post().to(update_weight_metadata_handler)) + .route(web::get().to(get_weight_handler)) + .route(web::post().to(update_weight_handler)) .route(web::delete().to(delete_weight_handler)) ) - .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) - .route( - "/by_user_and_type/{username}/{type}", - web::get().to(get_weights_by_user_and_type_handler) - ) + // .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + // .route( + // "/by_user_and_type/{username}/{type}", + // web::get().to(get_weights_by_user_and_type_handler) + // ) ) } From 1a3e111dd6967933452f1b1c6edf369e490d02b7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 17:30:03 -0500 Subject: [PATCH 004/136] Fix job pathing for outage and to make easier to manage --- .../src/filesystem/check_directory_exists.rs | 14 ++-- .../src/job/job_loop/process_single_job.rs | 19 ++--- .../lipsync/sad_talker/process_job.rs | 2 +- .../download_static_dependencies.rs | 6 +- .../tacotron2_v2_early_fakeyou/process_job.rs | 4 +- .../tts/vall_e_x/process_job_create_voice.rs | 2 +- .../vall_e_x/process_job_inference_voice.rs | 2 +- .../src/job/job_types/tts/vits/process_job.rs | 2 +- .../job/job_types/vc/rvc_v2/process_job.rs | 8 +-- .../job_types/vc/so_vits_svc/process_job.rs | 2 +- .../job/inference_job/src/job_dependencies.rs | 24 ++++--- crates/service/job/inference_job/src/main.rs | 71 ++++++++++++------- .../src/util/scoped_temp_dir_creator.rs | 6 +- 13 files changed, 97 insertions(+), 65 deletions(-) diff --git a/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs b/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs index 43487de769..8e770b171f 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs +++ b/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs @@ -1,15 +1,17 @@ -use std::path::PathBuf; +use std::path::Path; use anyhow::bail; use crate::anyhow_result::AnyhowResult; -pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { - if !path.exists() { - bail!("Path doesn't exist: {:?}", path); +pub fn check_directory_exists>(path: P) -> AnyhowResult<()> { + let check_path = path.as_ref(); + + if !check_path.exists() { + bail!("Path doesn't exist: {:?}", check_path); } - if !path.is_dir() { - bail!("Path isn't a directory: {:?}", path); + if !check_path.is_dir() { + bail!("Path isn't a directory: {:?}", check_path); } Ok(()) } diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 401fdd76dd..4309db6e60 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -160,18 +160,19 @@ async fn do_process_single_job( } } - // ==================== SETUP TEMP DIRS ==================== // - - let temp_dir = format!("temp_{}", job.id.0); - let temp_dir = job_dependencies.fs.scoped_temp_dir_creator_for_downloads.new_tempdir(&temp_dir) - .map_err(|err| ProcessSingleJobError::Other(anyhow!("filesystem error: {:?}", err)))?; - - let _p = temp_dir.path(); // TODO: Just so the build doesn't complain about unused. Remove. +// NB(bt,2023-11-29): This looks dead, but hopefully isn't leaned on for downstream side effects. +// I'll leave this commented out while dealing with the outage, but this can probably be removed +// in a short while after we redeploy and verify all the jobs. +// // ==================== SETUP TEMP DIRS ==================== // +// +// let temp_dir = format!("temp_{}", job.id.0); +// let temp_dir = job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads.new_tempdir(&temp_dir) +// .map_err(|err| ProcessSingleJobError::Other(anyhow!("filesystem error: {:?}", err)))?; +// +// let _p = temp_dir.path(); // TODO: Just so the build doesn't complain about unused. Remove. // ==================== HANDLE DIFFERENT INFERENCE TYPES ==================== // - let mut maybe_result_entity : Option; - let job_success_result = match job.inference_category { InferenceCategory::LipsyncAnimation => { process_single_lipsync_job(job_dependencies, job).await? diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 4c424abeec..009fb63b28 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -74,7 +74,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result( &mut job_progress_reporter, "downloading vocoder (1 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, ).await?; waveglow_vocoder_model_fs_path @@ -61,7 +61,7 @@ pub async fn download_static_dependencies<'a>( &mut job_progress_reporter, "downloading vocoder (2 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, ).await?; hifigan_vocoder_model_fs_path @@ -83,7 +83,7 @@ pub async fn download_static_dependencies<'a>( &mut job_progress_reporter, "downloading vocoder (3 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, ).await?; hifigan_superres_vocoder_model_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 8428d099c1..177791f92c 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -127,7 +127,7 @@ async fn process_job_with_cleanup( &mut job_progress_reporter, "downloading user vocoder", job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, ).await?; Some(custom_vocoder_fs_path) @@ -148,7 +148,7 @@ async fn process_job_with_cleanup( &mut job_progress_reporter, "downloading synthesizer", job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, ).await?; tts_synthesizer_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index dbc9980393..e91582ac01 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -177,7 +177,7 @@ pub async fn process_create_voice( for downloader in model_dependencies.downloaders.all_downloaders() { let result = downloader.download_if_not_on_filesystem( &args.job_dependencies.buckets.private_bucket_client, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads + &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads ).await; if let Err(e) = result { error!("could not download: {:?}", e); diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index cfbbb2852d..a9aabac937 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -82,7 +82,7 @@ pub async fn process_inference_voice( for downloader in model_dependencies.downloaders.all_downloaders() { let result = downloader.download_if_not_on_filesystem( &args.job_dependencies.buckets.private_bucket_client, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads + &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads ).await; if let Err(e) = result { diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index efad6075e4..d42cff9da9 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -78,7 +78,7 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, - // TODO: Rename + /// Directory to store long-term downloads (models) + /// (In prod, typically model files from GCS / NFS PVC mount) + pub directory_long_lived_downloads: PathBuf, + + /// Temporary directory for storing downloads /// Creates temp directories for scratch files - pub scoped_temp_dir_creator_for_downloads: ScopedTempDirCreator, + pub scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator, + + /// Temporary directory for storing downloads + /// (In prod, typically model files from GCS / NFS PVC mount) + pub scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator, + + /// Temporary directory for storing work + /// (In prod, typically python inference outputs / emptyDir mount) pub scoped_temp_dir_creator_for_work: ScopedTempDirCreator, - // TODO: Rename /// Organizes downloaded files pub semi_persistent_cache: SemiPersistentCacheDir, } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index d6b7e66565..d7dc736356 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -124,31 +124,16 @@ async fn main() -> AnyhowResult<()> { PathBuf::from("/tmp/downloads") ); - // Where we store scratch files for workloads (temporary processing). - let temp_directory_work = easyenv::get_env_pathbuf_or_default( - "TEMP_DIR_WORK", - PathBuf::from("/tmp/work") - ); + // //// Where we store scratch files for workloads (temporary processing). + // let temp_directory_work = easyenv::get_env_pathbuf_or_default( + // "TEMP_DIR_WORK", + // PathBuf::from("/tmp/work") + // ); - if !container_environment.server_environment.is_deployed_in_production() { - warn!("Creating temporary directories for non-production / development only!"); - create_dir_all_if_missing(&temp_directory_downloads)?; - create_dir_all_if_missing(&temp_directory_work)?; - } - - check_directory_exists(&temp_directory_downloads)?; - check_directory_exists(&temp_directory_work)?; let semi_persistent_cache = SemiPersistentCacheDir::configured_root(&temp_directory_downloads); - // TODO(bt,2023.05.22): create_all_paths() for some subset of jobs. - info!("Creating pod semi-persistent cache dirs..."); - semi_persistent_cache.create_custom_vocoder_model_path()?; - semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?; - semi_persistent_cache.create_tts_synthesizer_model_path()?; - semi_persistent_cache.create_voice_conversion_model_path()?; - let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -242,11 +227,18 @@ async fn main() -> AnyhowResult<()> { maybe_keepalive_redis_pool, }, fs: FileSystemDetails { - temp_directory_downloads: temp_directory_downloads.clone(), - temp_directory_work: temp_directory_work.clone(), maybe_pause_file: easyenv::get_env_pathbuf_optional("PAUSE_FILE"), - scoped_temp_dir_creator_for_downloads: ScopedTempDirCreator::for_directory(&temp_directory_downloads), - scoped_temp_dir_creator_for_work: ScopedTempDirCreator::for_directory(&temp_directory_work), + directory_long_lived_downloads: easyenv::get_env_pathbuf_or_default( + "DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")), + scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory( + easyenv::get_env_pathbuf_or_default( + "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))), + scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator::for_directory( + easyenv::get_env_pathbuf_or_default( + "SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))), + scoped_temp_dir_creator_for_work: ScopedTempDirCreator::for_directory( + easyenv::get_env_pathbuf_or_default( + "SCOPED_TEMP_DIR_WORK", PathBuf::from("/tmp/downloads_long_lived"))), semi_persistent_cache, }, buckets: BucketDependencies { @@ -306,6 +298,8 @@ async fn main() -> AnyhowResult<()> { }, }; + set_up_directories(&job_dependencies)?; + std::thread::spawn(move || { let actix_runtime = actix_web::rt::System::new(); let http_server_handle = launch_http_server(create_server_args); @@ -322,3 +316,32 @@ async fn main() -> AnyhowResult<()> { Ok(()) } +fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> { + info!("Setting up and checking file system paths for temporary work, ephemeral and persistent downloads, etc..."); + + let fs = &job_dependencies.fs; + + if !job_dependencies.job.info.container.server_environment.is_deployed_in_production() { + warn!("Creating directories for non-production / development only!"); + + create_dir_all_if_missing(&fs.directory_long_lived_downloads)?; + create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; + create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; + create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_work.get_base_dir())?; + } + + check_directory_exists(&fs.directory_long_lived_downloads)?; + check_directory_exists(fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; + check_directory_exists(fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; + check_directory_exists(fs.scoped_temp_dir_creator_for_work.get_base_dir())?; + + // TODO(bt,2023.05.22): create_all_paths() for some subset of jobs. + info!("Creating semi-persistent cache dirs..."); + + fs.semi_persistent_cache.create_custom_vocoder_model_path()?; + fs.semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?; + fs.semi_persistent_cache.create_tts_synthesizer_model_path()?; + fs.semi_persistent_cache.create_voice_conversion_model_path()?; + + Ok(()) +} diff --git a/crates/service/job/inference_job/src/util/scoped_temp_dir_creator.rs b/crates/service/job/inference_job/src/util/scoped_temp_dir_creator.rs index 0c5dec1a03..e6e0dc4077 100644 --- a/crates/service/job/inference_job/src/util/scoped_temp_dir_creator.rs +++ b/crates/service/job/inference_job/src/util/scoped_temp_dir_creator.rs @@ -15,7 +15,11 @@ impl ScopedTempDirCreator { } } + pub fn get_base_dir(&self) -> &Path { + &self.base_dir + } + pub fn new_tempdir(&self, name: &str) -> std::io::Result { TempDir::new_in(&self.base_dir, name) } -} \ No newline at end of file +} From 37c016cecbd8a990b8c165d61ddac0a43b673191 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 17:48:29 -0500 Subject: [PATCH 005/136] "semi persistent dir" needs cleanup, this is confusing --- .../download_static_dependencies.rs | 6 +++--- crates/service/job/inference_job/src/main.rs | 16 +++------------- .../src/util/maybe_download_file_from_bucket.rs | 12 ++++++------ 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs index 10a529a897..f0b3d2a0cd 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs @@ -39,7 +39,7 @@ pub async fn download_static_dependencies<'a>( &mut job_progress_reporter, "downloading vocoder (1 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, ).await?; waveglow_vocoder_model_fs_path @@ -61,7 +61,7 @@ pub async fn download_static_dependencies<'a>( &mut job_progress_reporter, "downloading vocoder (2 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, ).await?; hifigan_vocoder_model_fs_path @@ -83,7 +83,7 @@ pub async fn download_static_dependencies<'a>( &mut job_progress_reporter, "downloading vocoder (3 of 3)", job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, ).await?; hifigan_superres_vocoder_model_fs_path diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index d7dc736356..3cd39e5e90 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -119,20 +119,10 @@ async fn main() -> AnyhowResult<()> { )?; // Where we download models and resources to (typically a shared NFS volume in prod). - let temp_directory_downloads = easyenv::get_env_pathbuf_or_default( - "TEMP_DIR_DOWNLOADS", - PathBuf::from("/tmp/downloads") - ); - // //// Where we store scratch files for workloads (temporary processing). - // let temp_directory_work = easyenv::get_env_pathbuf_or_default( - // "TEMP_DIR_WORK", - // PathBuf::from("/tmp/work") - // ); - - - let semi_persistent_cache = - SemiPersistentCacheDir::configured_root(&temp_directory_downloads); + let semi_persistent_cache = SemiPersistentCacheDir::configured_root( + easyenv::get_env_pathbuf_or_default("SEMI_PERSISTENT_DIR_ROOT", + PathBuf::from("/tmp/persistent"))); let db_connection_string = easyenv::get_env_string_or_default( diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index bd294c51cb..98a49512ca 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -13,7 +13,7 @@ use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; pub async fn maybe_download_file_from_bucket( name_or_description_of_file: &str, - file_path: &Path, + final_filesystem_file_path: &Path, bucket_object_path: &Path, bucket_client: &BucketClient, job_progress_reporter: &mut Box, @@ -22,12 +22,12 @@ pub async fn maybe_download_file_from_bucket( scoped_tempdir_creator: &ScopedTempDirCreator, ) -> Result<(), ProcessSingleJobError> { - if file_path.exists() { + if final_filesystem_file_path.exists() { // TODO(bt, 2022-07-15): Check signature of file return Ok(()) } - warn!("{} does not exist at path: {:?}", name_or_description_of_file, &file_path); + warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); job_progress_reporter.log_status(job_progress_update_description) .map_err(|e| ProcessSingleJobError::Other(e))?; @@ -54,15 +54,15 @@ pub async fn maybe_download_file_from_bucket( info!("Downloaded {} from bucket!", name_or_description_of_file); info!("Renaming {} temp file from {:?} to {:?}!", - name_or_description_of_file, &temp_path, &file_path); + name_or_description_of_file, &temp_path, &final_filesystem_file_path); - std::fs::rename(&temp_path, &file_path) + std::fs::rename(&temp_path, &final_filesystem_file_path) .map_err(|e| { safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::from_io_error(e) })?; - info!("Finished downloading {} file to {:?}", name_or_description_of_file, &file_path); + info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path); safe_delete_temp_directory(&temp_dir); From f48cbb7978aee70cc5c86bf4d0a58d471569131f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 18:16:45 -0500 Subject: [PATCH 006/136] fix --- crates/service/job/inference_job/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 3cd39e5e90..43a7ab5569 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -222,7 +222,7 @@ async fn main() -> AnyhowResult<()> { "DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")), scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory( easyenv::get_env_pathbuf_or_default( - "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))), + "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))), scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator::for_directory( easyenv::get_env_pathbuf_or_default( "SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))), From f2e75f863712464cb735fb4be660eec0c1dc6bdb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 18:46:00 -0500 Subject: [PATCH 007/136] rename files across fs devices --- .../service/job/inference_job/src/job_dependencies.rs | 6 ++---- crates/service/job/inference_job/src/main.rs | 4 ---- .../src/util/maybe_download_file_from_bucket.rs | 10 ++++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 7b99bad23a..448554e1ee 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -144,10 +144,6 @@ pub struct FileSystemDetails { /// redeploying. pub maybe_pause_file: Option, - /// Directory to store long-term downloads (models) - /// (In prod, typically model files from GCS / NFS PVC mount) - pub directory_long_lived_downloads: PathBuf, - /// Temporary directory for storing downloads /// Creates temp directories for scratch files pub scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator, @@ -161,6 +157,8 @@ pub struct FileSystemDetails { pub scoped_temp_dir_creator_for_work: ScopedTempDirCreator, /// Organizes downloaded files + /// Directory to store long-term downloads (models) + /// (In prod, typically model files from GCS / NFS PVC mount) pub semi_persistent_cache: SemiPersistentCacheDir, } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 43a7ab5569..ceff31645a 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -218,8 +218,6 @@ async fn main() -> AnyhowResult<()> { }, fs: FileSystemDetails { maybe_pause_file: easyenv::get_env_pathbuf_optional("PAUSE_FILE"), - directory_long_lived_downloads: easyenv::get_env_pathbuf_or_default( - "DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")), scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory( easyenv::get_env_pathbuf_or_default( "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))), @@ -314,13 +312,11 @@ fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> { if !job_dependencies.job.info.container.server_environment.is_deployed_in_production() { warn!("Creating directories for non-production / development only!"); - create_dir_all_if_missing(&fs.directory_long_lived_downloads)?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_work.get_base_dir())?; } - check_directory_exists(&fs.directory_long_lived_downloads)?; check_directory_exists(fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; check_directory_exists(fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; check_directory_exists(fs.scoped_temp_dir_creator_for_work.get_base_dir())?; diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 98a49512ca..60626e7985 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -1,9 +1,10 @@ use std::path::Path; -use log::{info, warn}; +use log::{error, info, warn}; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::rename_across_devices::rename_across_devices; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -56,10 +57,11 @@ pub async fn maybe_download_file_from_bucket( info!("Renaming {} temp file from {:?} to {:?}!", name_or_description_of_file, &temp_path, &final_filesystem_file_path); - std::fs::rename(&temp_path, &final_filesystem_file_path) - .map_err(|e| { + rename_across_devices(&temp_path, &final_filesystem_file_path) + .map_err(|err| { + error!("could not rename on disk: {:?}", err); safe_delete_temp_directory(&temp_dir); - ProcessSingleJobError::from_io_error(e) + ProcessSingleJobError::from_io_error(err) })?; info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path); From 363f8cbc96a5695af7cc6bc77be955158dcd9203 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 19:28:32 -0500 Subject: [PATCH 008/136] debug log statements, FileCacheDir (not yet used) --- .../tacotron2_v2_early_fakeyou/process_job.rs | 10 +++- .../inference_job/src/util/file_cache_dir.rs | 55 +++++++++++++++++++ .../service/job/inference_job/src/util/mod.rs | 1 + 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 crates/service/job/inference_job/src/util/file_cache_dir.rs diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 177791f92c..7c088abd9d 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -58,7 +58,7 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result>(env_var_name: &str, default_path: P) -> AnyhowResult { + Ok(Self { + directory: easyenv::get_env_pathbuf_or_default(env_var_name, default_path), + }) + } + + pub fn create_dir_all_if_missing(&self) -> std::io::Result<()> { + create_dir_all_if_missing(&self.directory) + } + + pub fn err_unless_exists(&self) -> AnyhowResult<()> { + check_directory_exists(&self.directory)?; + Ok(()) + } + + pub fn as_path(&self) -> &Path { + &self.directory + } + + /// Return the number of files in the directory (non-recursive). + /// If there is a concern for a long-running list operation, a `max_count` can be returned. + pub fn file_count(&self, maybe_max_count: Option) -> io::Result { + let mut count = 0; + // NB: Two for loops to prevent if-checks in optimized loop case. + if let Some(max_count) = maybe_max_count { + for entry in std::fs::read_dir(&self.directory)? { + let _entry = entry?; + if max_count > count { + return Ok(max_count); + } + count += 1; + } + } else { + for entry in std::fs::read_dir(&self.directory)? { + let _entry = entry?; + count += 1; + } + } + Ok(count) + } +} diff --git a/crates/service/job/inference_job/src/util/mod.rs b/crates/service/job/inference_job/src/util/mod.rs index 1ad41d8ada..b586a0a792 100644 --- a/crates/service/job/inference_job/src/util/mod.rs +++ b/crates/service/job/inference_job/src/util/mod.rs @@ -1,4 +1,5 @@ pub mod common_commands; +pub mod file_cache_dir; pub mod from_env; pub mod maybe_download_file_from_bucket; #[macro_use] From 542b79dd9082838e06f52292ed2e0ee438620fe2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 19:49:52 -0500 Subject: [PATCH 009/136] log http errors --- .../tacotron2_inference_sidecar_client.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs index 613a47bfb0..64ae25df90 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs @@ -160,7 +160,16 @@ impl Tacotron2InferenceSidecarClient { let client = Client::new(); - let _maybe_response = client.request(req).await?; + let response = client.request(req).await?; + let status = response.status(); + + info!("Response status: {} ({:?})", status, status.canonical_reason()); + + if status.is_server_error() { + return Err(anyhow!("sidecar server error: {} {:?}", status, status.canonical_reason())); + } else if status.is_client_error() { + return Err(anyhow!("sidecar client request error: {} {:?}", status, status.canonical_reason())); + } Ok(()) } From 9e0772141ad2581235f576c1909c26d6903efe07 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 20:25:37 -0500 Subject: [PATCH 010/136] download file size check and logging --- crates/lib/files/filesys/src/file_size.rs | 1 + .../lipsync/sad_talker/download_audio_file.rs | 1 + .../lipsync/sad_talker/download_image_file.rs | 1 + .../download_static_dependencies.rs | 4 ++- .../tacotron2_v2_early_fakeyou/process_job.rs | 2 ++ .../src/job/job_types/tts/vits/process_job.rs | 1 + .../job/job_types/vc/rvc_v2/process_job.rs | 3 ++ .../job_types/vc/so_vits_svc/process_job.rs | 2 ++ .../util/maybe_download_file_from_bucket.rs | 36 ++++++++++++++++--- 9 files changed, 46 insertions(+), 5 deletions(-) diff --git a/crates/lib/files/filesys/src/file_size.rs b/crates/lib/files/filesys/src/file_size.rs index 89dcd79416..c64723f256 100644 --- a/crates/lib/files/filesys/src/file_size.rs +++ b/crates/lib/files/filesys/src/file_size.rs @@ -7,6 +7,7 @@ use errors::{anyhow, AnyhowResult}; pub fn file_size>(filename: P) -> AnyhowResult { let metadata = std::fs::metadata(filename.as_ref())?; + // TODO(bt,2023-11-29): Return io::Result and errors here. if !metadata.is_file() { Err(anyhow!("path {:?} is not a file", filename.as_ref())) } else if metadata.is_symlink() { diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 6c0cb689a8..e810dc46d4 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -103,6 +103,7 @@ pub async fn download_audio_file( "downloading", job.id.0, &temp_dir_creator, + None, ).await?; Ok(AudioFile { diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 506580bfe0..3ad2e15ab9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -74,6 +74,7 @@ pub async fn download_image_file( "downloading", job.id.0, &temp_dir_creator, + None, ).await?; Ok(ImageFile { diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs index f0b3d2a0cd..60a43fe7a6 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs @@ -22,7 +22,6 @@ pub async fn download_static_dependencies<'a>( mut job_progress_reporter: &mut Box, ) -> Result { - // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== // let waveglow_vocoder_model_fs_path = { @@ -40,6 +39,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (1 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; waveglow_vocoder_model_fs_path @@ -62,6 +62,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (2 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; hifigan_vocoder_model_fs_path @@ -84,6 +85,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (3 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; hifigan_superres_vocoder_model_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 7c088abd9d..f5fbbf6457 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -128,6 +128,7 @@ async fn process_job_with_cleanup( "downloading user vocoder", job.id.0, &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + None, ).await?; Some(custom_vocoder_fs_path) @@ -149,6 +150,7 @@ async fn process_job_with_cleanup( "downloading synthesizer", job.id.0, &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + None, ).await?; tts_synthesizer_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index d42cff9da9..5789d3a56b 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -79,6 +79,7 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, ) -> Result<(), ProcessSingleJobError> { if final_filesystem_file_path.exists() { - // TODO(bt, 2022-07-15): Check signature of file - return Ok(()) - } + // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity + let mut existing_file_is_valid = true; - warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required { + // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file + // size, but the Rust system may report a non-zero (but small) number of bytes. This should + // later be investigated and the heuristic simplified. The intent here is merely to make sure + // we don't consider these okay: + // + // -rw-r--r-- 1 root root 55824433 Nov 30 00:12 VM:012rkwsv91zb + // -rw-r--r-- 1 root root 55823149 Nov 30 00:13 VM:79etbx4fdksv + // -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc + // -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb + // + let size = file_size(final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}", + &final_filesystem_file_path); + + if size < existing_file_minimum_size_required { + existing_file_is_valid = false; + } + } + + if existing_file_is_valid { + return Ok(()) + } + } else { + warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + } job_progress_reporter.log_status(job_progress_update_description) .map_err(|e| ProcessSingleJobError::Other(e))?; From 003f0cdeeabdd23c4373e8f342639c5b0d27ded7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 20:46:33 -0500 Subject: [PATCH 011/136] struct args for downloader (safer) --- .../filesys/src/rename_across_devices.rs | 1 + .../lipsync/sad_talker/download_audio_file.rs | 22 +++--- .../lipsync/sad_talker/download_image_file.rs | 22 +++--- .../download_static_dependencies.rs | 68 +++++++++---------- .../tacotron2_v2_early_fakeyou/process_job.rs | 46 ++++++------- .../src/job/job_types/tts/vits/process_job.rs | 24 +++---- .../job/job_types/vc/rvc_v2/process_job.rs | 68 +++++++++---------- .../job_types/vc/so_vits_svc/process_job.rs | 46 ++++++------- .../util/maybe_download_file_from_bucket.rs | 54 ++++++++------- 9 files changed, 178 insertions(+), 173 deletions(-) diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs index f990bc671f..91a5ae9e67 100644 --- a/crates/lib/files/filesys/src/rename_across_devices.rs +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -3,6 +3,7 @@ use std::path::Path; /// Rename a file. /// Ordinary rename will fail in Linux if it is across physical devices. /// This function will perform a copy followed by delete in that case. +/// In both cases, this function will overwrite the destination if it already exists. pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> std::io::Result<()> { let result = std::fs::rename(&from, &to); diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index e810dc46d4..19c0d958c9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -16,7 +16,7 @@ use mysql_queries::queries::voice_conversion::results::get_voice_conversion_resu use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; pub struct AudioFile { @@ -94,17 +94,17 @@ pub async fn download_audio_file( let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin"); - maybe_download_file_from_bucket( - "audio file", - &downloaded_filesystem_path, - &bucket_object_path, - public_bucket_client, + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "audio file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, job_progress_reporter, - "downloading", - job.id.0, - &temp_dir_creator, - None, - ).await?; + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; Ok(AudioFile { filesystem_path: downloaded_filesystem_path, diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 3ad2e15ab9..6ac3b4d3e9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -14,7 +14,7 @@ use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_m use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; pub struct ImageFile { @@ -65,17 +65,17 @@ pub async fn download_image_file( let downloaded_filesystem_path = work_temp_dir.path().join("image.bin"); - maybe_download_file_from_bucket( - "image file", - &downloaded_filesystem_path, - &bucket_object_path, - public_bucket_client, + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "image file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, job_progress_reporter, - "downloading", - job.id.0, - &temp_dir_creator, - None, - ).await?; + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; Ok(ImageFile { filesystem_path: downloaded_filesystem_path, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs index 60a43fe7a6..536c25d260 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs @@ -6,7 +6,7 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; pub struct StaticDependencies { pub waveglow_vocoder_model_fs_path : PathBuf, @@ -30,17 +30,17 @@ pub async fn download_static_dependencies<'a>( let waveglow_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename); let waveglow_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename); - maybe_download_file_from_bucket( - "waveglow vocoder model", - &waveglow_vocoder_model_fs_path, - &waveglow_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (1 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "waveglow vocoder model", + final_filesystem_file_path: &waveglow_vocoder_model_fs_path, + bucket_object_path: &waveglow_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (1 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; waveglow_vocoder_model_fs_path }; @@ -53,17 +53,17 @@ pub async fn download_static_dependencies<'a>( let hifigan_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename); let hifigan_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename); - maybe_download_file_from_bucket( - "hifigan vocoder model", - &hifigan_vocoder_model_fs_path, - &hifigan_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (2 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "hifigan vocoder model", + final_filesystem_file_path: &hifigan_vocoder_model_fs_path, + bucket_object_path: &hifigan_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (2 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; hifigan_vocoder_model_fs_path }; @@ -76,17 +76,17 @@ pub async fn download_static_dependencies<'a>( let hifigan_superres_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename); let hifigan_superres_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename); - maybe_download_file_from_bucket( - "hifigan superres vocoder model", - &hifigan_superres_vocoder_model_fs_path, - &hifigan_superres_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (3 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "hifigan superres vocoder model", + final_filesystem_file_path: &hifigan_superres_vocoder_model_fs_path, + bucket_object_path: &hifigan_superres_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (3 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; hifigan_superres_vocoder_model_fs_path }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index f5fbbf6457..17e1e4a594 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -29,7 +29,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_s use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor}; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// Text starting with this will be treated as a test request. /// This allows the request to bypass the model cache and query the latest TTS model. @@ -119,17 +119,17 @@ async fn process_job_with_cleanup( let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token); let custom_vocoder_object_path = args.job_dependencies.buckets.bucket_path_unifier.vocoder_path(&vocoder.vocoder_private_bucket_hash); - maybe_download_file_from_bucket( - "custom vocoder", - &custom_vocoder_fs_path, - &custom_vocoder_object_path, - &args.job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading user vocoder", - job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - None, - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "custom vocoder", + final_filesystem_file_path: &custom_vocoder_fs_path, + bucket_object_path: &custom_vocoder_object_path, + bucket_client: &args.job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading user vocoder", + job_id: job.id.0, + scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + maybe_existing_file_minimum_size_required: None, + }).await?; Some(custom_vocoder_fs_path) } @@ -141,17 +141,17 @@ async fn process_job_with_cleanup( let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str()); let tts_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash); - maybe_download_file_from_bucket( - "synthesizer", - &tts_synthesizer_fs_path, - &tts_synthesizer_object_path, - &args.job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading synthesizer", - job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - None, - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "synthesizer", + final_filesystem_file_path: & tts_synthesizer_fs_path, + bucket_object_path: &tts_synthesizer_object_path, + bucket_client: &args.job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading synthesizer", + job_id: job.id.0, + scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + maybe_existing_file_minimum_size_required: None, + }).await?; tts_synthesizer_fs_path }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index 5789d3a56b..15ee5a53f8 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -22,7 +22,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::vits::vits_inference_command::{Device, VitsInferenceArgs}; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// Text starting with this will be treated as a test request. /// This allows the request to bypass the model cache and query the latest TTS model. @@ -70,17 +70,17 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result { + pub name_or_description_of_file: &'a str, + pub final_filesystem_file_path: &'a Path, + pub bucket_object_path: &'a Path, + pub bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job_progress_update_description: &'a str, + pub job_id: i64, + pub scoped_tempdir_creator: &'a ScopedTempDirCreator, + pub maybe_existing_file_minimum_size_required: Option, +} + pub async fn maybe_download_file_from_bucket( - name_or_description_of_file: &str, - final_filesystem_file_path: &Path, - bucket_object_path: &Path, - bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job_progress_update_description: &str, - job_id: i64, - scoped_tempdir_creator: &ScopedTempDirCreator, - maybe_existing_file_minimum_size_required: Option, + args: MaybeDownloadArgs<'_> ) -> Result<(), ProcessSingleJobError> { - if final_filesystem_file_path.exists() { + if args.final_filesystem_file_path.exists() { // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity let mut existing_file_is_valid = true; - if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required { + if let Some(existing_file_minimum_size_required) = args.maybe_existing_file_minimum_size_required { // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file // size, but the Rust system may report a non-zero (but small) number of bytes. This should // later be investigated and the heuristic simplified. The intent here is merely to make sure @@ -40,11 +44,11 @@ pub async fn maybe_download_file_from_bucket( // -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc // -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb // - let size = file_size(final_filesystem_file_path) + let size = file_size(args.final_filesystem_file_path) .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}", - &final_filesystem_file_path); + info!("{} exists at path {:?} ; file size = {size}", + &args.name_or_description_of_file,&args.final_filesystem_file_path); if size < existing_file_minimum_size_required { existing_file_is_valid = false; @@ -55,46 +59,46 @@ pub async fn maybe_download_file_from_bucket( return Ok(()) } } else { - warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + warn!("{} does not exist at path: {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); } - job_progress_reporter.log_status(job_progress_update_description) + args.job_progress_reporter.log_status(args.job_progress_update_description) .map_err(|e| ProcessSingleJobError::Other(e))?; // NB: Download to temp directory to stop concurrent writes and race conditions from other // workers writing to a shared volume. - let temp_dir = format!("temp_download_{}", job_id); + let temp_dir = format!("temp_download_{}", args.job_id); // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem. - let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir) + let temp_dir = args.scoped_tempdir_creator.new_tempdir(&temp_dir) .map_err(|e| ProcessSingleJobError::from_io_error(e))?; let temp_path = temp_dir.path().join("download.part"); - info!("Downloading {} from bucket path: {:?}", name_or_description_of_file, &bucket_object_path); + info!("Downloading {} from bucket path: {:?}", args.name_or_description_of_file, &args.bucket_object_path); - bucket_client.download_file_to_disk(&bucket_object_path, &temp_path) + args.bucket_client.download_file_to_disk(&args.bucket_object_path, &temp_path) .await .map_err(|e| { safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::Other(e) })?; - info!("Downloaded {} from bucket!", name_or_description_of_file); + info!("Downloaded {} from bucket!", args.name_or_description_of_file); info!("Renaming {} temp file from {:?} to {:?}!", - name_or_description_of_file, &temp_path, &final_filesystem_file_path); + args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - rename_across_devices(&temp_path, &final_filesystem_file_path) + rename_across_devices(&temp_path, &args.final_filesystem_file_path) .map_err(|err| { error!("could not rename on disk: {:?}", err); safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::from_io_error(err) })?; - info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path); + info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); safe_delete_temp_directory(&temp_dir); Ok(()) -} \ No newline at end of file +} From a549ceb609113d3b69d78e51c82d5806a0f4b3e1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 20:53:00 -0500 Subject: [PATCH 012/136] check vocoder and synthesizer sizes --- .../job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 17e1e4a594..8c3d143f80 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -128,7 +128,7 @@ async fn process_job_with_cleanup( job_progress_update_description: "downloading user vocoder", job_id: job.id.0, scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - maybe_existing_file_minimum_size_required: None, + maybe_existing_file_minimum_size_required: Some(1000), }).await?; Some(custom_vocoder_fs_path) @@ -150,7 +150,7 @@ async fn process_job_with_cleanup( job_progress_update_description: "downloading synthesizer", job_id: job.id.0, scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - maybe_existing_file_minimum_size_required: None, + maybe_existing_file_minimum_size_required: Some(1000), }).await?; tts_synthesizer_fs_path From 4fb2a38c68d64df8638c4e054dab923ef884c74d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 21:10:01 -0500 Subject: [PATCH 013/136] report temp file size --- .../src/util/maybe_download_file_from_bucket.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 44fb823aa8..3ca960d4f7 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -86,6 +86,12 @@ pub async fn maybe_download_file_from_bucket( info!("Downloaded {} from bucket!", args.name_or_description_of_file); + let size = file_size(&temp_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("File size of {} temp download file {:?} is {size}", + args.name_or_description_of_file, &temp_path); + info!("Renaming {} temp file from {:?} to {:?}!", args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); From ee4af2976f521f11d0b099492fa33994a3169896 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 21:26:03 -0500 Subject: [PATCH 014/136] flag to permit cold filesystem cache --- .../inference_job/src/job/job_loop/process_single_job.rs | 8 ++++---- crates/service/job/inference_job/src/job_dependencies.rs | 4 ++++ crates/service/job/inference_job/src/main.rs | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 4309db6e60..39b3b895b4 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -40,14 +40,14 @@ pub async fn process_single_job( } } - // TODO(bt,2023-07-23): Re-review the following. It looks sus. - // TODO(bt,2023-07-23): Re-review the following. It looks sus. - // TODO(bt,2023-07-23): Re-review the following. It looks sus. // TODO(bt,2023-07-23): Re-review the following. It looks sus. let dependency_status = determine_dependency_status(job_dependencies, job) .await?; - if !force_execution && !dependency_status.models_already_on_filesystem { + if !force_execution + && !job_dependencies.job.system.always_allow_cold_filesystem_cache + && !dependency_status.models_already_on_filesystem + { match dependency_status.maybe_model_token { None => {} // No model token, proceed Some(model_token) => { diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 448554e1ee..504d2fb08d 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -60,6 +60,10 @@ pub struct JobSystemControls { /// This is provided at job start. pub scoped_execution: ScopedExecution, + // Allow for models not to exist on the filesystem. All jobs will execute when first tried + // regardless of whether their models were previously downloaded. + pub always_allow_cold_filesystem_cache: bool, + // How many times to skip jobs (on cold filesystem cache) before proceeding with execution. pub cold_filesystem_cache_starvation_threshold: u64, diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index ceff31645a..f0ac58a93f 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -243,6 +243,7 @@ async fn main() -> AnyhowResult<()> { job: JobSystemDependencies { system: JobSystemControls { scoped_execution, + always_allow_cold_filesystem_cache: easyenv::get_env_bool_or_default("ALWAYS_ALLOW_COLD_FILESYSTEM_CACHE", false), cold_filesystem_cache_starvation_threshold: easyenv::get_env_num("COLD_FILESYSTEM_CACHE_STARVATION_THRESHOLD", 3)?, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as u16, From 3243cc51b7344893250f5b5c3c6d40dd50239908 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 21:36:47 -0500 Subject: [PATCH 015/136] try to delete and re-copy --- .../util/maybe_download_file_from_bucket.rs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 3ca960d4f7..624faa2636 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -86,10 +86,10 @@ pub async fn maybe_download_file_from_bucket( info!("Downloaded {} from bucket!", args.name_or_description_of_file); - let size = file_size(&temp_path) + let original_size = file_size(&temp_path) .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - info!("File size of {} temp download file {:?} is {size}", + info!("File size of {} temp download file {:?} is {original_size}", args.name_or_description_of_file, &temp_path); info!("Renaming {} temp file from {:?} to {:?}!", @@ -104,6 +104,33 @@ pub async fn maybe_download_file_from_bucket( info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); + // NB: We're now seeing a bug where the resultant copied file is 0 bytes + let copied_size = file_size(&args.final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("Final copied size of {} file {:?} is {copied_size} (original size was {original_size})", + args.name_or_description_of_file, &args.final_filesystem_file_path); + + if copied_size == 0 { + error!("Copied size was 0! Removing and retrying..."); + + std::fs::remove_file(&args.final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_io_error(err))?; + + rename_across_devices(&temp_path, &args.final_filesystem_file_path) + .map_err(|err| { + error!("could not rename on disk: {:?}", err); + safe_delete_temp_directory(&temp_dir); + ProcessSingleJobError::from_io_error(err) + })?; + + let copied_size = file_size(&args.final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", + args.name_or_description_of_file, &args.final_filesystem_file_path); + } + safe_delete_temp_directory(&temp_dir); Ok(()) From c207356ac5b31f0c7c70b833b41b375d831ff9fb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 21:44:20 -0500 Subject: [PATCH 016/136] cleanup retry --- .../util/maybe_download_file_from_bucket.rs | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 624faa2636..b9f9c8039b 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -1,9 +1,11 @@ use std::path::Path; use log::{error, info, warn}; +use tempdir::TempDir; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use errors::AnyhowResult; use filesys::file_size::file_size; use filesys::rename_across_devices::rename_across_devices; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; @@ -105,33 +107,41 @@ pub async fn maybe_download_file_from_bucket( info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); // NB: We're now seeing a bug where the resultant copied file is 0 bytes + reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?; + + safe_delete_temp_directory(&temp_dir); + + Ok(()) +} + +fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_path: &Path, original_size: u64) -> Result<(), ProcessSingleJobError> { let copied_size = file_size(&args.final_filesystem_file_path) .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; info!("Final copied size of {} file {:?} is {copied_size} (original size was {original_size})", args.name_or_description_of_file, &args.final_filesystem_file_path); - if copied_size == 0 { - error!("Copied size was 0! Removing and retrying..."); - - std::fs::remove_file(&args.final_filesystem_file_path) - .map_err(|err| ProcessSingleJobError::from_io_error(err))?; - - rename_across_devices(&temp_path, &args.final_filesystem_file_path) - .map_err(|err| { - error!("could not rename on disk: {:?}", err); - safe_delete_temp_directory(&temp_dir); - ProcessSingleJobError::from_io_error(err) - })?; - - let copied_size = file_size(&args.final_filesystem_file_path) - .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - - info!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", - args.name_or_description_of_file, &args.final_filesystem_file_path); + if copied_size != 0 { + return Ok(()); } - safe_delete_temp_directory(&temp_dir); + error!("Copied size was 0! Removing destination file and retrying..."); + + std::fs::remove_file(&args.final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_io_error(err))?; + + rename_across_devices(&temp_path, &args.final_filesystem_file_path) + .map_err(|err| { + error!("could not rename on disk: {:?}", err); + safe_delete_temp_directory(&temp_dir); + ProcessSingleJobError::from_io_error(err) + })?; + + let copied_size = file_size(&args.final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", + args.name_or_description_of_file, &args.final_filesystem_file_path); Ok(()) } From 32331f9890e8370284072e4226c90ce93edf8773 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 21:46:42 -0500 Subject: [PATCH 017/136] compiles working on list queries --- .../queries/model_weights/update_weight.rs | 19 +- .../endpoints/weights/update_weight.rs | 181 +++++++++--------- 2 files changed, 102 insertions(+), 98 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 21430a06e5..2b1d4a9c0f 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -12,12 +12,19 @@ pub struct UpdateWeightArgs<'a> { pub description_markdown: Option<&'a str>, pub description_rendered_html: Option<&'a str>, pub creator_set_visibility: Option<&'a Visibility>, + pub weight_type: Option, + pub weight_category: Option, pub mysql_pool: &'a MySqlPool, } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { let mut transaction = args.mysql_pool.begin().await?; + let visbility: &str = match args.creator_set_visibility { + Some(visibility) => visibility.to_str(), + None => "", + }; + let query_result = sqlx::query!( r#" UPDATE model_weights @@ -29,13 +36,12 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { creator_set_visibility = COALESCE(?, creator_set_visibility), version = version + 1 WHERE token = ? - LIMIT 1 "#, - args.title.as_deref().unwrap_or(Some("")), - args.description_markdown.as_deref().unwrap_or(Some("")), - args.maybe_thumbnail_token.as_deref().unwrap_or(Some("")), - args.description_rendered_html.as_deref().unwrap_or(Some("")), - args.creator_set_visibility.as_deref().unwrap_or(Some("")), + args.title.as_deref().unwrap_or(""), + args.description_markdown.as_deref().unwrap_or(""), + args.maybe_thumbnail_token.as_deref().unwrap_or(""), + args.description_rendered_html.as_deref().unwrap_or(""), + visbility, args.weight_token.as_str() ) .execute(args.mysql_pool).await; @@ -45,7 +51,6 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { match query_result { Ok(_) => Ok(()), Err(err) => { - transaction.rollback().await?; Err(anyhow!("weights update error: {:?}", err)) } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 5b6c301f53..4ab26cab50 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -1,73 +1,75 @@ use std::fmt; use std::sync::Arc; -use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::{ HttpRequest, HttpResponse, web }; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; -use log::{error, warn}; +use log::{ error, warn }; use enums::common::visibility::Visibility; use http_server_common::response::response_success_helpers::simple_json_success; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; -use mysql_queries::queries::model_weights::update_weight::{update_weights, UpdateWeightArgs}; +use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; -/// TODO will eventually be polymorphic +/// TODO will eventually be polymorphic #[derive(Deserialize)] pub struct UpdateWeightRequest { - pub weight_token: String, - pub title: Option, - pub description_markdown: Option, - pub description_rendered_html: Option, - pub weight_type: Option, - pub weight_category: Option, + pub weight_token: String, + pub title: Option, + pub thumbnail_token: Option, + pub description_markdown: Option, + pub description_rendered_html: Option, + pub weight_type: Option, + pub weight_category: Option, + pub visibility: Option, } #[derive(Serialize)] pub struct UpdateWeightResponse { - pub success: bool, + pub success: bool, } /// For the URL PathInfo #[derive(Deserialize)] pub struct UpdateWeightPathInfo { - weight_token: String, + weight_token: String, } // =============== Error Response =============== #[derive(Debug, Serialize)] pub enum UpdateWeightError { - BadInput(String), - NotFound, - NotAuthorized, - ServerError, + BadInput(String), + NotFound, + NotAuthorized, + ServerError, } impl ResponseError for UpdateWeightError { - fn status_code(&self) -> StatusCode { - match *self { - UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, - UpdateWeightError::NotFound => StatusCode::NOT_FOUND, - UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + fn status_code(&self) -> StatusCode { + match *self { + UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateWeightError::NotFound => StatusCode::NOT_FOUND, + UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } } - } - fn error_response(&self) -> HttpResponse { - serialize_as_json_error(self) - } + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } } // NB: Not using derive_more::Display since Clion doesn't understand it. impl fmt::Display for UpdateWeightError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } } // =============== Handler =============== @@ -76,94 +78,91 @@ pub async fn update_weight_handler( http_request: HttpRequest, path: Path, request: web::Json, - server_state: web::Data>) -> Result - { + server_state: web::Data> +) -> Result { let my_sql_pool = &server_state.mysql_pool; - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool) - .await + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { - warn!("Session checker error: {:?}", e); - UpdateWeightError::ServerError + warn!("Session checker error: {:?}", e); + UpdateWeightError::ServerError })?; - + let user_session = match maybe_user_session { - Some(session) => session, - None => { - warn!("not logged in"); - return Err(UpdateWeightError::NotAuthorized); - } + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateWeightError::NotAuthorized); + } }; - + let weight_token = path.weight_token.clone(); // TODO wouldn't we want to instead use a function that will query the DB for the user and determine if they are a mod? let is_mod = user_session.can_ban_users; - + let weight_lookup_result = get_weight_by_token( - &ModelWeightToken::new(weight_token.clone()), - is_mod, - &server_state.mysql_pool, + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool ).await; - + let weight = match weight_lookup_result { - Ok(Some(weight)) => weight, - Ok(None) => { - warn!("Weight not found: {:?}", weight_token); - return Err(UpdateWeightError::NotFound); - }, - Err(err) => { - warn!("Error looking up weight: {:?}", err); - return Err(UpdateWeightError::ServerError); - } + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(UpdateWeightError::NotFound); + } + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(UpdateWeightError::ServerError); + } }; - + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; - + if !is_creator && !is_mod { - warn!("user is not allowed to edit this weight: {}", user_session.user_token); - return Err(UpdateWeightError::NotAuthorized); + warn!("user is not allowed to edit this weight: {}", user_session.user_token); + return Err(UpdateWeightError::NotAuthorized); } - + let mut weight_type = None; let mut weight_category = None; - + if let Some(wt) = request.weight_type.as_deref() { - weight_type = Some(wt.to_string()); + weight_type = Some(wt.to_string()); } - + if let Some(wc) = request.weight_category.as_deref() { - weight_category = Some(wc.to_string()); + weight_category = Some(wc.to_string()); } - + let mut maybe_mod_user_token = None; - + if is_mod { - maybe_mod_user_token = Some(user_session.user_token.clone()); + maybe_mod_user_token = Some(user_session.user_token.clone()); } - - let query_result = update_weights( - UpdateWeightArgs { + + let query_result = update_weights(UpdateWeightArgs { weight_token: &ModelWeightToken::new(weight_token.clone()), - mysql_pool: &server_state.mysql_pool, - title:request.title, - maybe_thumbnail_token: request.thumbnail_token, - description_markdown: request.description_markdown, - description_rendered_html: request.description_rendered_html, - creator_set_visibility: request.creator_set_visibility, - mysql_pool:my_sql_pool, - } - ).await; - + mysql_pool: &server_state.mysql_pool, + title: request.title.as_deref(), + maybe_thumbnail_token: request.thumbnail_token.as_deref(), + description_markdown: request.description_markdown.as_deref(), + description_rendered_html: request.description_rendered_html.as_deref(), + creator_set_visibility: request.visibility.as_ref(), + weight_type: request.weight_type.as_deref().map(|s| s.to_string()), + weight_category: request.weight_category.as_deref().map(|s| s.to_string()), + }).await; + match query_result { - Ok(_) => {}, - Err(err) => { - warn!("Update Weight DB error: {:?}", err); - return Err(UpdateWeightError::ServerError); - } - }; - + Ok(_) => {} + Err(err) => { + warn!("Update Weight DB error: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + } + Ok(simple_json_success()) - } \ No newline at end of file +} From efdd03bbbd8ab29f1ccddd5f276d1539dcd86277 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 21:50:30 -0500 Subject: [PATCH 018/136] logging --- .../inference_job/src/util/maybe_download_file_from_bucket.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index b9f9c8039b..071dc43715 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -130,6 +130,8 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p std::fs::remove_file(&args.final_filesystem_file_path) .map_err(|err| ProcessSingleJobError::from_io_error(err))?; + warn!("Removed. Retrying copy..."); + rename_across_devices(&temp_path, &args.final_filesystem_file_path) .map_err(|err| { error!("could not rename on disk: {:?}", err); @@ -140,7 +142,7 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p let copied_size = file_size(&args.final_filesystem_file_path) .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - info!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", + warn!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", args.name_or_description_of_file, &args.final_filesystem_file_path); Ok(()) From 6cc3062da87766b7b54cbcb8c4b3886492e72812 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 22:01:53 -0500 Subject: [PATCH 019/136] don't fail if file is gone (not sure why this happens) --- .../util/maybe_download_file_from_bucket.rs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 071dc43715..4640844b60 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -1,3 +1,4 @@ +use std::io::ErrorKind; use std::path::Path; use log::{error, info, warn}; @@ -127,10 +128,23 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p error!("Copied size was 0! Removing destination file and retrying..."); - std::fs::remove_file(&args.final_filesystem_file_path) - .map_err(|err| ProcessSingleJobError::from_io_error(err))?; + if let Err(err) = std::fs::remove_file(&args.final_filesystem_file_path) { + match err.kind() { + ErrorKind::NotFound => { + // NB: We seem to be seeing this in production. + // Perhaps another pod deleted it in a race condition? + // Fall through case. + warn!("File couldn't be removed: it's already gone.") + }, + _ => { + return Err(ProcessSingleJobError::from_io_error(err)); + } + } + } else { + warn!("File removed."); + } - warn!("Removed. Retrying copy..."); + warn!("Retrying copy..."); rename_across_devices(&temp_path, &args.final_filesystem_file_path) .map_err(|err| { From cec7bce4506accc5af072a995876346f6fbe53ea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 22:49:00 -0500 Subject: [PATCH 020/136] try copy instead --- .../filesys/src/rename_across_devices.rs | 4 ++ .../util/maybe_download_file_from_bucket.rs | 58 +++++++++++++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs index 91a5ae9e67..60a4046976 100644 --- a/crates/lib/files/filesys/src/rename_across_devices.rs +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -24,6 +24,10 @@ pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> } } + // FIXME(bt,2023-11-29): There appears to be a new bug where the std::fs::copy succeeds in + // copying zero bytes (perhaps the file was truncated to zero bytes before in std::fs::rename?), + // and we delete the source file (below). This only appears to happen in GCP currently, and this + // code has worked for months prior to this. let _num_bytes = std::fs::copy(&from, &to)?; std::fs::remove_file(&from)?; diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 4640844b60..7c3285e70e 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -95,21 +95,40 @@ pub async fn maybe_download_file_from_bucket( info!("File size of {} temp download file {:?} is {original_size}", args.name_or_description_of_file, &temp_path); - info!("Renaming {} temp file from {:?} to {:?}!", + //info!("Renaming {} temp file from {:?} to {:?}!", + // args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); + + //rename_across_devices(&temp_path, &args.final_filesystem_file_path) + // .map_err(|err| { + // error!("could not rename on disk: {:?}", err); + // safe_delete_temp_directory(&temp_dir); + // ProcessSingleJobError::from_io_error(err) + // })?; + + info!("Copying {} temp file from {:?} to {:?}!", args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - rename_across_devices(&temp_path, &args.final_filesystem_file_path) - .map_err(|err| { - error!("could not rename on disk: {:?}", err); - safe_delete_temp_directory(&temp_dir); - ProcessSingleJobError::from_io_error(err) - })?; + // NB: We're now seeing a bug where the resultant copied file is 0 bytes + match copy_with_logging(&temp_path, &args.final_filesystem_file_path) { + Err(err) => { + error!("Error Copying {} temp file from {:?} to {:?}! {err}", + args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); + + return Err(ProcessSingleJobError::from_anyhow_error(err)); + } + Ok(false) => { + error!("Error copying {} temp file from {:?} to {:?}! File size in bytes did not match.", + args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); + + reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?; + } + Ok(true) => { + // Success case + } + } info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); - // NB: We're now seeing a bug where the resultant copied file is 0 bytes - reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?; - safe_delete_temp_directory(&temp_dir); Ok(()) @@ -161,3 +180,22 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p Ok(()) } + + +pub fn copy_with_logging, Q: AsRef>(from: P, to: Q) -> AnyhowResult { + let original_size = file_size(&from)?; + + let num_bytes_copied = std::fs::copy(&from, &to)?; + + let destination_size = file_size(&to)?; + + info!(r#"Copied {:?} to {:?} + - Copied bytes: {num_bytes_copied} + - Orig. bytes: {original_size} + - Dest. bytes: {destination_size} + "#, from.as_ref(), to.as_ref()); + + let file_sizes_match = original_size == destination_size; + + Ok(file_sizes_match) +} From 19dcfdc6296a18ad184e72de3367799071ca78f5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 22:51:12 -0500 Subject: [PATCH 021/136] cleanup --- .../src/util/maybe_download_file_from_bucket.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 7c3285e70e..ef653dd30e 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -6,6 +6,7 @@ use tempdir::TempDir; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use errors::AnyhowResult; use filesys::file_size::file_size; use filesys::rename_across_devices::rename_across_devices; @@ -114,6 +115,9 @@ pub async fn maybe_download_file_from_bucket( error!("Error Copying {} temp file from {:?} to {:?}! {err}", args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); + safe_delete_temp_file(&temp_path); + safe_delete_temp_directory(&temp_dir); + return Err(ProcessSingleJobError::from_anyhow_error(err)); } Ok(false) => { @@ -129,6 +133,7 @@ pub async fn maybe_download_file_from_bucket( info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); + safe_delete_temp_file(&temp_path); safe_delete_temp_directory(&temp_dir); Ok(()) @@ -156,6 +161,8 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p warn!("File couldn't be removed: it's already gone.") }, _ => { + safe_delete_temp_file(&temp_path); + safe_delete_temp_directory(&temp_dir); return Err(ProcessSingleJobError::from_io_error(err)); } } @@ -168,6 +175,7 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p rename_across_devices(&temp_path, &args.final_filesystem_file_path) .map_err(|err| { error!("could not rename on disk: {:?}", err); + safe_delete_temp_file(&temp_path); safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::from_io_error(err) })?; From 896e8591ba6e8e049277eb8e99a46f50a66dd1b1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 23:10:15 -0500 Subject: [PATCH 022/136] cleanup --- .../inference_job/src/util/maybe_download_file_from_bucket.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index ef653dd30e..16bc900d13 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -147,6 +147,9 @@ fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_p args.name_or_description_of_file, &args.final_filesystem_file_path); if copied_size != 0 { + safe_delete_temp_file(&temp_path); + safe_delete_temp_directory(&temp_dir); + return Ok(()); } From 347fe685cd31abb363c2d7bbbf85b8505a53a604 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 29 Nov 2023 23:11:47 -0500 Subject: [PATCH 023/136] List list_availible_weights needs testing but is written. --- .../weights/list_availible_weights.rs | 244 ++++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 4 +- 2 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs new file mode 100644 index 0000000000..3e134413d1 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -0,0 +1,244 @@ +use std::sync::Arc; + +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{ DateTime, Utc }; +use log::{ info, warn }; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; +use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct ListAvailibleWeightsQuery { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, +} + +#[derive(Serialize)] +pub struct ListAvailibleWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + pub title: String, + + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub creator_user_token: UserToken, + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + + pub maybe_last_update_user_token: Option, + + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub version: i32, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, +} + +#[derive(Debug)] +pub enum ListWeightError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListWeightError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn list_availible_weights_handler( + http_request: HttpRequest, + query: web::Query, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightError::ServerError + })?; + + let mut is_mod = false; + let user_session = match maybe_user_session { + Some(session) => { + is_mod = session.can_ban_users; + session + } + None => { + info!("not logged in"); + return Err(ListWeightError::NotAuthorized); + } + }; + + let limit = query.limit.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let include_user_hidden = is_mod; + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + + + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; + + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + + // generate parse a response + let response = ListAvailibleWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + + weights_type: WeightsType, + weights_category: WeightsCategory, + + title: String, + + maybe_thumbnail_token: Option, + + description_markdown: String, + description_rendered_html: String, + + creator_user_token: UserToken, + creator_ip_address: String, + creator_set_visibility: Visibility, + + maybe_last_update_user_token: Option, + + original_download_url: Option, + original_filename: Option, + + file_size_bytes: i32, + file_checksum_sha2: String, + + private_bucket_hash: String, + + maybe_private_bucket_prefix: Option, + maybe_private_bucket_extension: Option, + + cached_user_ratings_total_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_negative_count: u32, + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + version: i32, + + created_at: DateTime, + updated_at: DateTime, + + user_deleted_at: Option>, + mod_deleted_at: Option>, + + creator_username: String, + creator_display_name: String, + creator_email_gravatar_hash: String, + }).collect::>(), + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightError::ServerError)?; + + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index afc73c47a8..66601e374d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -1,3 +1,5 @@ pub mod get_weight; pub mod delete_weight; -pub mod update_weight; \ No newline at end of file +pub mod update_weight; +pub mod list_weights_by_user; +pub mod list_availible_weights; \ No newline at end of file From fba7d2a1ebaabc1b7a1d52cc9969e084319e4af1 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 00:48:54 -0500 Subject: [PATCH 024/136] needs some tests for users and query builds with pagination --- .../weights/list_availible_weights.rs | 73 ++++------ .../endpoints/weights/list_weights_by_user.rs | 136 ++++++++++++++++++ 2 files changed, 164 insertions(+), 45 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 3e134413d1..fa64de0d58 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -8,7 +8,12 @@ use log::{ info, warn }; use enums::common::visibility::Visibility; use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; -use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; + +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; +use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; @@ -33,7 +38,6 @@ pub struct ListAvailibleWeightsSuccessResponse { #[derive(Serialize)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, - pub title: String, pub weights_type: WeightsType, @@ -47,11 +51,8 @@ pub struct ModelWeightForList { pub description_rendered_html: String, pub creator_user_token: UserToken, - pub creator_ip_address: String, pub creator_set_visibility: Visibility, - pub maybe_last_update_user_token: Option, - pub file_size_bytes: i32, pub file_checksum_sha2: String, @@ -59,7 +60,6 @@ pub struct ModelWeightForList { pub cached_user_ratings_positive_count: u32, pub cached_user_ratings_negative_count: u32, pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, pub version: i32, @@ -184,53 +184,36 @@ pub async fn list_availible_weights_handler( success: true, weights: weights_page.weights.into_iter() .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - - weights_type: WeightsType, - weights_category: WeightsCategory, + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, - title: String, + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, - maybe_thumbnail_token: Option, + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, - description_markdown: String, - description_rendered_html: String, + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, - creator_user_token: UserToken, - creator_ip_address: String, - creator_set_visibility: Visibility, + - maybe_last_update_user_token: Option, + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - original_download_url: Option, - original_filename: Option, + version: weights.version, - file_size_bytes: i32, - file_checksum_sha2: String, + created_at: weights.created_at, + updated_at: weights.updated_at, - private_bucket_hash: String, - - maybe_private_bucket_prefix: Option, - maybe_private_bucket_extension: Option, - - cached_user_ratings_total_count: u32, - cached_user_ratings_positive_count: u32, - cached_user_ratings_negative_count: u32, - maybe_cached_user_ratings_ratio: Option, - cached_user_ratings_last_updated_at: DateTime, - - version: i32, - - created_at: DateTime, - updated_at: DateTime, - - user_deleted_at: Option>, - mod_deleted_at: Option>, - - creator_username: String, - creator_display_name: String, - creator_email_gravatar_hash: String, + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash }).collect::>(), cursor_next, cursor_previous, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index e69de29bb2..21ac39c7a8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -0,0 +1,136 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, Clone)] +pub struct Weight { + weight_token: ModelWeightToken, + title: String, + + creator: UserDetailsLight, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} + + +#[derive(Serialize)] +pub struct ListWeightsByUserSuccessResponse { + pub success: bool, + pub weights: Vec, +} + +#[derive(Deserialize)] +pub struct ListWeightsByUserPathInfo { + username: String, +} + +#[derive(Debug)] +pub enum ListWeightsByUserError { + NotAuthorized, + ServerError, +} + +impl fmt::Display for ListWeightsByUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightsByUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightsByUserError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightsByUserError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn list_weights_by_user_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightsByUserError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListWeightsByUserError::NotAuthorized); + } + }; + + let username = path.username.as_ref(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let query_results = list_weights_by_creator_username( + &server_state.mysql_pool, + &username, + is_mod, + ).await.map_err(|e| { + warn!("Error querying for weights: {:?}", e); + ListWeightsByUserError::ServerError + }); + let weights = match query_results { + Ok(weights) => weights, + Err(e) => { + warn!("Error querying for weights: {:?}", e); + + + return Err(ListWeightsByUserError::ServerError); + } + }; + + let weights = weights.into_iter().map(|weight| { + Weight { + weight_token: weight.token, + title: weight.title, + creator: UserDetailsLight::from_db_fields( + &weight.creator_user_token, + &weight.creator_username, + &weight.creator_display_name, + &weight.creator_email_gravatar_hash, + ), + creator_set_visibility: weight.creator_set_visibility, + created_at: weight.created_at, + updated_at: weight.updated_at, + } + }).collect(); + + let response = ListWeightsByUserSuccessResponse { + success: true, + weights, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightsByUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file From e2c5839d0184c7628b2a15e594c7930c7b66f0fc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 01:05:26 -0500 Subject: [PATCH 025/136] surface storage full errors --- Cargo.lock | 1 + _sql/actions/generic_inference_jobs.sql | 5 + crates/lib/files/filesys/Cargo.toml | 1 + .../filesys/src/rename_across_devices.rs | 69 +++++- .../util/maybe_download_file_from_bucket.rs | 217 +++++++++--------- 5 files changed, 175 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1a8c8cad4..7b0664ab08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2278,6 +2278,7 @@ name = "filesys" version = "0.0.1" dependencies = [ "errors", + "log", "my-workspace-hack", ] diff --git a/_sql/actions/generic_inference_jobs.sql b/_sql/actions/generic_inference_jobs.sql index 5002ccfd4e..d22d74a190 100644 --- a/_sql/actions/generic_inference_jobs.sql +++ b/_sql/actions/generic_inference_jobs.sql @@ -18,3 +18,8 @@ update generic_inference_jobs set status = 'dead' where status IN ('pending', 'started', 'attempt_failed') and maybe_model_type IN ('sad_talker', 'so_vits_svc'); + +update generic_inference_jobs +set status = 'dead' +where status IN ('pending', 'started', 'attempt_failed') + and maybe_model_type IN ('tacotron2'); diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index 23d984ef53..52dec1d28b 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -17,6 +17,7 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +log = "0.4.20" # External # KEEP THIS TO A MINIMUM OR NIL. diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs index 60a4046976..5d6999b597 100644 --- a/crates/lib/files/filesys/src/rename_across_devices.rs +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -1,35 +1,80 @@ +use std::error::Error; +use std::fmt; use std::path::Path; +use log::error; + +#[derive(Debug)] +pub enum RenameError { + StorageFull, + IoError(std::io::Error), +} + +impl Error for RenameError {} + +impl fmt::Display for RenameError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RenameError::StorageFull => write!(f, "RenameError: storage is full"), + RenameError::IoError(err) => write!(f, "{:?}", err), + } + } +} + +impl From for RenameError { + fn from(error: std::io::Error) -> Self { + RenameError::IoError(error) + } +} + /// Rename a file. /// Ordinary rename will fail in Linux if it is across physical devices. /// This function will perform a copy followed by delete in that case. /// In both cases, this function will overwrite the destination if it already exists. -pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> std::io::Result<()> { +pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> Result<(), RenameError> { let result = std::fs::rename(&from, &to); - let err = match result.as_ref() { + let err = match result { Ok(_) => return Ok(()), // Rename succeeded. Err(err) => err, }; + // Nearly a third of `ErrorKind` is nightly rust, so we're going to have to match on + // unix error codes until the rest of the ErrorKind enum variants are stable. + // The error codes are maintained in sys/unix/mod.rs - decode_error_kind(i32) + + const E_CROSSES_DEVICES : i32 = 18; // pub const EXDEV: ::c_int = 18; + const E_FILESYSTEM_QUOTA_EXCEEDED : i32 = 122; // pub const EDQUOT: ::c_int = 122; + const E_STORAGE_FULL : i32 = 28; // pub const ENOSPC: ::c_int = 28; + match err.raw_os_error() { - Some(18) => { - // NO-OP: pass-through - // NB: We can clean this up in the future when `err.kind() == ErrorKind::CrossDevices` - // stabilizes. For now we can assume code "18" means we're trying to rename across devices. + Some(E_CROSSES_DEVICES) => { + // NB: Fall through. + } + Some(E_STORAGE_FULL) | Some(E_FILESYSTEM_QUOTA_EXCEEDED) => { + error!("Filesystem is full during rename."); + return Err(RenameError::StorageFull); } _ => { // Something else happened. Return original error. - return result; + return Err(RenameError::IoError(err)); } } - // FIXME(bt,2023-11-29): There appears to be a new bug where the std::fs::copy succeeds in - // copying zero bytes (perhaps the file was truncated to zero bytes before in std::fs::rename?), - // and we delete the source file (below). This only appears to happen in GCP currently, and this - // code has worked for months prior to this. - let _num_bytes = std::fs::copy(&from, &to)?; + // NB: In production we've seen std::fs::copy silently succeed, yet copy zero bytes. The reason + // is more than likely that the filesystem is full. We'll check the number of copied bytes to + // confirm. Moreover, the Rust docs say wrt std::fs::copy, "On success, the total number of bytes + // copied is returned and it is equal to the length of the to file as reported by metadata." + let original_file_size = std::fs::metadata(from.as_ref())?.len(); + let num_bytes_copied = std::fs::copy(&from, &to)?; + + if num_bytes_copied != original_file_size { + error!("Filesystem is full during copy. Not all bytes copied."); + return Err(RenameError::StorageFull); + } + + // An emulated "rename" would delete the source file. std::fs::remove_file(&from)?; Ok(()) diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 16bc900d13..c0905e4e7f 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -1,15 +1,12 @@ -use std::io::ErrorKind; use std::path::Path; use log::{error, info, warn}; -use tempdir::TempDir; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; -use errors::AnyhowResult; use filesys::file_size::file_size; -use filesys::rename_across_devices::rename_across_devices; +use filesys::rename_across_devices::{rename_across_devices, RenameError}; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -96,40 +93,44 @@ pub async fn maybe_download_file_from_bucket( info!("File size of {} temp download file {:?} is {original_size}", args.name_or_description_of_file, &temp_path); - //info!("Renaming {} temp file from {:?} to {:?}!", - // args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - - //rename_across_devices(&temp_path, &args.final_filesystem_file_path) - // .map_err(|err| { - // error!("could not rename on disk: {:?}", err); - // safe_delete_temp_directory(&temp_dir); - // ProcessSingleJobError::from_io_error(err) - // })?; - - info!("Copying {} temp file from {:?} to {:?}!", + info!("Renaming {} temp file from {:?} to {:?}!", args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - // NB: We're now seeing a bug where the resultant copied file is 0 bytes - match copy_with_logging(&temp_path, &args.final_filesystem_file_path) { - Err(err) => { - error!("Error Copying {} temp file from {:?} to {:?}! {err}", - args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); + rename_across_devices(&temp_path, &args.final_filesystem_file_path) + .map_err(|err| { + error!("could not rename on disk: {:?}", err); + safe_delete_temp_file(&temp_path); + safe_delete_temp_directory(&temp_dir); + match err { + RenameError::StorageFull => ProcessSingleJobError::FilesystemFull, + RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err), + } + })?; - safe_delete_temp_file(&temp_path); - safe_delete_temp_directory(&temp_dir); - - return Err(ProcessSingleJobError::from_anyhow_error(err)); - } - Ok(false) => { - error!("Error copying {} temp file from {:?} to {:?}! File size in bytes did not match.", - args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - - reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?; - } - Ok(true) => { - // Success case - } - } +// info!("Copying {} temp file from {:?} to {:?}!", +// args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); +// +// // NB: We're now seeing a bug where the resultant copied file is 0 bytes +// match copy_with_logging(&temp_path, &args.final_filesystem_file_path) { +// Err(err) => { +// error!("Error Copying {} temp file from {:?} to {:?}! {err}", +// args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); +// +// safe_delete_temp_file(&temp_path); +// safe_delete_temp_directory(&temp_dir); +// +// return Err(ProcessSingleJobError::from_anyhow_error(err)); +// } +// Ok(false) => { +// error!("Error copying {} temp file from {:?} to {:?}! File size in bytes did not match.", +// args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); +// +// reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?; +// } +// Ok(true) => { +// // Success case +// } +// } info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); @@ -139,74 +140,78 @@ pub async fn maybe_download_file_from_bucket( Ok(()) } -fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_path: &Path, original_size: u64) -> Result<(), ProcessSingleJobError> { - let copied_size = file_size(&args.final_filesystem_file_path) - .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - - info!("Final copied size of {} file {:?} is {copied_size} (original size was {original_size})", - args.name_or_description_of_file, &args.final_filesystem_file_path); - - if copied_size != 0 { - safe_delete_temp_file(&temp_path); - safe_delete_temp_directory(&temp_dir); - - return Ok(()); - } - - error!("Copied size was 0! Removing destination file and retrying..."); - - if let Err(err) = std::fs::remove_file(&args.final_filesystem_file_path) { - match err.kind() { - ErrorKind::NotFound => { - // NB: We seem to be seeing this in production. - // Perhaps another pod deleted it in a race condition? - // Fall through case. - warn!("File couldn't be removed: it's already gone.") - }, - _ => { - safe_delete_temp_file(&temp_path); - safe_delete_temp_directory(&temp_dir); - return Err(ProcessSingleJobError::from_io_error(err)); - } - } - } else { - warn!("File removed."); - } - - warn!("Retrying copy..."); - - rename_across_devices(&temp_path, &args.final_filesystem_file_path) - .map_err(|err| { - error!("could not rename on disk: {:?}", err); - safe_delete_temp_file(&temp_path); - safe_delete_temp_directory(&temp_dir); - ProcessSingleJobError::from_io_error(err) - })?; - - let copied_size = file_size(&args.final_filesystem_file_path) - .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - - warn!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", - args.name_or_description_of_file, &args.final_filesystem_file_path); - - Ok(()) -} - - -pub fn copy_with_logging, Q: AsRef>(from: P, to: Q) -> AnyhowResult { - let original_size = file_size(&from)?; - - let num_bytes_copied = std::fs::copy(&from, &to)?; - - let destination_size = file_size(&to)?; - - info!(r#"Copied {:?} to {:?} - - Copied bytes: {num_bytes_copied} - - Orig. bytes: {original_size} - - Dest. bytes: {destination_size} - "#, from.as_ref(), to.as_ref()); - - let file_sizes_match = original_size == destination_size; - - Ok(file_sizes_match) -} +// NB: Debugging failures with rename_across_devices() - +// fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_path: &Path, original_size: u64) -> Result<(), ProcessSingleJobError> { +// let copied_size = file_size(&args.final_filesystem_file_path) +// .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; +// +// info!("Final copied size of {} file {:?} is {copied_size} (original size was {original_size})", +// args.name_or_description_of_file, &args.final_filesystem_file_path); +// +// if copied_size != 0 { +// safe_delete_temp_file(&temp_path); +// safe_delete_temp_directory(&temp_dir); +// +// return Ok(()); +// } +// +// error!("Copied size was 0! Removing destination file and retrying..."); +// +// if let Err(err) = std::fs::remove_file(&args.final_filesystem_file_path) { +// match err.kind() { +// ErrorKind::NotFound => { +// // NB: We seem to be seeing this in production. +// // Perhaps another pod deleted it in a race condition? +// // Fall through case. +// warn!("File couldn't be removed: it's already gone.") +// }, +// _ => { +// safe_delete_temp_file(&temp_path); +// safe_delete_temp_directory(&temp_dir); +// return Err(ProcessSingleJobError::from_io_error(err)); +// } +// } +// } else { +// warn!("File removed."); +// } +// +// warn!("Retrying copy..."); +// +// rename_across_devices(&temp_path, &args.final_filesystem_file_path) +// .map_err(|err| { +// error!("could not rename on disk: {:?}", err); +// safe_delete_temp_file(&temp_path); +// safe_delete_temp_directory(&temp_dir); +// match err { +// RenameError::StorageFull => ProcessSingleJobError::FilesystemFull, +// RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err), +// } +// })?; +// +// let copied_size = file_size(&args.final_filesystem_file_path) +// .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; +// +// warn!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.", +// args.name_or_description_of_file, &args.final_filesystem_file_path); +// +// Ok(()) +// } +// +// +// pub fn copy_with_logging, Q: AsRef>(from: P, to: Q) -> AnyhowResult { +// let original_size = file_size(&from)?; +// +// let num_bytes_copied = std::fs::copy(&from, &to)?; +// +// let destination_size = file_size(&to)?; +// +// info!(r#"Copied {:?} to {:?} +// - Copied bytes: {num_bytes_copied} +// - Orig. bytes: {original_size} +// - Dest. bytes: {destination_size} +// "#, from.as_ref(), to.as_ref()); +// +// let file_sizes_match = original_size == destination_size; +// +// Ok(file_sizes_match) +// } From b82ffcce8656fd1a33526ac2659027e3362bbe43 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 01:26:03 -0500 Subject: [PATCH 026/136] much more StorageFull error ! --- .../filesys/src/is_filesystem_full_error.rs | 16 ++++++++++++++++ crates/lib/files/filesys/src/lib.rs | 1 + .../files/filesys/src/rename_across_devices.rs | 14 +++++++------- .../job/job_loop/process_single_job_error.rs | 18 ++++++++++++++++++ .../lipsync/sad_talker/process_job.rs | 6 +++--- .../tts/vall_e_x/process_job_create_voice.rs | 6 +++--- .../vall_e_x/process_job_inference_voice.rs | 6 +++--- .../vc/rvc_v2/pretrained_hubert_model.rs | 3 ++- .../src/job/job_types/vc/rvc_v2/process_job.rs | 12 ++++++------ .../inference_job/src/util/model_downloader.rs | 14 +++++++++----- 10 files changed, 68 insertions(+), 28 deletions(-) create mode 100644 crates/lib/files/filesys/src/is_filesystem_full_error.rs diff --git a/crates/lib/files/filesys/src/is_filesystem_full_error.rs b/crates/lib/files/filesys/src/is_filesystem_full_error.rs new file mode 100644 index 0000000000..f4f8301523 --- /dev/null +++ b/crates/lib/files/filesys/src/is_filesystem_full_error.rs @@ -0,0 +1,16 @@ + +// Nearly a third of `ErrorKind` is nightly rust, so we're going to have to match on +// unix error codes until the rest of the ErrorKind enum variants are stable. +// The error codes are maintained in sys/unix/mod.rs - decode_error_kind(i32) + +const E_FILESYSTEM_QUOTA_EXCEEDED : i32 = 122; // pub const EDQUOT: ::c_int = 122; +const E_STORAGE_FULL : i32 = 28; // pub const ENOSPC: ::c_int = 28; + +/// Reports if an io::Error was caused by the filesystem being full. +pub fn is_filesystem_full_error(err: &std::io::Error) -> bool { + match err.raw_os_error() { + Some(E_FILESYSTEM_QUOTA_EXCEEDED) => true, + Some(E_STORAGE_FULL) => true, + _ => false, + } +} diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 04a482ddcc..9d227d7a92 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -24,5 +24,6 @@ pub mod file_exists; pub mod file_read_bytes; pub mod file_size; pub mod filename_concat; +pub mod is_filesystem_full_error; pub mod path_to_string; pub mod rename_across_devices; diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs index 5d6999b597..25b9e05a31 100644 --- a/crates/lib/files/filesys/src/rename_across_devices.rs +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -4,6 +4,8 @@ use std::path::Path; use log::error; +use crate::is_filesystem_full_error::is_filesystem_full_error; + #[derive(Debug)] pub enum RenameError { StorageFull, @@ -42,19 +44,17 @@ pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> // Nearly a third of `ErrorKind` is nightly rust, so we're going to have to match on // unix error codes until the rest of the ErrorKind enum variants are stable. // The error codes are maintained in sys/unix/mod.rs - decode_error_kind(i32) - const E_CROSSES_DEVICES : i32 = 18; // pub const EXDEV: ::c_int = 18; - const E_FILESYSTEM_QUOTA_EXCEEDED : i32 = 122; // pub const EDQUOT: ::c_int = 122; - const E_STORAGE_FULL : i32 = 28; // pub const ENOSPC: ::c_int = 28; + + if is_filesystem_full_error(&err) { + error!("Filesystem is full during rename."); + return Err(RenameError::StorageFull); + } match err.raw_os_error() { Some(E_CROSSES_DEVICES) => { // NB: Fall through. } - Some(E_STORAGE_FULL) | Some(E_FILESYSTEM_QUOTA_EXCEEDED) => { - error!("Filesystem is full during rename."); - return Err(RenameError::StorageFull); - } _ => { // Something else happened. Return original error. return Err(RenameError::IoError(err)); diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs index 7277671180..c7108e69e9 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs @@ -1,5 +1,7 @@ use anyhow::anyhow; +use filesys::is_filesystem_full_error::is_filesystem_full_error; + /// Error from processing a single job #[derive(Debug)] pub enum ProcessSingleJobError { @@ -37,6 +39,22 @@ pub enum ProcessSingleJobError { NotYetImplemented } +impl From for ProcessSingleJobError { + fn from(error: std::io::Error) -> Self { + if is_filesystem_full_error(&error) { + ProcessSingleJobError::FilesystemFull + } else { + ProcessSingleJobError::Other(anyhow!(error)) + } + } +} + +impl From for ProcessSingleJobError { + fn from(error: anyhow::Error) -> Self { + ProcessSingleJobError::Other(error) + } +} + impl ProcessSingleJobError { pub fn from_io_error(error: std::io::Error) -> Self { match error.raw_os_error() { diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 009fb63b28..298918d363 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -77,9 +77,9 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result AnyhowResult<()> { + ) -> Result<(), ProcessSingleJobError> { if file_exists(&self.filesystem_path) { return Ok(()); } diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index e357f367db..344657eaf4 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -66,9 +66,9 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result) -> Result AnyhowResult<()> { + ) -> Result<(), ProcessSingleJobError> { let filesystem_path = self.get_filesystem_path(); if file_exists(filesystem_path) { @@ -71,10 +72,13 @@ pub trait ModelDownloader { info!("Renaming {} file from {:?} to {:?}!", model_name, &temp_path, filesystem_path); rename_across_devices(&temp_path, filesystem_path) - .map_err(|e| { - error!("could not rename on disk: {:?}", e); + .map_err(|err| { + error!("could not rename on disk: {:?}", err); safe_delete_temp_directory(&temp_dir); - anyhow!("couldn't rename disk files: {:?}", e) + match err { + RenameError::StorageFull => ProcessSingleJobError::FilesystemFull, + RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err), + } })?; info!("Finished downloading {} file to {:?}", model_name, filesystem_path); From f93421eadc08b8c7d4e9d7184f21af16ea195c55 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 01:35:48 -0500 Subject: [PATCH 027/136] keep IoError around; better casting --- .../src/job/job_loop/main_loop.rs | 10 +++++- .../job/job_loop/process_single_job_error.rs | 36 +++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 8da6cc00c7..207a63051d 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -216,7 +216,14 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere None, // Obviously don't tell the user about errors even we're not sure about Some(FrontendFailureCategory::RetryableWorkerError), ), - + ProcessSingleJobError::IoError(ref err) => + ( + JobFailureClass::TransientFailure, + ContainerHealth::IncrementContainerFailCount, + format!("IoError: {:?}", err), + None, // Obviously don't tell the user about errors even we're not sure about + Some(FrontendFailureCategory::RetryableWorkerError), + ), ProcessSingleJobError::JobSystemMisconfiguration(ref maybe_reason) => ( JobFailureClass::TransientFailure, @@ -269,6 +276,7 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere ProcessSingleJobError::FaceDetectionFailure => {} ProcessSingleJobError::JobSystemMisconfiguration(_) => {} ProcessSingleJobError::ModelDeleted => {} + ProcessSingleJobError::IoError(_) => {} } Ok(()) diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs index c7108e69e9..eaa0d498cd 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs @@ -27,6 +27,10 @@ pub enum ProcessSingleJobError { /// It might be important, we just haven't special cased it yet. Other(anyhow::Error), + /// This is a non-special-cased IO Error + /// It might be important, we just haven't special cased it yet. + IoError(std::io::Error), + /// Job underlying resources are missing - model deleted ModelDeleted, @@ -41,39 +45,35 @@ pub enum ProcessSingleJobError { impl From for ProcessSingleJobError { fn from(error: std::io::Error) -> Self { - if is_filesystem_full_error(&error) { - ProcessSingleJobError::FilesystemFull - } else { - ProcessSingleJobError::Other(anyhow!(error)) - } + ProcessSingleJobError::from_io_error(error) } } impl From for ProcessSingleJobError { fn from(error: anyhow::Error) -> Self { - ProcessSingleJobError::Other(error) + ProcessSingleJobError::from_anyhow_error(error) } } impl ProcessSingleJobError { pub fn from_io_error(error: std::io::Error) -> Self { - match error.raw_os_error() { - // NB: We can't use err.kind() -> ErrorKind::StorageFull, because it's marked unstable: - // `io_error_more` is unstable [E0658] - Some(28) => ProcessSingleJobError::FilesystemFull, - _ => ProcessSingleJobError::Other(anyhow!(error)), + if is_filesystem_full_error(&error) { + ProcessSingleJobError::FilesystemFull + } else { + ProcessSingleJobError::IoError(error) } } pub fn from_anyhow_error(error: anyhow::Error) -> Self { - match error.downcast_ref::() { - Some(e) => match e.raw_os_error() { - // NB: We can't use err.kind() -> ErrorKind::StorageFull, because it's marked unstable: - // `io_error_more` is unstable [E0658] - Some(28) => ProcessSingleJobError::FilesystemFull, - _ => ProcessSingleJobError::Other(anyhow!(error)), + match error.downcast::() { + Ok(io_error) => { + if is_filesystem_full_error(&io_error) { + ProcessSingleJobError::FilesystemFull + } else { + ProcessSingleJobError::IoError(io_error) + } }, - None => ProcessSingleJobError::Other(error), + Err(anyhow_error) => ProcessSingleJobError::Other(anyhow_error), } } } From ff2aaa9f3de758f003cfe303724198c88cde76e3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 01:53:42 -0500 Subject: [PATCH 028/136] move `check_file_exists` to filesys crate --- Cargo.lock | 4 ++++ crates/lib/deprecated/container_common/src/filesystem/mod.rs | 1 - .../filesystem => files/filesys/src}/check_file_exists.rs | 5 ++--- crates/lib/files/filesys/src/lib.rs | 1 + .../src/job_types/tts/tacotron/process_tacotron_model.rs | 2 +- .../src/job_types/tts/vits/process_vits_model.rs | 2 +- .../vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs | 2 +- .../vocoder/hifigan_tacotron/process_hifigan_vocoder.rs | 2 +- .../voice_conversion/rvc_v2/process_rvc_v2_model.rs | 2 +- .../so_vits_svc/process_so_vits_svc_model.rs | 2 +- .../voice_conversion/softvc/process_softvc_model.rs | 2 +- .../src/job/job_loop/determine_dependency_status.rs | 4 ++-- .../job/inference_job/src/job/job_loop/process_single_job.rs | 1 - .../src/job/job_loop/process_single_job_error.rs | 2 -- .../src/job/job_types/lipsync/sad_talker/process_job.rs | 2 +- .../job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs | 2 +- .../inference_job/src/job/job_types/tts/vits/process_job.rs | 2 +- .../src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs | 3 +-- .../inference_job/src/job/job_types/vc/rvc_v2/process_job.rs | 2 +- .../src/job/job_types/vc/so_vits_svc/process_job.rs | 2 +- .../service/job/inference_job/src/util/model_downloader.rs | 3 +-- crates/service/job/tts_download_job/Cargo.toml | 1 + crates/service/job/tts_download_job/src/main.rs | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 1 + .../tts_inference_job/src/job_steps/process_single_job.rs | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 1 + crates/service/job/w2l_download_job/src/main.rs | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 1 + crates/service/job/w2l_inference_job/src/main.rs | 2 +- 29 files changed, 31 insertions(+), 29 deletions(-) rename crates/lib/{deprecated/container_common/src/filesystem => files/filesys/src}/check_file_exists.rs (80%) diff --git a/Cargo.lock b/Cargo.lock index 7b0664ab08..e23d4f99cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6659,6 +6659,7 @@ dependencies = [ "dotenv", "easyenv", "enums", + "filesys", "google_drive_common", "hashing", "hostname", @@ -6689,6 +6690,7 @@ dependencies = [ "container_common", "dotenv", "easyenv", + "filesys", "hashing", "hostname", "hyper", @@ -7091,6 +7093,7 @@ dependencies = [ "container_common", "dotenv", "easyenv", + "filesys", "google_drive_common", "hashing", "hostname", @@ -7120,6 +7123,7 @@ dependencies = [ "container_common", "dotenv", "easyenv", + "filesys", "google_drive_common", "hostname", "hyper", diff --git a/crates/lib/deprecated/container_common/src/filesystem/mod.rs b/crates/lib/deprecated/container_common/src/filesystem/mod.rs index 4bcf98c71a..654eb2fbf2 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/mod.rs +++ b/crates/lib/deprecated/container_common/src/filesystem/mod.rs @@ -1,5 +1,4 @@ pub mod check_directory_exists; -pub mod check_file_exists; pub mod safe_delete_possible_temp_file; pub mod safe_delete_temp_directory; pub mod safe_delete_temp_file; \ No newline at end of file diff --git a/crates/lib/deprecated/container_common/src/filesystem/check_file_exists.rs b/crates/lib/files/filesys/src/check_file_exists.rs similarity index 80% rename from crates/lib/deprecated/container_common/src/filesystem/check_file_exists.rs rename to crates/lib/files/filesys/src/check_file_exists.rs index ef239c0bf8..62805c4caf 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/check_file_exists.rs +++ b/crates/lib/files/filesys/src/check_file_exists.rs @@ -1,8 +1,7 @@ use std::path::PathBuf; -use anyhow::bail; - -use crate::anyhow_result::AnyhowResult; +use errors::AnyhowResult; +use errors::bail; pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> { if !path.exists() { diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 9d227d7a92..3920688931 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -18,6 +18,7 @@ #![allow(dead_code)] #![allow(non_snake_case)] +pub mod check_file_exists; pub mod create_dir_all_if_missing; pub mod directory_exists; pub mod file_exists; diff --git a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs index 3f17a79862..9b9dc6020f 100644 --- a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs @@ -6,12 +6,12 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs index 715f94f454..0d97715b51 100644 --- a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs @@ -4,12 +4,12 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; use filesys::filename_concat::filename_concat; use hashing::sha256::sha256_hash_file::sha256_hash_file; diff --git a/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs b/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs index 9263285186..43e8c7c80e 100644 --- a/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs +++ b/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs @@ -6,11 +6,11 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::common::vocoder_type::VocoderType; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs b/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs index fc3ff817ec..c6673fc4bd 100644 --- a/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs +++ b/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs @@ -6,11 +6,11 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::common::vocoder_type::VocoderType; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs index fc54d31f58..b141bef100 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs @@ -4,7 +4,6 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_possible_temp_file::safe_delete_possible_temp_file; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; @@ -12,6 +11,7 @@ use crockford::crockford_entropy_lower; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs index 3747879250..c8eec94c11 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs @@ -4,12 +4,12 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs index 9fdf46f18a..9e381b22f4 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs @@ -6,12 +6,12 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs index c89ccd8228..4d150bc421 100644 --- a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs +++ b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs @@ -1,17 +1,17 @@ use std::path::PathBuf; + use anyhow::anyhow; use log::{info, warn}; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; -use errors::AnyhowResult; use filesys::file_exists::file_exists; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord}; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference}; -use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job_dependencies::JobDependencies; pub struct DependencyStatus { diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 39b3b895b4..9934ab1efd 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -11,7 +11,6 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s use redis_common::redis_keys::RedisKeys; use crate::job::job_loop::determine_dependency_status::determine_dependency_status; -use crate::job::job_loop::job_success_result::ResultEntity; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase; use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs index eaa0d498cd..ee530863d0 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs @@ -1,5 +1,3 @@ -use anyhow::anyhow; - use filesys::is_filesystem_full_error::is_filesystem_full_error; /// Error from processing a single job diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 298918d363..688b493261 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -5,10 +5,10 @@ use anyhow::anyhow; use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 8c3d143f80..b3f9ed7f54 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -7,11 +7,11 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index 15ee5a53f8..802a09073a 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -7,11 +7,11 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs index 3276decb18..3f9f2f8c75 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs @@ -5,12 +5,11 @@ use log::{error, info}; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use errors::AnyhowResult; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::rename_across_devices; -use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; #[derive(Clone)] diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 344657eaf4..45efc8f1bc 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -6,10 +6,10 @@ use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 7b36c150b8..dd41ff0d5f 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -5,10 +5,10 @@ use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; diff --git a/crates/service/job/inference_job/src/util/model_downloader.rs b/crates/service/job/inference_job/src/util/model_downloader.rs index 126e9b9862..57bf7d8bbc 100644 --- a/crates/service/job/inference_job/src/util/model_downloader.rs +++ b/crates/service/job/inference_job/src/util/model_downloader.rs @@ -6,12 +6,11 @@ use log::{error, info}; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use errors::AnyhowResult; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::{rename_across_devices, RenameError}; -use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; /// Helper utility for downloading pretrained models from GCS. diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 14d14e3793..ccdd04bd2b 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -20,6 +20,7 @@ config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } enums = { path = "../../../_types/enums" } +filesys = { path = "../../../lib/files/filesys" } google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } diff --git a/crates/service/job/tts_download_job/src/main.rs b/crates/service/job/tts_download_job/src/main.rs index 332b2f874b..9a39832650 100644 --- a/crates/service/job/tts_download_job/src/main.rs +++ b/crates/service/job/tts_download_job/src/main.rs @@ -36,10 +36,10 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::filesystem::check_directory_exists::check_directory_exists; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; +use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::noop_logger::NoOpLogger; diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index a72aa180ea..204902744f 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -19,6 +19,7 @@ cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } +filesys = { path = "../../../lib/files/filesys" } hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } memory_caching = { path = "../../../lib/caching/memory_caching" } diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index 7fe1b60d90..117b7ac8fe 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -13,10 +13,10 @@ use log::{error, info, warn}; use tempdir::TempDir; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::token::random_uuid::generate_random_uuid; +use filesys::check_file_exists::check_file_exists; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 1619016588..c71e96c24b 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -20,6 +20,7 @@ cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } +filesys = { path = "../../../lib/files/filesys" } google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } diff --git a/crates/service/job/w2l_download_job/src/main.rs b/crates/service/job/w2l_download_job/src/main.rs index 20544c5036..63e3318849 100644 --- a/crates/service/job/w2l_download_job/src/main.rs +++ b/crates/service/job/w2l_download_job/src/main.rs @@ -36,9 +36,9 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::filesystem::check_directory_exists::check_directory_exists; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::noop_logger::NoOpLogger; diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 85e15340c1..04d38e114d 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -19,6 +19,7 @@ cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } +filesys = { path = "../../../lib/files/filesys" } google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } diff --git a/crates/service/job/w2l_inference_job/src/main.rs b/crates/service/job/w2l_inference_job/src/main.rs index bdc642e952..4485eefc55 100644 --- a/crates/service/job/w2l_inference_job/src/main.rs +++ b/crates/service/job/w2l_inference_job/src/main.rs @@ -37,9 +37,9 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::filesystem::check_directory_exists::check_directory_exists; -use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; From b5ed9acdd468d7412686b9c695f9f2f6a83a0f61 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 01:56:55 -0500 Subject: [PATCH 029/136] move `check_directory_exists` to filesys crate --- crates/lib/deprecated/container_common/src/filesystem/mod.rs | 1 - .../filesys/src}/check_directory_exists.rs | 4 +--- crates/lib/files/filesys/src/lib.rs | 1 + crates/service/job/download_job/src/main.rs | 2 +- crates/service/job/inference_job/src/main.rs | 2 +- crates/service/job/inference_job/src/util/file_cache_dir.rs | 2 +- crates/service/job/tts_download_job/src/main.rs | 2 +- crates/service/job/tts_inference_job/src/main.rs | 2 +- crates/service/job/w2l_download_job/src/main.rs | 2 +- crates/service/job/w2l_inference_job/src/main.rs | 2 +- 10 files changed, 9 insertions(+), 11 deletions(-) rename crates/lib/{deprecated/container_common/src/filesystem => files/filesys/src}/check_directory_exists.rs (84%) diff --git a/crates/lib/deprecated/container_common/src/filesystem/mod.rs b/crates/lib/deprecated/container_common/src/filesystem/mod.rs index 654eb2fbf2..7f4cf77e01 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/mod.rs +++ b/crates/lib/deprecated/container_common/src/filesystem/mod.rs @@ -1,4 +1,3 @@ -pub mod check_directory_exists; pub mod safe_delete_possible_temp_file; pub mod safe_delete_temp_directory; pub mod safe_delete_temp_file; \ No newline at end of file diff --git a/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs b/crates/lib/files/filesys/src/check_directory_exists.rs similarity index 84% rename from crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs rename to crates/lib/files/filesys/src/check_directory_exists.rs index 8e770b171f..adef055017 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/check_directory_exists.rs +++ b/crates/lib/files/filesys/src/check_directory_exists.rs @@ -1,8 +1,6 @@ use std::path::Path; -use anyhow::bail; - -use crate::anyhow_result::AnyhowResult; +use errors::{AnyhowResult, bail}; pub fn check_directory_exists>(path: P) -> AnyhowResult<()> { let check_path = path.as_ref(); diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 3920688931..619d529781 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -28,3 +28,4 @@ pub mod filename_concat; pub mod is_filesystem_full_error; pub mod path_to_string; pub mod rename_across_devices; +pub mod check_directory_exists; diff --git a/crates/service/job/download_job/src/main.rs b/crates/service/job/download_job/src/main.rs index 7687a017fc..6967c0772f 100644 --- a/crates/service/job/download_job/src/main.rs +++ b/crates/service/job/download_job/src/main.rs @@ -33,8 +33,8 @@ use cloud_storage::bucket_path_unifier::BucketPathUnifier; use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::filesystem::check_directory_exists::check_directory_exists; use errors::AnyhowResult; +use filesys::check_directory_exists::check_directory_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg; use mysql_queries::mediators::badge_granter::BadgeGranter; diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index f0ac58a93f..e289e14d77 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -33,7 +33,7 @@ use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::check_directory_exists::check_directory_exists; +use filesys::check_directory_exists::check_directory_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder; diff --git a/crates/service/job/inference_job/src/util/file_cache_dir.rs b/crates/service/job/inference_job/src/util/file_cache_dir.rs index 0389c9a4c9..82af362d5b 100644 --- a/crates/service/job/inference_job/src/util/file_cache_dir.rs +++ b/crates/service/job/inference_job/src/util/file_cache_dir.rs @@ -1,8 +1,8 @@ use std::io; use std::path::{Path, PathBuf}; -use container_common::filesystem::check_directory_exists::check_directory_exists; use errors::AnyhowResult; +use filesys::check_directory_exists::check_directory_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; /// Utilities for directories serving as file caches diff --git a/crates/service/job/tts_download_job/src/main.rs b/crates/service/job/tts_download_job/src/main.rs index 9a39832650..36d88f885d 100644 --- a/crates/service/job/tts_download_job/src/main.rs +++ b/crates/service/job/tts_download_job/src/main.rs @@ -35,10 +35,10 @@ use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::check_directory_exists::check_directory_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; +use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index 88f73081b4..7244d4057c 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -32,7 +32,7 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::collections::multiple_random_from_vec::multiple_random_from_vec; -use container_common::filesystem::check_directory_exists::check_directory_exists; +use filesys::check_directory_exists::check_directory_exists; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder; use jobs_common::job_progress_reporter::redis_job_progress_reporter::RedisJobProgressReporterBuilder; diff --git a/crates/service/job/w2l_download_job/src/main.rs b/crates/service/job/w2l_download_job/src/main.rs index 63e3318849..b84b3075b3 100644 --- a/crates/service/job/w2l_download_job/src/main.rs +++ b/crates/service/job/w2l_download_job/src/main.rs @@ -35,9 +35,9 @@ use config::is_bad_video_download_url::is_bad_video_download_url; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::check_directory_exists::check_directory_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; diff --git a/crates/service/job/w2l_inference_job/src/main.rs b/crates/service/job/w2l_inference_job/src/main.rs index 4485eefc55..f5682e76c9 100644 --- a/crates/service/job/w2l_inference_job/src/main.rs +++ b/crates/service/job/w2l_inference_job/src/main.rs @@ -36,9 +36,9 @@ use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::check_directory_exists::check_directory_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; From a669ac7f03817d778981d5fa8be5547df65bd7e1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:04:41 -0500 Subject: [PATCH 030/136] move remaining container_common::files -> filesys crate --- .../lib/deprecated/container_common/src/filesystem/mod.rs | 4 +--- crates/lib/files/filesys/src/lib.rs | 3 +++ .../filesys/src}/safe_delete_possible_temp_file.rs | 2 +- .../filesys/src}/safe_delete_temp_directory.rs | 0 .../filesys/src}/safe_delete_temp_file.rs | 0 .../job/download_job/src/job_loop/process_single_job.rs | 2 +- .../src/job_types/tts/tacotron/process_tacotron_model.rs | 4 ++-- .../src/job_types/tts/vits/process_vits_model.rs | 4 ++-- .../hifigan_softvc/process_hifigan_softvc_vocoder.rs | 4 ++-- .../vocoder/hifigan_tacotron/process_hifigan_vocoder.rs | 4 ++-- .../job_types/voice_conversion/rvc_v2/extract_rvc_files.rs | 2 +- .../voice_conversion/rvc_v2/pretrained_hubert_model.rs | 2 +- .../voice_conversion/rvc_v2/process_rvc_v2_model.rs | 6 +++--- .../so_vits_svc/process_so_vits_svc_model.rs | 4 ++-- .../voice_conversion/softvc/process_softvc_model.rs | 4 ++-- .../src/job/job_types/lipsync/sad_talker/process_job.rs | 4 ++-- .../job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs | 4 ++-- .../inference_job/src/job/job_types/tts/vits/process_job.rs | 4 ++-- .../src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs | 2 +- .../src/job/job_types/vc/rvc_v2/process_job.rs | 4 ++-- .../src/job/job_types/vc/so_vits_svc/process_job.rs | 4 ++-- .../src/util/maybe_download_file_from_bucket.rs | 4 ++-- .../service/job/inference_job/src/util/model_downloader.rs | 2 +- crates/service/job/tts_download_job/src/main.rs | 4 ++-- .../tts_inference_job/src/job_steps/process_single_job.rs | 4 ++-- crates/service/job/w2l_download_job/src/main.rs | 4 ++-- crates/service/job/w2l_inference_job/src/main.rs | 4 ++-- 27 files changed, 45 insertions(+), 44 deletions(-) rename crates/lib/{deprecated/container_common/src/filesystem => files/filesys/src}/safe_delete_possible_temp_file.rs (73%) rename crates/lib/{deprecated/container_common/src/filesystem => files/filesys/src}/safe_delete_temp_directory.rs (100%) rename crates/lib/{deprecated/container_common/src/filesystem => files/filesys/src}/safe_delete_temp_file.rs (100%) diff --git a/crates/lib/deprecated/container_common/src/filesystem/mod.rs b/crates/lib/deprecated/container_common/src/filesystem/mod.rs index 7f4cf77e01..8b13789179 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/mod.rs +++ b/crates/lib/deprecated/container_common/src/filesystem/mod.rs @@ -1,3 +1 @@ -pub mod safe_delete_possible_temp_file; -pub mod safe_delete_temp_directory; -pub mod safe_delete_temp_file; \ No newline at end of file + diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 619d529781..05defce271 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -29,3 +29,6 @@ pub mod is_filesystem_full_error; pub mod path_to_string; pub mod rename_across_devices; pub mod check_directory_exists; +pub mod safe_delete_temp_file; +pub mod safe_delete_possible_temp_file; +pub mod safe_delete_temp_directory; diff --git a/crates/lib/deprecated/container_common/src/filesystem/safe_delete_possible_temp_file.rs b/crates/lib/files/filesys/src/safe_delete_possible_temp_file.rs similarity index 73% rename from crates/lib/deprecated/container_common/src/filesystem/safe_delete_possible_temp_file.rs rename to crates/lib/files/filesys/src/safe_delete_possible_temp_file.rs index 5d2f25c8eb..437193b31d 100644 --- a/crates/lib/deprecated/container_common/src/filesystem/safe_delete_possible_temp_file.rs +++ b/crates/lib/files/filesys/src/safe_delete_possible_temp_file.rs @@ -1,6 +1,6 @@ use std::path::Path; -use crate::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use crate::safe_delete_temp_file::safe_delete_temp_file; pub fn safe_delete_possible_temp_file>(maybe_file_path: Option

) { if let Some(file_path) = maybe_file_path { diff --git a/crates/lib/deprecated/container_common/src/filesystem/safe_delete_temp_directory.rs b/crates/lib/files/filesys/src/safe_delete_temp_directory.rs similarity index 100% rename from crates/lib/deprecated/container_common/src/filesystem/safe_delete_temp_directory.rs rename to crates/lib/files/filesys/src/safe_delete_temp_directory.rs diff --git a/crates/lib/deprecated/container_common/src/filesystem/safe_delete_temp_file.rs b/crates/lib/files/filesys/src/safe_delete_temp_file.rs similarity index 100% rename from crates/lib/deprecated/container_common/src/filesystem/safe_delete_temp_file.rs rename to crates/lib/files/filesys/src/safe_delete_temp_file.rs diff --git a/crates/service/job/download_job/src/job_loop/process_single_job.rs b/crates/service/job/download_job/src/job_loop/process_single_job.rs index 91bb79ddec..ba69531e70 100644 --- a/crates/service/job/download_job/src/job_loop/process_single_job.rs +++ b/crates/service/job/download_job/src/job_loop/process_single_job.rs @@ -6,7 +6,7 @@ use tempdir::TempDir; use config::is_bad_download_url::is_bad_download_url; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; use mysql_queries::queries::generic_download::job::mark_generic_download_job_done::mark_generic_download_job_done; diff --git a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs index 9b9dc6020f..255e4b3df4 100644 --- a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs @@ -6,12 +6,12 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs index 0d97715b51..e6b94eff1e 100644 --- a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs @@ -4,14 +4,14 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; use filesys::filename_concat::filename_concat; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs b/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs index 43e8c7c80e..ac73745c77 100644 --- a/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs +++ b/crates/service/job/download_job/src/job_types/vocoder/hifigan_softvc/process_hifigan_softvc_vocoder.rs @@ -6,11 +6,11 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::common::vocoder_type::VocoderType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs b/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs index c6673fc4bd..610f61dc99 100644 --- a/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs +++ b/crates/service/job/download_job/src/job_types/vocoder/hifigan_tacotron/process_hifigan_vocoder.rs @@ -6,11 +6,11 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::common::vocoder_type::VocoderType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/extract_rvc_files.rs b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/extract_rvc_files.rs index 1992d388d8..fcd7457424 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/extract_rvc_files.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/extract_rvc_files.rs @@ -3,8 +3,8 @@ use std::path::{Path, PathBuf}; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use errors::AnyhowResult; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; #[derive(Debug)] diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs index cc18241030..f6aa5807fc 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs @@ -5,10 +5,10 @@ use log::info; use tempdir::TempDir; use cloud_storage::bucket_client::BucketClient; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use errors::AnyhowResult; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; #[derive(Clone)] pub struct PretrainedHubertModel { diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs index b141bef100..a7f7fea4d3 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/rvc_v2/process_rvc_v2_model.rs @@ -4,15 +4,15 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_possible_temp_file::safe_delete_possible_temp_file; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use crockford::crockford_entropy_lower; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; +use filesys::safe_delete_possible_temp_file::safe_delete_possible_temp_file; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; use mysql_queries::queries::voice_conversion::models::insert_voice_conversion_model_from_download_job::{insert_voice_conversion_model_from_download_job, InsertVoiceConversionModelArgs}; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs index c8eec94c11..d7500c91f3 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/so_vits_svc/process_so_vits_svc_model.rs @@ -4,13 +4,13 @@ use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs b/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs index 9e381b22f4..ea656c8dab 100644 --- a/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs +++ b/crates/service/job/download_job/src/job_types/voice_conversion/softvc/process_softvc_model.rs @@ -6,12 +6,12 @@ use anyhow::anyhow; use log::{info, warn}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 688b493261..14fd9a32e0 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -5,11 +5,11 @@ use anyhow::anyhow; use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index b3f9ed7f54..c6559a478a 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -7,11 +7,11 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index 802a09073a..336631315d 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -7,11 +7,11 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs index 3f9f2f8c75..baf225cc84 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs @@ -4,10 +4,10 @@ use anyhow::anyhow; use log::{error, info}; use cloud_storage::bucket_client::BucketClient; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::rename_across_devices; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 45efc8f1bc..16a8992dfd 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -6,12 +6,12 @@ use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use media::decode_basic_audio_info::decode_basic_audio_file_info; use mimetypes::mimetype_for_file::get_mimetype_for_file; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index dd41ff0d5f..5a54968f7a 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -5,12 +5,12 @@ use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use media::decode_basic_audio_info::decode_basic_audio_file_info; use mimetypes::mimetype_for_file::get_mimetype_for_file; diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index c0905e4e7f..d5e6bffa90 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -3,10 +3,10 @@ use std::path::Path; use log::{error, info, warn}; use cloud_storage::bucket_client::BucketClient; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use filesys::file_size::file_size; use filesys::rename_across_devices::{rename_across_devices, RenameError}; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; diff --git a/crates/service/job/inference_job/src/util/model_downloader.rs b/crates/service/job/inference_job/src/util/model_downloader.rs index 57bf7d8bbc..e450ce6354 100644 --- a/crates/service/job/inference_job/src/util/model_downloader.rs +++ b/crates/service/job/inference_job/src/util/model_downloader.rs @@ -5,10 +5,10 @@ use async_trait::async_trait; use log::{error, info}; use cloud_storage::bucket_client::BucketClient; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::{rename_across_devices, RenameError}; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; diff --git a/crates/service/job/tts_download_job/src/main.rs b/crates/service/job/tts_download_job/src/main.rs index 36d88f885d..f7a915a4af 100644 --- a/crates/service/job/tts_download_job/src/main.rs +++ b/crates/service/job/tts_download_job/src/main.rs @@ -35,11 +35,11 @@ use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::tts_models::tts_model_type::TtsModelType; use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::noop_logger::NoOpLogger; diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index 117b7ac8fe..bb9cd522a8 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -13,10 +13,10 @@ use log::{error, info, warn}; use tempdir::TempDir; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::token::random_uuid::generate_random_uuid; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; diff --git a/crates/service/job/w2l_download_job/src/main.rs b/crates/service/job/w2l_download_job/src/main.rs index b84b3075b3..324f2337c5 100644 --- a/crates/service/job/w2l_download_job/src/main.rs +++ b/crates/service/job/w2l_download_job/src/main.rs @@ -35,10 +35,10 @@ use config::is_bad_video_download_url::is_bad_video_download_url; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use hashing::sha256::sha256_hash_file::sha256_hash_file; use jobs_common::noop_logger::NoOpLogger; diff --git a/crates/service/job/w2l_inference_job/src/main.rs b/crates/service/job/w2l_inference_job/src/main.rs index f5682e76c9..a15ed3d06a 100644 --- a/crates/service/job/w2l_inference_job/src/main.rs +++ b/crates/service/job/w2l_inference_job/src/main.rs @@ -36,10 +36,10 @@ use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; -use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use filesys::check_directory_exists::check_directory_exists; use filesys::check_file_exists::check_file_exists; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; From 8e45a4ea45b474922327526edcefb02bb077e203 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:05:15 -0500 Subject: [PATCH 031/136] delete mod --- crates/lib/deprecated/container_common/src/filesystem/mod.rs | 1 - crates/lib/deprecated/container_common/src/lib.rs | 1 - 2 files changed, 2 deletions(-) delete mode 100644 crates/lib/deprecated/container_common/src/filesystem/mod.rs diff --git a/crates/lib/deprecated/container_common/src/filesystem/mod.rs b/crates/lib/deprecated/container_common/src/filesystem/mod.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/crates/lib/deprecated/container_common/src/filesystem/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/lib/deprecated/container_common/src/lib.rs b/crates/lib/deprecated/container_common/src/lib.rs index 7db94caa9d..49c1946532 100644 --- a/crates/lib/deprecated/container_common/src/lib.rs +++ b/crates/lib/deprecated/container_common/src/lib.rs @@ -1,7 +1,6 @@ pub mod anyhow_result; pub mod collections; pub mod files; -pub mod filesystem; pub mod i18n; pub mod numerics; pub mod thread; From b27070e23e4b4698ac7bb6697815158f317fa1f8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:25:16 -0500 Subject: [PATCH 032/136] get rid of more of container_common --- Cargo.lock | 13 ++- Cargo.toml | 1 + crates/lib/collections/Cargo.toml | 26 +++++ crates/lib/collections/src/lib.rs | 22 ++++ .../src}/multiple_random_from_vec.rs | 2 +- .../src}/random_from_vec.rs | 2 +- .../container_common/src/collections/mod.rs | 2 - .../deprecated/container_common/src/lib.rs | 1 - .../container_common/src/token/mod.rs | 1 - .../token/random_prefix_crockford_token.rs | 100 ------------------ crates/service/job/inference_job/Cargo.toml | 2 +- .../src/job/job_loop/clear_full_filesystem.rs | 2 +- .../lipsync/sad_talker/categorize_error.rs | 4 +- .../sad_talker_inference_command.rs | 2 +- .../tacotron2_inference_sidecar_client.rs | 2 +- .../tacotron2_sidecar_health_check_client.rs | 2 +- .../virtual_lfu_cache.rs | 2 +- .../vall_e_x/vall_e_x_inference_command.rs | 8 +- .../vc/rvc_v2/rvc_v2_inference_command.rs | 2 +- .../so_vits_svc_inference_command.rs | 2 +- crates/service/job/inference_job/src/main.rs | 2 +- .../ffmpeg_logo_watermark_command.rs | 2 +- .../service/job/tts_inference_job/Cargo.toml | 1 + .../service/job/tts_inference_job/src/main.rs | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 1 + .../event_handlers/bits_event_handler.rs | 2 +- .../channel_points_event_handler.rs | 2 +- 27 files changed, 85 insertions(+), 125 deletions(-) create mode 100644 crates/lib/collections/Cargo.toml create mode 100644 crates/lib/collections/src/lib.rs rename crates/lib/{deprecated/container_common/src/collections => collections/src}/multiple_random_from_vec.rs (95%) rename crates/lib/{deprecated/container_common/src/collections => collections/src}/random_from_vec.rs (92%) delete mode 100644 crates/lib/deprecated/container_common/src/collections/mod.rs delete mode 100644 crates/lib/deprecated/container_common/src/token/random_prefix_crockford_token.rs diff --git a/Cargo.lock b/Cargo.lock index e23d4f99cd..38e238f7d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,6 +1169,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "collections" +version = "0.0.1" +dependencies = [ + "errors", + "my-workspace-hack", + "rand 0.8.5", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -3015,9 +3024,9 @@ dependencies = [ "buckets", "chrono", "cloud_storage", + "collections", "concurrency", "config", - "container_common", "dotenv", "easyenv", "enums", @@ -6686,6 +6695,7 @@ dependencies = [ "chrono", "clap 2.34.0", "cloud_storage", + "collections", "config", "container_common", "dotenv", @@ -6758,6 +6768,7 @@ dependencies = [ "anyhow", "base64 0.21.5", "chrono", + "collections", "config", "container_common", "diesel", diff --git a/Cargo.toml b/Cargo.toml index 7bf961b20f..4729fc4d0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/lib/bootstrap", "crates/lib/caching/memory_caching", "crates/lib/caching/redis_caching", + "crates/lib/collections", "crates/lib/concurrency", "crates/lib/cookies", "crates/lib/crockford", diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml new file mode 100644 index 0000000000..c6f7ca1452 --- /dev/null +++ b/crates/lib/collections/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "collections" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +# NB: Public "collections" crate is not yet taken (!!), so we're fine to use this internally for now. +name = "collections" +path = "src/lib.rs" + +[dependencies] +# Internal +errors = { path = "../errors" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + +# External +# KEEP THIS TO A MINIMUM OR NIL. +rand = "0.8.3" + +[dev-dependencies] +# None yet diff --git a/crates/lib/collections/src/lib.rs b/crates/lib/collections/src/lib.rs new file mode 100644 index 0000000000..a103362742 --- /dev/null +++ b/crates/lib/collections/src/lib.rs @@ -0,0 +1,22 @@ +//! collections +//! +//! The purpose of this library is to provide "extra" data structures or utilities related to +//! data structures. There should be minimal dependency introduction here. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod multiple_random_from_vec; +pub mod random_from_vec; diff --git a/crates/lib/deprecated/container_common/src/collections/multiple_random_from_vec.rs b/crates/lib/collections/src/multiple_random_from_vec.rs similarity index 95% rename from crates/lib/deprecated/container_common/src/collections/multiple_random_from_vec.rs rename to crates/lib/collections/src/multiple_random_from_vec.rs index 3318b9ee2d..7c1b0f99d8 100644 --- a/crates/lib/deprecated/container_common/src/collections/multiple_random_from_vec.rs +++ b/crates/lib/collections/src/multiple_random_from_vec.rs @@ -12,7 +12,7 @@ pub fn multiple_random_from_vec(vec: &Vec, amount: usize) -> Vec<&T> { #[cfg(test)] mod tests { - use crate::collections::multiple_random_from_vec::multiple_random_from_vec; + use collections::multiple_random_from_vec::multiple_random_from_vec; #[test] fn pulls_item_from_vec() { diff --git a/crates/lib/deprecated/container_common/src/collections/random_from_vec.rs b/crates/lib/collections/src/random_from_vec.rs similarity index 92% rename from crates/lib/deprecated/container_common/src/collections/random_from_vec.rs rename to crates/lib/collections/src/random_from_vec.rs index 652da76e83..73a88a63ff 100644 --- a/crates/lib/deprecated/container_common/src/collections/random_from_vec.rs +++ b/crates/lib/collections/src/random_from_vec.rs @@ -7,7 +7,7 @@ pub fn random_from_vec(vec: &Vec) -> Option<&T> { #[cfg(test)] mod tests { - use crate::collections::random_from_vec::random_from_vec; + use collections::random_from_vec::random_from_vec; #[test] fn pulls_item_from_vec() { diff --git a/crates/lib/deprecated/container_common/src/collections/mod.rs b/crates/lib/deprecated/container_common/src/collections/mod.rs deleted file mode 100644 index 62cd192f83..0000000000 --- a/crates/lib/deprecated/container_common/src/collections/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod multiple_random_from_vec; -pub mod random_from_vec; \ No newline at end of file diff --git a/crates/lib/deprecated/container_common/src/lib.rs b/crates/lib/deprecated/container_common/src/lib.rs index 49c1946532..99dae25ed9 100644 --- a/crates/lib/deprecated/container_common/src/lib.rs +++ b/crates/lib/deprecated/container_common/src/lib.rs @@ -1,5 +1,4 @@ pub mod anyhow_result; -pub mod collections; pub mod files; pub mod i18n; pub mod numerics; diff --git a/crates/lib/deprecated/container_common/src/token/mod.rs b/crates/lib/deprecated/container_common/src/token/mod.rs index 023a5ee511..d523ef2bb0 100644 --- a/crates/lib/deprecated/container_common/src/token/mod.rs +++ b/crates/lib/deprecated/container_common/src/token/mod.rs @@ -1,3 +1,2 @@ pub mod random_crockford_token; -pub mod random_prefix_crockford_token; pub mod random_uuid; \ No newline at end of file diff --git a/crates/lib/deprecated/container_common/src/token/random_prefix_crockford_token.rs b/crates/lib/deprecated/container_common/src/token/random_prefix_crockford_token.rs deleted file mode 100644 index c03cd5fe6c..0000000000 --- a/crates/lib/deprecated/container_common/src/token/random_prefix_crockford_token.rs +++ /dev/null @@ -1,100 +0,0 @@ -use anyhow::anyhow; -use rand::Rng; - -use crate::anyhow_result::AnyhowResult; - -// Crockford characters -const CROCKFORD_UPPERCASE_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; - -const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz"; - -// Random part can't be any less than this. -const MIN_ENTROPY_LENGTH : usize = 8; - -#[deprecated(note = "see internal 'crockford' and 'token' crates")] -pub fn random_prefix_crockford_token(prefix: &str, length: usize, uppercase: bool) -> AnyhowResult { - let mut prefix = prefix.trim().to_string(); - - if prefix.is_empty() { - return Err(anyhow!("prefix not set; use other crockford functions")); - } - - if !prefix.ends_with(":") { - prefix = format!("{}:", prefix); - } - - let entropy_length : usize = length.saturating_sub(prefix.len()); - - if entropy_length < MIN_ENTROPY_LENGTH { - return Err(anyhow!("not enough entropy in token")); - } - - let mut rng = rand::thread_rng(); - - let charset = if uppercase { CROCKFORD_UPPERCASE_CHARSET } else { CROCKFORD_LOWERCASE_CHARSET }; - - let entropy_part: String = (0..entropy_length) - .map(|_| { - let idx = rng.gen_range(0..charset.len()); - charset[idx] as char - }) - .collect(); - - let token = format!("{}{}", prefix, entropy_part); - Ok(token) -} - -#[cfg(test)] -mod tests { - use crate::token::random_prefix_crockford_token::random_prefix_crockford_token; - - #[test] - fn no_prefix_error() { - assert!(random_prefix_crockford_token("", 0, false).is_err()); - assert!(random_prefix_crockford_token("", 10, false).is_err()); - assert!(random_prefix_crockford_token("", 32, false).is_err()); - assert!(random_prefix_crockford_token("", 0, true).is_err()); - assert!(random_prefix_crockford_token("", 10, true).is_err()); - assert!(random_prefix_crockford_token("", 32, true).is_err()); - } - - #[test] - fn not_enough_entropy_error() { - assert!(random_prefix_crockford_token("foo", 0, false).is_err()); - assert!(random_prefix_crockford_token("foo", 5, false).is_err()); - assert!(random_prefix_crockford_token("foo", 11, false).is_err()); - assert!(random_prefix_crockford_token("foo:", 11, false).is_err()); - assert!(random_prefix_crockford_token("foo", 0, true).is_err()); - assert!(random_prefix_crockford_token("foo", 5, true).is_err()); - assert!(random_prefix_crockford_token("foo", 11, true).is_err()); - assert!(random_prefix_crockford_token("foo:", 11, true).is_err()); - } - - #[test] - fn starts_with() { - assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("prefix", 16, false).unwrap().starts_with("prefix:")); - assert!(random_prefix_crockford_token("prefix:", 16, false).unwrap().starts_with("prefix:")); - assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("prefix", 16, true).unwrap().starts_with("prefix:")); - assert!(random_prefix_crockford_token("prefix:", 16, true).unwrap().starts_with("prefix:")); - } - - #[test] - fn length() { - assert_eq!(random_prefix_crockford_token("w", 10, false).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("w:", 10, false).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("prefix", 16, false).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("prefix:", 16, false).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("asdf:", 32, false).unwrap().len(), 32); - assert_eq!(random_prefix_crockford_token("asdf", 32, false).unwrap().len(), 32); - assert_eq!(random_prefix_crockford_token("w", 10, true).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("w:", 10, true).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("prefix", 16, true).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("prefix:", 16, true).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("asdf:", 32, true).unwrap().len(), 32); - assert_eq!(random_prefix_crockford_token("asdf", 32, true).unwrap().len(), 32); - } -} diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 20c39c490c..36194c73a9 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -19,9 +19,9 @@ actix_helpers = { path = "../../../lib/actix_helpers" } bootstrap = { path = "../../../lib/bootstrap" } buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } +collections = { path = "../../../lib/collections" } concurrency = { path = "../../../lib/concurrency" } config = { path = "../../../lib/deprecated/config" } -container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } enums = { path = "../../../_types/enums" } errors = { path = "../../../lib/errors" } diff --git a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs index 13b779e945..24c3ea7ee9 100644 --- a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs +++ b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs @@ -1,6 +1,6 @@ use log::warn; -use container_common::collections::multiple_random_from_vec::multiple_random_from_vec; +use collections::multiple_random_from_vec::multiple_random_from_vec; use errors::AnyhowResult; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/categorize_error.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/categorize_error.rs index 6bad85f61c..e4688dc863 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/categorize_error.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/categorize_error.rs @@ -13,7 +13,9 @@ pub fn categorize_error(stderr_contents: &str) -> Option #[cfg(test)] mod tests { use anyhow::bail; - use container_common::anyhow_result::AnyhowResult; + + use errors::AnyhowResult; + use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_inference_command.rs index 55b25cea9d..05c6ab4a6b 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_inference_command.rs @@ -10,7 +10,7 @@ use log::info; use once_cell::sync::Lazy; use subprocess::{Popen, PopenConfig, Redirection}; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs index 64ae25df90..2a94dc7a0e 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs @@ -5,7 +5,7 @@ use hyper::{Body, Request}; use hyper::client::Client; use log::info; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use mysql_queries::column_types::vocoder_type::VocoderType; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_sidecar_health_check_client.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_sidecar_health_check_client.rs index ade8c79e8f..eaaec22dd0 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_sidecar_health_check_client.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_sidecar_health_check_client.rs @@ -4,7 +4,7 @@ use hyper::client::Client; use hyper::Uri; use log::info; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; // TODO(bt): As more microservices adopt a common health check protocol, // it may make sense to make this a common library. diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/virtual_lfu_cache.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/virtual_lfu_cache.rs index fa9171b3e6..1bdb360c7b 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/virtual_lfu_cache.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/virtual_lfu_cache.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, RwLock}; use anyhow::anyhow; use lfu::LFUCache; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; /// This stands in front of the python HTTP sidecar and controls which /// models get to remain in memory. The Python sidecar keeps multiple diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs index 7d7aa8aa4c..09721a2a22 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -2,17 +2,17 @@ use std::collections::HashSet; use std::env; use std::ffi::OsString; use std::fs::File; -use std::path::{ Path, PathBuf }; +use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::anyhow; use log::info; use once_cell::sync::Lazy; -use subprocess::{ Popen, PopenConfig, Redirection }; +use subprocess::{Popen, PopenConfig, Redirection}; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; -use subprocess_common::docker_options::{ DockerFilesystemMount, DockerGpu, DockerOptions }; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; use crate::job::job_loop::command_exit_status::CommandExitStatus; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs index 34a04a226f..bb3be2e01f 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs @@ -9,7 +9,7 @@ use log::info; use once_cell::sync::Lazy; use subprocess::{Popen, PopenConfig}; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob; use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs index daa44a3666..e99384933c 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs @@ -9,7 +9,7 @@ use log::info; use once_cell::sync::Lazy; use subprocess::{Popen, PopenConfig}; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions}; diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index e289e14d77..7779535799 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -32,7 +32,7 @@ use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::check_directory_exists::check_directory_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; diff --git a/crates/service/job/inference_job/src/util/common_commands/ffmpeg_logo_watermark_command.rs b/crates/service/job/inference_job/src/util/common_commands/ffmpeg_logo_watermark_command.rs index a2a60daec6..2718b05866 100644 --- a/crates/service/job/inference_job/src/util/common_commands/ffmpeg_logo_watermark_command.rs +++ b/crates/service/job/inference_job/src/util/common_commands/ffmpeg_logo_watermark_command.rs @@ -7,7 +7,7 @@ use log::{info, warn}; use once_cell::sync::Lazy; use subprocess::{Popen, PopenConfig}; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use crate::job::job_loop::command_exit_status::CommandExitStatus; diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 204902744f..846405c909 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" # Internal cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } +collections = { path = "../../../lib/collections" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index 7244d4057c..9738caa13c 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -27,11 +27,11 @@ use sqlx::MySqlPool; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use collections::multiple_random_from_vec::multiple_random_from_vec; use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use container_common::collections::multiple_random_from_vec::multiple_random_from_vec; use filesys::check_directory_exists::check_directory_exists; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder; diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index d4149fd0e6..905c4b0081 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] # Internal +collections = { path = "../../../lib/collections" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } diff --git a/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index 5530844919..79df12b766 100644 --- a/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -6,8 +6,8 @@ use log::info; use sqlx::MySql; use twitch_api2::pubsub::channel_bits::{BitsEventData, ChannelBitsEventsV2, ChannelBitsEventsV2Reply}; +use collections::random_from_vec::random_from_vec; use container_common::anyhow_result::AnyhowResult; -use container_common::collections::random_from_vec::random_from_vec; use container_common::numerics::signed_to_unsigned::i64_to_unsigned_zeroing_negatives; use mysql_queries::column_types::twitch_event_category::TwitchEventCategory; use mysql_queries::complex_models::event_match_predicate::EventMatchPredicate; diff --git a/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index 565b61599c..fdbde78357 100644 --- a/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/other/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -6,8 +6,8 @@ use log::info; use sqlx::MySql; use twitch_api2::pubsub::channel_points::{ChannelPointsChannelV1, ChannelPointsChannelV1Reply, Redemption}; +use collections::random_from_vec::random_from_vec; use container_common::anyhow_result::AnyhowResult; -use container_common::collections::random_from_vec::random_from_vec; use mysql_queries::column_types::twitch_event_category::TwitchEventCategory; use mysql_queries::complex_models::event_match_predicate::EventMatchPredicate; use mysql_queries::complex_models::event_responses::EventResponse; From 12ef105ced518796b306b644030a3a3c359c3c4c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:27:08 -0500 Subject: [PATCH 033/136] fix errors --- crates/lib/collections/src/multiple_random_from_vec.rs | 2 +- crates/lib/collections/src/random_from_vec.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/lib/collections/src/multiple_random_from_vec.rs b/crates/lib/collections/src/multiple_random_from_vec.rs index 7c1b0f99d8..81bc05ca49 100644 --- a/crates/lib/collections/src/multiple_random_from_vec.rs +++ b/crates/lib/collections/src/multiple_random_from_vec.rs @@ -12,7 +12,7 @@ pub fn multiple_random_from_vec(vec: &Vec, amount: usize) -> Vec<&T> { #[cfg(test)] mod tests { - use collections::multiple_random_from_vec::multiple_random_from_vec; + use crate::multiple_random_from_vec::multiple_random_from_vec; #[test] fn pulls_item_from_vec() { diff --git a/crates/lib/collections/src/random_from_vec.rs b/crates/lib/collections/src/random_from_vec.rs index 73a88a63ff..f67f0aa64e 100644 --- a/crates/lib/collections/src/random_from_vec.rs +++ b/crates/lib/collections/src/random_from_vec.rs @@ -7,7 +7,7 @@ pub fn random_from_vec(vec: &Vec) -> Option<&T> { #[cfg(test)] mod tests { - use collections::random_from_vec::random_from_vec; + use crate::random_from_vec::random_from_vec; #[test] fn pulls_item_from_vec() { From ccae943037340463d771846fa84f807299088ab1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 07:32:36 +0000 Subject: [PATCH 034/136] Bump rand from 0.7.3 to 0.8.5 Bumps [rand](https://github.com/rust-random/rand) from 0.7.3 to 0.8.5. - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.7.3...0.8.5) --- updated-dependencies: - dependency-name: rand dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 2 +- crates/_types/tokens/Cargo.toml | 2 +- crates/cli/kubectl-pod-cleanup/Cargo.toml | 2 +- crates/lib/collections/Cargo.toml | 2 +- crates/lib/crockford/Cargo.toml | 2 +- crates/lib/deprecated/container_common/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- crates/service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38e238f7d5..b9fef2a7fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3852,7 +3852,7 @@ dependencies = [ "hyper-tls", "log", "my-workspace-hack", - "rand 0.7.3", + "rand 0.8.5", "serde", "serde_json", "tokio", diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index 9f242e6ee7..c688d59a69 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -18,7 +18,7 @@ serde = "1.0.125" serde_derive = "1.0.125" # Random -rand = "0.8.3" +rand = "0.8.5" # MySql Database # "offline" supports typesafe builds without a live database diff --git a/crates/cli/kubectl-pod-cleanup/Cargo.toml b/crates/cli/kubectl-pod-cleanup/Cargo.toml index a4595271a7..78beb7813b 100644 --- a/crates/cli/kubectl-pod-cleanup/Cargo.toml +++ b/crates/cli/kubectl-pod-cleanup/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../lib/errors" } # External log = "0.4.20" once_cell = "1.18.0" -rand = "0.8.1" +rand = "0.8.5" regex = "1.10.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml index c6f7ca1452..a31edbb379 100644 --- a/crates/lib/collections/Cargo.toml +++ b/crates/lib/collections/Cargo.toml @@ -20,7 +20,7 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External # KEEP THIS TO A MINIMUM OR NIL. -rand = "0.8.3" +rand = "0.8.5" [dev-dependencies] # None yet diff --git a/crates/lib/crockford/Cargo.toml b/crates/lib/crockford/Cargo.toml index ca5afa1a35..e66d821a32 100644 --- a/crates/lib/crockford/Cargo.toml +++ b/crates/lib/crockford/Cargo.toml @@ -15,7 +15,7 @@ name = "crockford" path = "src/lib.rs" [dependencies] -rand = "0.8.3" +rand = "0.8.5" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/deprecated/container_common/Cargo.toml b/crates/lib/deprecated/container_common/Cargo.toml index 3a15d81dd1..ea97b1fc3d 100644 --- a/crates/lib/deprecated/container_common/Cargo.toml +++ b/crates/lib/deprecated/container_common/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" # External anyhow = "1.0.75" once_cell = "1.18.0" -rand = "0.8.3" +rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } log = "0.4.14" diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 66509f8eb6..30640d7446 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -30,7 +30,7 @@ chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" -rand = "0.8.3" +rand = "0.8.5" # MySql Database # "offline" supports typesafe builds without a live database diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 905c4b0081..8de44b6db1 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -33,7 +33,7 @@ dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" log = "0.4.14" -rand = "0.8.3" +rand = "0.8.5" regex = "1.5.4" time = "0.3.30" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 68d3c2122d..bee1b60deb 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -39,7 +39,7 @@ dotenv = "0.15.0" lazy_static = "1.4.0" # Regex caching (TODO: Use OnceCell instead) log = "0.4.14" once_cell = "1.18.0" -rand = "0.8.3" +rand = "0.8.5" regex = "1.5.4" time = "0.3.30" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 63da7fddf6..50328fb0c6 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -63,7 +63,7 @@ lazy_static = "1.4.0" lexical-sort = "0.3.1" log = "0.4.14" once_cell = "1.18.0" -rand = "0.8.3" +rand = "0.8.5" regex = "1.5.4" #symphonia = { version = "0.5.2", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } # Opus support still hasn't landed in Symphonia, so we have to use an older version (~0.5) off-master. diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index c3f089f7c6..f1202bcdd1 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -33,7 +33,7 @@ dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" log = "0.4.14" -rand = "0.8.3" +rand = "0.8.5" regex = "1.5.4" time = "0.3.30" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index ba2dbf7e63..3982508ec4 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -24,7 +24,7 @@ uuid = { version = "0.8.1", features = ["v4"] } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] -rand = "0.7.3" +rand = "0.8.5" env_logger = "0.7.1" futures = { version = "0.3.1" } hyper = { version = "0.14", features = ["http1", "server"] } From ff4a3115d285f62c586559a6930070e8dd37b441 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:34:33 -0500 Subject: [PATCH 035/136] clear up more directories when file system is full --- .../src/job/job_loop/clear_full_filesystem.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs index 24c3ea7ee9..b38be72157 100644 --- a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs +++ b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use log::warn; use collections::multiple_random_from_vec::multiple_random_from_vec; @@ -5,12 +7,18 @@ use errors::AnyhowResult; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; pub fn clear_full_filesystem(cache_dir: &SemiPersistentCacheDir) -> AnyhowResult<()> { - warn!("Deleting cached VC models to free up disk space."); + warn!("Deleting cached models to free up disk space."); - let model_dir = cache_dir.voice_conversion_model_directory(); + decimate_directory(cache_dir.custom_vocoder_model_directory())?; + decimate_directory(cache_dir.tts_synthesizer_model_directory())?; + decimate_directory(cache_dir.voice_conversion_model_directory())?; + Ok(()) +} + +fn decimate_directory(path: &Path) -> AnyhowResult<()> { // TODO: When this is no longer sufficient, delete other types of locally-cached data. - let paths = std::fs::read_dir(model_dir)? + let paths = std::fs::read_dir(path)? .map(|res| res.map(|e| e.path())) .collect::, std::io::Error>>()?; @@ -21,7 +29,7 @@ pub fn clear_full_filesystem(cache_dir: &SemiPersistentCacheDir) -> AnyhowResult for model_to_delete in models_to_delete { warn!("Deleting cached model file: {:?}", model_to_delete); - let full_model_path = cache_dir.voice_conversion_model_path(model_to_delete); + let full_model_path = path.join(model_to_delete); std::fs::remove_file(full_model_path)?; } From 18ad28135f9f269cab9457500a6164474facb1de Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 02:36:33 -0500 Subject: [PATCH 036/136] safe delete --- .../inference_job/src/job/job_loop/clear_full_filesystem.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs index b38be72157..a039ea5ddc 100644 --- a/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs +++ b/crates/service/job/inference_job/src/job/job_loop/clear_full_filesystem.rs @@ -4,6 +4,7 @@ use log::warn; use collections::multiple_random_from_vec::multiple_random_from_vec; use errors::AnyhowResult; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; pub fn clear_full_filesystem(cache_dir: &SemiPersistentCacheDir) -> AnyhowResult<()> { @@ -30,7 +31,7 @@ fn decimate_directory(path: &Path) -> AnyhowResult<()> { warn!("Deleting cached model file: {:?}", model_to_delete); let full_model_path = path.join(model_to_delete); - std::fs::remove_file(full_model_path)?; + safe_delete_temp_file(full_model_path); } Ok(()) From 456b9e1ce63b935361038a9c12bf6b335d9dd653 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 09:35:18 +0000 Subject: [PATCH 037/136] Bump env_logger from 0.7.1 to 0.10.1 Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.7.1 to 0.10.1. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.7.1...v0.10.1) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 36 +++---------------- crates/lib/easyenv/Cargo.toml | 2 +- crates/lib/logging/Cargo.toml | 2 +- .../newrelic-telemetry-sdk-rust/Cargo.toml | 2 +- 4 files changed, 7 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9fef2a7fa..ae154198df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1994,7 +1994,7 @@ name = "easyenv" version = "0.1.0" dependencies = [ "dotenv", - "env_logger 0.10.1", + "env_logger", "log", "my-workspace-hack", "storyteller_root", @@ -2175,26 +2175,13 @@ dependencies = [ "toml", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ - "humantime 2.1.0", + "humantime", "is-terminal", "log", "regex", @@ -2840,15 +2827,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "humantime" version = "2.1.0" @@ -3392,7 +3370,7 @@ name = "logging" version = "0.0.1" dependencies = [ "dotenv", - "env_logger 0.10.1", + "env_logger", "log", "my-workspace-hack", ] @@ -3845,7 +3823,7 @@ name = "newrelic-telemetry" version = "0.1.0" dependencies = [ "anyhow", - "env_logger 0.7.1", + "env_logger", "flate2", "futures 0.3.29", "hyper", @@ -4454,12 +4432,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quick-xml" version = "0.26.0" diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index a691303c16..3af179ed9e 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -18,7 +18,7 @@ storyteller_root = { path = "../storyteller_root" } # External dotenv = "0.15.0" -env_logger = "0.10.0" +env_logger = "0.10.1" log = "0.4.14" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index 88cd766872..d5db48f134 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -15,7 +15,7 @@ path = "src/lib.rs" [dependencies] dotenv = "0.15.0" -env_logger = "0.10.0" +env_logger = "0.10.1" log = "0.4.20" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 3982508ec4..2e8dfb8c5e 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -25,7 +25,7 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] rand = "0.8.5" -env_logger = "0.7.1" +env_logger = "0.10.1" futures = { version = "0.3.1" } hyper = { version = "0.14", features = ["http1", "server"] } tokio = { version = "1.1.0", features = ["macros", "rt-multi-thread"] } From b628fb0a11c3838fa85981f6481a18b56b4c7016 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 15:20:50 -0500 Subject: [PATCH 038/136] premium TTS duration --- .../web/insert_generic_inference_job.rs | 9 +++++++++ .../endpoints/animation/enqueue_lipsync_animation.rs | 1 + .../enqueue_infer_tts_handler.rs | 5 ++++- .../inference/enqueue_voice_conversion_inference.rs | 1 + .../voice_designer/inference/enqueue_tts_request.rs | 1 + .../endpoints/voice_designer/voices/create_voice.rs | 2 ++ 6 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs index 6c51c5a693..3f54c5306a 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs @@ -24,6 +24,8 @@ pub struct InsertGenericInferenceArgs<'a> { pub maybe_raw_inference_text: Option<&'a str>, + pub maybe_max_duration_seconds: Option, + pub maybe_inference_args: Option, pub maybe_creator_user_token: Option<&'a UserToken>, @@ -54,6 +56,11 @@ pub async fn insert_generic_inference_job(args: InsertGenericInferenceArgs<'_>) routing_tag }); + // This only applies to certain types of inference. + // "0" is the default value, typically 12 seconds for TTS. + // "-1" means "unlimited" + let max_duration_seconds = args.maybe_max_duration_seconds.unwrap_or(0); + let query = sqlx::query!( r#" INSERT INTO generic_inference_jobs @@ -80,6 +87,7 @@ SET priority_level = ?, is_keepalive_required = ?, + max_duration_seconds = ?, is_debug_request = ?, maybe_routing_tag = ?, @@ -109,6 +117,7 @@ SET args.priority_level, args.requires_keepalive, + max_duration_seconds, args.is_debug_request, maybe_routing_tag, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs index bf44d619be..8d1e536dcc 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -309,6 +309,7 @@ pub async fn enqueue_lipsync_animation_handler( maybe_input_source_token: None, // TODO: Introduce a second foreign key ? maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? maybe_raw_inference_text: None, // No text + maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation), args: Some(PolymorphicInferenceArgs::La(inference_args)), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 61aa92b715..b01dd37a5d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -375,6 +375,8 @@ pub async fn enqueue_infer_tts_handler( TtsModelType::Vits => InferenceModelType::Vits, }; + let max_duration_seconds = plan.tts_max_duration_seconds(); + if use_new_job_system { // This branch uses the `generic-inference-job` service and tables. info!("Creating tts inference job record (new generic job system)..."); @@ -393,6 +395,7 @@ pub async fn enqueue_infer_tts_handler( maybe_input_source_token: None, // NB: TTS doesn't have input media maybe_input_source_token_type: None, // NB: TTS doesn't have input media maybe_raw_inference_text: Some(inference_text.as_str()), + maybe_max_duration_seconds: Some(max_duration_seconds), maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), args: None, // NB: We don't need to encode any args yet. @@ -435,7 +438,7 @@ pub async fn enqueue_infer_tts_handler( .set_creator_ip_address(&ip_address) .set_creator_set_visibility(set_visibility.to_str()) .set_priority_level(priority_level) - .set_max_duration_seconds(plan.tts_max_duration_seconds()) + .set_max_duration_seconds(max_duration_seconds) .set_is_from_api(is_from_api) .set_is_debug_request(is_debug_request) .insert(&server_state.mysql_pool) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index d8e97c102e..22131729c0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -297,6 +297,7 @@ pub async fn enqueue_voice_conversion_inference_handler( maybe_input_source_token: Some(media_token.as_str()), maybe_input_source_token_type: Some(InferenceInputSourceTokenType::MediaUpload), maybe_raw_inference_text: None, // NB: Voice conversion isn't TTS, so there's no text. + maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), args: maybe_args, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index de07d8083c..d394edabcd 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -177,6 +177,7 @@ pub async fn enqueue_tts_request( maybe_input_source_token: None, maybe_input_source_token_type: None, maybe_raw_inference_text: Some(&request.text), + maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), args: Some(PolymorphicInferenceArgs::Tts(inference_args)), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs index d2cef170f4..b47de2c732 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs @@ -164,6 +164,7 @@ pub async fn create_voice_handler( voice_token: None, dataset_token: Some(request.voice_dataset_token.clone()), }; + let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); // create the inference args here @@ -177,6 +178,7 @@ pub async fn create_voice_handler( maybe_input_source_token: None, maybe_input_source_token_type: None, maybe_raw_inference_text: None, + maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), args: Some(PolymorphicInferenceArgs::Tts(inference_args)), From 6d27214a148d7500c6313a3ff8b0e86298af8649 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 15:50:05 -0500 Subject: [PATCH 039/136] codegen --- ...84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405.json} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .sqlx/{query-b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7.json => query-0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405.json} (74%) diff --git a/.sqlx/query-b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7.json b/.sqlx/query-0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405.json similarity index 74% rename from .sqlx/query-b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7.json rename to .sqlx/query-0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405.json index 519668f709..8f51881a62 100644 --- a/.sqlx/query-b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7.json +++ b/.sqlx/query-0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405.json @@ -1,12 +1,12 @@ { "db_name": "MySQL", - "query": "\nINSERT INTO generic_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n inference_category = ?,\n maybe_model_type = ?,\n maybe_model_token = ?,\n\n maybe_input_source_token = ?,\n maybe_input_source_token_type = ?,\n\n maybe_raw_inference_text = ?,\n\n maybe_inference_args = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n priority_level = ?,\n is_keepalive_required = ?,\n\n is_debug_request = ?,\n maybe_routing_tag = ?,\n\n status = \"pending\"\n ", + "query": "\nINSERT INTO generic_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n inference_category = ?,\n maybe_model_type = ?,\n maybe_model_token = ?,\n\n maybe_input_source_token = ?,\n maybe_input_source_token_type = ?,\n\n maybe_raw_inference_text = ?,\n\n maybe_inference_args = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n priority_level = ?,\n is_keepalive_required = ?,\n max_duration_seconds = ?,\n\n is_debug_request = ?,\n maybe_routing_tag = ?,\n\n status = \"pending\"\n ", "describe": { "columns": [], "parameters": { - "Right": 17 + "Right": 18 }, "nullable": [] }, - "hash": "b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7" + "hash": "0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405" } From 4837106d3713cfa741d73738a733a7d6977f0de3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 16:10:01 -0500 Subject: [PATCH 040/136] distinguish logs --- .../service/job/inference_job/src/job/job_loop/main_loop.rs | 5 +++++ .../job/inference_job/src/job/job_loop/process_single_job.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 207a63051d..29bb8d8f7c 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -108,6 +108,9 @@ pub async fn main_loop(job_dependencies: JobDependencies) { async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { let result = process_single_job(job_dependencies, &job).await; + + println!("\n ----------------------------------------- RESULTS SUMMARY ----------------------------------------- \n"); + match result { Ok(success_case) => { info!("Job loop iteration \"success\": {:?}", success_case); @@ -129,6 +132,8 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec Date: Thu, 30 Nov 2023 16:27:25 -0500 Subject: [PATCH 041/136] cleanup logging --- .../job/inference_job/src/job/job_loop/main_loop.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 29bb8d8f7c..64a0ac038f 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -109,8 +109,6 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { info!("Job loop iteration \"success\": {:?}", success_case); @@ -128,12 +126,14 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { + println!("\n ----------------------------------------- FAILURE SUMMARY ----------------------------------------- \n"); + warn!("Failure to process job: {:?} - {:?}",job.inference_job_token, err); let _r = handle_error(&job_dependencies, &job, err).await?; + + println!("\n ----------------------------------------- FAILURE SUMMARY END ----------------------------------------- \n"); } } - - println!("\n ----------------------------------------- RESULTS SUMMARY END ----------------------------------------- \n"); } Ok(()) From b0a87e83a0ce16352616a1acfc77c70e6af00197 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 17:19:23 -0500 Subject: [PATCH 042/136] log batch index --- .../inference_job/src/job/job_loop/main_loop.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 64a0ac038f..d9f26b7995 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -106,12 +106,13 @@ pub async fn main_loop(job_dependencies: JobDependencies) { // This probably returns a struct or enum with some measure of how many GB need to be downloaded. async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec) -> AnyhowResult<()> { - for job in jobs.into_iter() { + let job_count = jobs.len(); + for (i, job) in jobs.into_iter().enumerate() { let result = process_single_job(job_dependencies, &job).await; match result { Ok(success_case) => { - info!("Job loop iteration \"success\": {:?}", success_case); + info!("Job loop iteration ({i} of {job_count} batch) \"success\": {:?}", success_case); let increment_success_count = match success_case { ProcessSingleJobSuccessCase::JobCompleted => true, @@ -126,12 +127,11 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { - println!("\n ----------------------------------------- FAILURE SUMMARY ----------------------------------------- \n"); - - warn!("Failure to process job: {:?} - {:?}",job.inference_job_token, err); + error!( + r#"Failure to process job ({i} of {job_count} batch): {:?} - + {:?} + "#,job.inference_job_token, err); let _r = handle_error(&job_dependencies, &job, err).await?; - - println!("\n ----------------------------------------- FAILURE SUMMARY END ----------------------------------------- \n"); } } } From 3cdce86162d6c90b1a24ce4f15d2c7b21033b7b2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 17:25:23 -0500 Subject: [PATCH 043/136] warn on slow batch queries --- .../job/inference_job/src/job/job_loop/main_loop.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index d9f26b7995..a1e11adb6b 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::time::{Duration, Instant}; use log::{error, info, warn}; @@ -23,6 +23,9 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; /// Pause file millis const PAUSE_FILE_EXISTS_WAIT_MILLIS : u64 = 1000 * 30; +/// Warn on slow batch queries +const SLOW_BATCH_QUERY_NOTICE_DURATION : Duration = Duration::from_millis(3500); + pub async fn main_loop(job_dependencies: JobDependencies) { let mut noop_logger = NoOpLogger::new(job_dependencies.job.system.no_op_logger_millis as i64); @@ -46,6 +49,8 @@ pub async fn main_loop(job_dependencies: JobDependencies) { let maybe_scoped_model_types = job_dependencies.job.system.scoped_execution.get_scoped_model_types(); + let batch_query_start_time = Instant::now(); + let maybe_available_jobs = list_available_generic_inference_jobs(ListAvailableGenericInferenceJobArgs { num_records: job_dependencies.job.system.job_batch_size, is_debug_worker: false, // TODO @@ -55,6 +60,12 @@ pub async fn main_loop(job_dependencies: JobDependencies) { mysql_pool: &job_dependencies.db.mysql_pool, }).await; + let batch_query_duration = Instant::now().duration_since(batch_query_start_time); + + if batch_query_duration > SLOW_BATCH_QUERY_NOTICE_DURATION { + warn!("Batch query took duration: {:?}", &batch_query_duration); + } + sort_by_priority = true; sort_by_priority_count += 1; From 72fdd75568cfe551b91351fb530decfd33152c10 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 17:33:25 -0500 Subject: [PATCH 044/136] added upload weights / create weight endpoint original file for the weights weights file type for safetensors or checkpoint Gotta add one more thing for record creation --- crates/_types/buckets/src/public/mod.rs | 3 +- .../src/public/weight_uploads/directory.rs | 77 ++++ .../buckets/src/public/weight_uploads/mod.rs | 2 + .../public/weight_uploads/original_file.rs | 104 ++++++ .../enums/src/by_table/model_weights/mod.rs | 1 + .../model_weights/weights_file_type.rs | 68 ++++ .../tokens/src/tokens/weight_uploads.rs | 14 + .../endpoints/weights/create_weight.rs | 335 ++++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 3 +- 9 files changed, 605 insertions(+), 2 deletions(-) create mode 100644 crates/_types/buckets/src/public/weight_uploads/directory.rs create mode 100644 crates/_types/buckets/src/public/weight_uploads/mod.rs create mode 100644 crates/_types/buckets/src/public/weight_uploads/original_file.rs create mode 100644 crates/_types/enums/src/by_table/model_weights/weights_file_type.rs create mode 100644 crates/_types/tokens/src/tokens/weight_uploads.rs diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index d85de56720..dc35c40ed4 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,4 +2,5 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod zs_voices; \ No newline at end of file +pub mod zs_voices; +pub mod weight_uploads; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/directory.rs b/crates/_types/buckets/src/public/weight_uploads/directory.rs new file mode 100644 index 0000000000..fcedda62dd --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/directory.rs @@ -0,0 +1,77 @@ +use crockford::crockford_entropy_lower; +// TODO Thoughts NOTE: +// I am wondering if it be easier if we just had a single struct and impl to handle downloads from public and private buckets. +// essentially, the only difference is the prefix and file postfix and the path of where the file goes and the path will be constructed from there. +// then you can tie in the downloader as a single object that can be used for both public and private buckets. + +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +const WEIGHT_UPLOAD_DIRECTORY : &str = "/weight_upload"; + +/// Directory for user media uploads. +/// Each uploaded file gets its own directory so that we can store the original +/// file alongside re-processed derivatives. +#[derive(Clone)] +pub struct WeightUploadDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for WeightUploadDirectory {} + +impl WeightUploadDirectory { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", WEIGHT_UPLOAD_DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } +} + +#[cfg(test)] +mod tests { + use crate::public::weight_uploads::directory::WeightUploadDirectory; + + #[test] + pub fn generate_new_entropy() { + let directory = WeightUploadDirectory::generate_new(); + assert_eq!(directory.get_object_hash().len(), 32); + } + + #[test] + pub fn get_directory_path_str() { + let directory = WeightUploadDirectory::from_object_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = WeightUploadDirectory::from_object_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/f/o/foo"); + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = WeightUploadDirectory::from_object_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } +} diff --git a/crates/_types/buckets/src/public/weight_uploads/mod.rs b/crates/_types/buckets/src/public/weight_uploads/mod.rs new file mode 100644 index 0000000000..42ede393cf --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/mod.rs @@ -0,0 +1,2 @@ +pub mod directory; +pub mod original_file; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/original_file.rs new file mode 100644 index 0000000000..7f0a40a6b7 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/original_file.rs @@ -0,0 +1,104 @@ +use std::path::PathBuf; + +use crockford::crockford_entropy_lower; + +use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::public_path::PublicPath; + +const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; + +// TODO: Generate these from a macro. + +/// The original user upload file. +/// It may have derivative files (down samples, crops, etc.) that live alongside it. +#[derive(Clone)] +pub struct MediaUploadOriginalFilePath { + directory: MediaUploadDirectory, + full_object_path: String, +} + +impl PublicPath for MediaUploadOriginalFilePath {} + +impl MediaUploadOriginalFilePath { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let directory = MediaUploadDirectory::from_object_hash(hash); + let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); + Self { + directory, + full_object_path, + } + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } + + pub fn to_full_object_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_object_path) + } + + pub fn get_directory(&self) -> &MediaUploadDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename() -> &'static str { + ORIGINAL_FILE_BASENAME + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + + #[test] + pub fn generate_new_entropy() { + let file = MediaUploadOriginalFilePath::generate_new(); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); + } + + #[test] + pub fn to_full_object_pathbuf() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = MediaUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/original_upload.bin"); + } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = MediaUploadOriginalFilePath::from_object_hash(hash); + assert_eq!(file.get_object_hash(), hash); + } +} diff --git a/crates/_types/enums/src/by_table/model_weights/mod.rs b/crates/_types/enums/src/by_table/model_weights/mod.rs index 5217cf739e..0755131487 100644 --- a/crates/_types/enums/src/by_table/model_weights/mod.rs +++ b/crates/_types/enums/src/by_table/model_weights/mod.rs @@ -1,2 +1,3 @@ pub mod weights_types; pub mod weights_category; +pub mod weights_file_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs new file mode 100644 index 0000000000..a7adeb72cd --- /dev/null +++ b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs @@ -0,0 +1,68 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; +// TODO we will need to scan the checkpoints for malicious code. We can't just trust the file extension. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +pub enum WeightsFileType { + #[serde(rename = "checkpoint")] + Checkpoint, + #[serde(rename = "safetensors")] + SafeTensors, +} + +impl WeightsFileType { + pub fn to_str(&self) -> &'static str { + match self { + Self::Checkpoint => "checkpoint", + Self::SafeTensors => "safetensors", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "checkpoint" => Ok(Self::Checkpoint), + "safetensors" => Ok(Self::SafeTensors), + _ => Err(format!("invalid value: {:?}", value)), + } + } + + pub fn all_variants() -> BTreeSet { + BTreeSet::from([ + Self::Checkpoint, + Self::SafeTensors, + ]) + } +} + +impl_enum_display_and_debug_using_to_str!(WeightsFileType); +impl_mysql_enum_coders!(WeightsFileType); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(WeightsFileType::Checkpoint.to_str(), "checkpoint"); + assert_eq!(WeightsFileType::SafeTensors.to_str(), "safetensors"); + } + + #[test] + fn test_from_str() { + assert_eq!(WeightsFileType::from_str("checkpoint").unwrap(), WeightsFileType::Checkpoint); + assert_eq!(WeightsFileType::from_str("safetensors").unwrap(), WeightsFileType::SafeTensors); + assert!(WeightsFileType::from_str("invalid").is_err()); + } + + #[test] + fn test_all_variants() { + let variants = WeightsFileType::all_variants(); + assert_eq!(variants.len(), 2); + assert!(variants.contains(&WeightsFileType::Checkpoint)); + assert!(variants.contains(&WeightsFileType::SafeTensors)); + } +} \ No newline at end of file diff --git a/crates/_types/tokens/src/tokens/weight_uploads.rs b/crates/_types/tokens/src/tokens/weight_uploads.rs new file mode 100644 index 0000000000..cd7c046fc8 --- /dev/null +++ b/crates/_types/tokens/src/tokens/weight_uploads.rs @@ -0,0 +1,14 @@ +use std::fmt::Debug; + +use serde::Deserialize; +use serde::Serialize; + +use crate::prefixes::TokenPrefix; + +/// The primary key for user weight uploads +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[sqlx(transparent)] +pub struct WeightUploadToken(pub String); + +impl_string_token!(WeightUploadToken); +impl_crockford_generator!(WeightUploadToken, 32usize, TokenPrefix::WeightUpload, CrockfordLower); \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index e69de29bb2..1db5217ba1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -0,0 +1,335 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use once_cell::sync::Lazy; + +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{error, info, warn}; + +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; + +#[derive(Serialize)] +pub struct UploadWeightsSuccessResponse { + pub success: bool, + pub upload_token: MediaUploadToken, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + "binary/octet-stream", + ]) +}); + +pub async fn upload_weights_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, +) -> Result { + + let response = handle_weight_upload( + &http_request, + &server_state, + multipart_payload, + &ALLOWED_MIME_TYPES).await?; + + let weight_upload_token = response.to_media_token(); + + let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { + success: true, + upload_token: weight_upload_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} + + +pub enum SuccessCase { + WeightAlreadyUploaded { + existing_upload_token: WeightUploadToken, + }, + WeightSuccessfullyUploaded { + upload_token: WeightUploadToken, + } +} + +impl SuccessCase { + pub fn to_media_token(self) -> WeightUploadToken { + match self { + SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, + SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, + } + } +} + +pub async fn handle_weight_upload( + http_request: &HttpRequest, + server_state: &web::Data>, + mut multipart_payload: Multipart, + allowed_mimetypes: &HashSet<&'static str>, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + UploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + UploadError::ServerError + })?; + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_weights_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + UploadError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token + .ok_or(UploadError::BadInput("no uuid".to_string()))?; + + let maybe_existing_upload = + get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + .await; + + match maybe_existing_upload { + Err(err) => { + error!("Error checking for previous upload: {:?}", err); + return Err(UploadError::ServerError); + } + Ok(Some(upload)) => { + // File already uploaded and frontend didn't protect us. + return Ok(SuccessCase::WeightAlreadyUploaded { + existing_upload_token: upload.token, + }); + } + Ok(None) => { + // Proceed. + } + } + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(UploadError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); + + let maybe_mimetype = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + let bytes = match upload_weights_request.file_bytes { + None => return Err(UploadError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + UploadError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_upload_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !allowed_mimetypes.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // NB: .aiff (audio/aiff) isn't supported by Symphonia: + // It contains uncompressed PCM-encoded audio similar to wav. + // See: https://github.com/pdeljanov/Symphonia/issues/75 + // NB: The following formats are not supported by Symphonia and + // do not have any open issues filed. They may simply be too old: + // - .wma (audio/x-ms-wma) + // - .avi (video/x-msvideo) + media_upload_type = match mimetype { + // Audio + "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + _ => None, + }; + + + let media_upload_type = match media_upload_type { + Some(m) => m, + None => { + warn!("Invalid mimetype: {:?}", maybe_mimetype); + return Err(UploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) + }, + }; + + // probably don't need this + let media_upload_source = match upload_weights_request.media_source { + MediaSource::Unknown => MediaUploadSource::Unknown, + MediaSource::UserFile => MediaUploadSource::File, + MediaSource::UserDeviceApi => MediaUploadSource::DeviceApi, + }; + + // This is fine for now we just don't expose the path! + let public_upload_path = MediaUploadOriginalFilePath::generate_new(); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + // need to upload based on visibility + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + UploadError::ServerError + })?; + + let token = create_weight(CreateModelWeightsArgs { + token: &MediaUploadToken::generate(), + weights_type: media_upload_type, + weights_category: media_upload_category, + title: upload_weights_request.file_name.as_deref(), + maybe_thumbnail_token: None, + description_markdown: None, + description_rendered_html: None, + creator_user_token: maybe_user_token.as_deref(), + creator_ip_address: &ip_address, + creator_set_visibility, + maybe_last_update_user_token: None, + original_download_url: None, + original_filename: upload_weights_request.file_name.as_deref(), + file_size_bytes: file_size_bytes as i32, + file_checksum_sha2: hash, + private_bucket_hash: private_bucket_hash, + maybe_private_bucket_prefix: None, + maybe_private_bucket_extension: None, + cached_user_ratings_total_count: 0, + cached_user_ratings_positive_count: 0, + cached_user_ratings_negative_count: 0, + maybe_cached_user_ratings_ratio: None, + version: 0, + mysql_pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + UploadError::ServerError + })?; + }).await?; + + + + let (token, record_id) = insert_media_upload(Args{ + uuid_idempotency_token: &uuid_idempotency_token, + media_type: media_upload_type, + media_source: media_upload_source, + maybe_original_filename: upload_weights_request.file_name.as_deref(), + original_file_size_bytes: file_size_bytes as u64, + maybe_original_duration_millis: maybe_duration_millis, + maybe_original_mime_type: maybe_mimetype, + maybe_original_audio_encoding: maybe_codec_name.as_deref(), + maybe_original_video_encoding: None, + maybe_original_frame_width: None, // TODO + maybe_original_frame_height: None, // TODO + checksum_sha2: &hash, + public_upload_path: &public_upload_path, + maybe_extra_file_modification_info: None, + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: None, + creator_ip_address: &ip_address, + creator_set_visibility, + mysql_pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + UploadError::ServerError + })?; + + info!("new media upload id: {} token: {:?}", record_id, &token); + + Ok(SuccessCase::WeightSuccessfullyUploaded { + upload_token: token, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index 66601e374d..b9e2bd05c2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -2,4 +2,5 @@ pub mod get_weight; pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; -pub mod list_availible_weights; \ No newline at end of file +pub mod list_availible_weights; +pub mod create_weight; \ No newline at end of file From b74cf7853db6a10abcf6b5353a5f2018bbfb7f32 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Nov 2023 22:46:35 -0500 Subject: [PATCH 045/136] fix web break --- service_cpu.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service_cpu.Dockerfile b/service_cpu.Dockerfile index 063bc574d3..4ddabe5b1e 100644 --- a/service_cpu.Dockerfile +++ b/service_cpu.Dockerfile @@ -178,8 +178,9 @@ COPY --from=builder /tmp/target/release/tts-download-job / COPY --from=builder /tmp/target/release/tts-inference-job / # NB(bt,2023-11-28): These still seem essential even after switching to rustls +# NB(bt,2023-11-30): I commented out the /etc/ssl copy and it broke certs, so this is *essential* # SSL certs are required for crypto -# COPY --from=builder /etc/ssl /etc/ssl # TODO TEST COMMENT +COPY --from=builder /etc/ssl /etc/ssl # NB(bt,2023-11-28): These still seem essential even after switching to rustls # Required dynamically linked libraries From 496ccd68f1a267060cf336742ea87d08d53bf435 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 30 Nov 2023 23:05:02 -0500 Subject: [PATCH 046/136] Working on uploads gonna be testing the endpoints soon. Need a ui for the upload page to understand how to write it but it's similar to prior art just need to check that out. --- .../public/weight_uploads/original_file.rs | 39 +++---- .../media_uploads/media_upload_type.rs | 8 ++ .../queries/model_weights/create_weight.rs | 1 + .../endpoints/weights/create_weight.rs | 110 +++++------------- .../drain_multipart_weights_request.rs | 100 ++++++++++++++++ .../src/http_server/endpoints/weights/mod.rs | 3 +- 6 files changed, 156 insertions(+), 105 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/original_file.rs index 7f0a40a6b7..42f203f849 100644 --- a/crates/_types/buckets/src/public/weight_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/weight_uploads/original_file.rs @@ -2,24 +2,20 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::weight_uploads::directory::WeightUploadDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; -// TODO: Generate these from a macro. - -/// The original user upload file. -/// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] -pub struct MediaUploadOriginalFilePath { - directory: MediaUploadDirectory, +pub struct WeightUploadOriginalFilePath { + directory: WeightUploadDirectory, full_object_path: String, } -impl PublicPath for MediaUploadOriginalFilePath {} +impl PublicPath for WeightUploadOriginalFilePath {} -impl MediaUploadOriginalFilePath { +impl WeightUploadOriginalFilePath { pub fn generate_new() -> Self { let entropy = crockford_entropy_lower(32); @@ -28,7 +24,7 @@ impl MediaUploadOriginalFilePath { pub fn from_object_hash(hash: &str) -> Self { // TODO: Path construction could be cleaner. - let directory = MediaUploadDirectory::from_object_hash(hash); + let directory = WeightUploadDirectory::from_object_hash(hash); let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); Self { directory, @@ -44,7 +40,7 @@ impl MediaUploadOriginalFilePath { PathBuf::from(&self.full_object_path) } - pub fn get_directory(&self) -> &MediaUploadDirectory { + pub fn get_directory(&self) -> &WeightUploadDirectory { &self.directory } @@ -60,37 +56,36 @@ impl MediaUploadOriginalFilePath { #[cfg(test)] mod tests { use std::path::PathBuf; - - use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + use crate::public::weight_uploads::original_file::WeightUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { - let file = MediaUploadOriginalFilePath::generate_new(); + let file = WeightUploadOriginalFilePath::generate_new(); assert_eq!(file.get_object_hash().len(), 32); assert_eq!(file.get_directory().get_object_hash().len(), 32); } #[test] pub fn get_full_object_path_str() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); - assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); } #[test] pub fn to_full_object_pathbuf() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); - assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); } #[test] pub fn get_full_object_path_str_short_name() { - let file = MediaUploadOriginalFilePath::from_object_hash("foo"); - assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/original_upload.bin"); + let file = WeightUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/f/o/foo/original_upload.bin"); } #[test] pub fn get_full_object_path_str_starts_with_directory() { - let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); let full_path = file.get_full_object_path_str(); assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) } @@ -98,7 +93,7 @@ mod tests { #[test] pub fn get_object_hash() { let hash = "abcdefghijk"; - let file = MediaUploadOriginalFilePath::from_object_hash(hash); + let file = WeightUploadOriginalFilePath::from_object_hash(hash); assert_eq!(file.get_object_hash(), hash); } } diff --git a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs index 91f06ed283..42fd07ce41 100644 --- a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs +++ b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs @@ -20,6 +20,9 @@ pub enum MediaUploadType { /// Video files: mp4, etc. Video, + + // Binary files: safetensors ... weights, etc. + Binary, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -33,6 +36,7 @@ impl MediaUploadType { Self::Audio => "audio", Self::Image => "image", Self::Video => "video", + Self::Binary => "binary", } } @@ -41,6 +45,7 @@ impl MediaUploadType { "audio" => Ok(Self::Audio), "image" => Ok(Self::Image), "video" => Ok(Self::Video), + "binary" => Ok(Self::Binary), _ => Err(format!("invalid value: {:?}", value)), } } @@ -56,6 +61,7 @@ mod tests { assert_serialization(MediaUploadType::Audio, "audio"); assert_serialization(MediaUploadType::Image, "image"); assert_serialization(MediaUploadType::Video, "video"); + assert_serialization(MediaUploadType::Binary, "binary"); } #[test] @@ -63,6 +69,7 @@ mod tests { assert_eq!(MediaUploadType::Audio.to_str(), "audio"); assert_eq!(MediaUploadType::Image.to_str(), "image"); assert_eq!(MediaUploadType::Video.to_str(), "video"); + assert_eq!(MediaUploadType::Binary.to_str(), "binary"); } #[test] @@ -70,6 +77,7 @@ mod tests { assert_eq!(MediaUploadType::from_str("audio").unwrap(), MediaUploadType::Audio); assert_eq!(MediaUploadType::from_str("image").unwrap(), MediaUploadType::Image); assert_eq!(MediaUploadType::from_str("video").unwrap(), MediaUploadType::Video); + assert_eq!(MediaUploadType::from_str("binary").unwrap(), MediaUploadType::Binary); assert!(MediaUploadType::from_str("foo").is_err()); } } diff --git a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs index d21f8a789e..4827edd907 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs @@ -7,6 +7,7 @@ use enums::by_table::model_weights::{ use log::warn; use enums::common::visibility::Visibility; use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + pub struct CreateModelWeightsArgs<'a> { pub token: &'a ModelWeightToken, pub weights_type: WeightsType, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 1db5217ba1..6b9194135f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -3,17 +3,19 @@ use std::sync::Arc; use actix_multipart::Multipart; use actix_web::{HttpRequest, HttpResponse, web}; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; use once_cell::sync::Lazy; -use tokens::tokens::media_uploads::MediaUploadToken; +use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use log::{error, info, warn}; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; @@ -23,17 +25,14 @@ use media::decode_basic_audio_info::decode_basic_audio_bytes_info; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; -use tokens::tokens::media_uploads::MediaUploadToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; -use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; -use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; #[derive(Serialize)] pub struct UploadWeightsSuccessResponse { pub success: bool, - pub upload_token: MediaUploadToken, + pub upload_token: ModelWeightToken, } static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { @@ -53,8 +52,10 @@ pub async fn upload_weights_handler( &server_state, multipart_payload, &ALLOWED_MIME_TYPES).await?; + + // send a body with the required payload along side the multipart upload - let weight_upload_token = response.to_media_token(); + let weight_upload_token = response.to_model_token(); let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { success: true, @@ -69,18 +70,17 @@ pub async fn upload_weights_handler( .body(body)); } - pub enum SuccessCase { WeightAlreadyUploaded { - existing_upload_token: WeightUploadToken, + existing_upload_token: ModelWeightToken, }, WeightSuccessfullyUploaded { - upload_token: WeightUploadToken, + upload_token: ModelWeightToken, } } impl SuccessCase { - pub fn to_media_token(self) -> WeightUploadToken { + pub fn to_model_token(self) -> ModelWeightToken { match self { SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, @@ -190,6 +190,7 @@ pub async fn handle_weight_upload( .as_ref() .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) .flatten(); + let bytes = match upload_weights_request.file_bytes { None => return Err(UploadError::BadInput("missing file contents".to_string())), @@ -219,38 +220,14 @@ pub async fn handle_weight_upload( .collect::(); return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); } - - // NB: .aiff (audio/aiff) isn't supported by Symphonia: - // It contains uncompressed PCM-encoded audio similar to wav. - // See: https://github.com/pdeljanov/Symphonia/issues/75 - // NB: The following formats are not supported by Symphonia and - // do not have any open issues filed. They may simply be too old: - // - .wma (audio/x-ms-wma) - // - .avi (video/x-msvideo) media_upload_type = match mimetype { - // Audio - "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), _ => None, }; - - - let media_upload_type = match media_upload_type { - Some(m) => m, - None => { - warn!("Invalid mimetype: {:?}", maybe_mimetype); - return Err(UploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) - }, - }; - - // probably don't need this - let media_upload_source = match upload_weights_request.media_source { - MediaSource::Unknown => MediaUploadSource::Unknown, - MediaSource::UserFile => MediaUploadSource::File, - MediaSource::UserDeviceApi => MediaUploadSource::DeviceApi, - }; + } // This is fine for now we just don't expose the path! - let public_upload_path = MediaUploadOriginalFilePath::generate_new(); + let public_upload_path = WeightUploadOriginalFilePath::generate_new(); info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); @@ -266,18 +243,18 @@ pub async fn handle_weight_upload( })?; let token = create_weight(CreateModelWeightsArgs { - token: &MediaUploadToken::generate(), - weights_type: media_upload_type, - weights_category: media_upload_category, + token: &ModelWeightToken::generate(), + weights_type: WeightsType::LoRA, // to do parse multi part for this + weights_category: WeightsCategory::ImageGeneration, title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: None, - description_markdown: None, - description_rendered_html: None, - creator_user_token: maybe_user_token.as_deref(), + maybe_thumbnail_token: &ThumbnailToken::generate(), + description_markdown: "", + description_rendered_html: "", + creator_user_token: maybe_user_token.as_str(), creator_ip_address: &ip_address, creator_set_visibility, - maybe_last_update_user_token: None, - original_download_url: None, + maybe_last_update_user_token: "", + original_download_url: "", original_filename: upload_weights_request.file_name.as_deref(), file_size_bytes: file_size_bytes as i32, file_checksum_sha2: hash, @@ -290,44 +267,13 @@ pub async fn handle_weight_upload( maybe_cached_user_ratings_ratio: None, version: 0, mysql_pool: &server_state.mysql_pool, - }) - .await - .map_err(|err| { - warn!("New generic download creation DB error: {:?}", err); - UploadError::ServerError - })?; - }).await?; - - - - let (token, record_id) = insert_media_upload(Args{ - uuid_idempotency_token: &uuid_idempotency_token, - media_type: media_upload_type, - media_source: media_upload_source, - maybe_original_filename: upload_weights_request.file_name.as_deref(), - original_file_size_bytes: file_size_bytes as u64, - maybe_original_duration_millis: maybe_duration_millis, - maybe_original_mime_type: maybe_mimetype, - maybe_original_audio_encoding: maybe_codec_name.as_deref(), - maybe_original_video_encoding: None, - maybe_original_frame_width: None, // TODO - maybe_original_frame_height: None, // TODO - checksum_sha2: &hash, - public_upload_path: &public_upload_path, - maybe_extra_file_modification_info: None, - maybe_creator_user_token: maybe_user_token.as_ref(), - maybe_creator_anonymous_visitor_token: None, - creator_ip_address: &ip_address, - creator_set_visibility, - mysql_pool: &server_state.mysql_pool, - }) - .await - .map_err(|err| { - warn!("New generic download creation DB error: {:?}", err); + }).await + .map_err(|e| { + error!("Error creating weights upload: {:?}", e); UploadError::ServerError })?; - info!("new media upload id: {} token: {:?}", record_id, &token); + info!("new weights upload id: {} token: {:?}", record_id, &token); Ok(SuccessCase::WeightSuccessfullyUploaded { upload_token: token, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs new file mode 100644 index 0000000000..c87a9d82c5 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs @@ -0,0 +1,100 @@ +use actix_multipart::Multipart; +use actix_web::web::BytesMut; +use futures::TryStreamExt; +use log::warn; + +use errors::AnyhowResult; +use tokens::tokens::weights::ModelWeightToken; + +use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; + +pub struct WeightUploadRequest { + pub uuid_idempotency_token: Option, + pub model_weights_token: Option, + pub file_name: Option, + pub file_bytes: Option, + pub media_source: MediaSource, +} + +/// Where the frontend tells us the file came from. +#[derive(Eq, PartialEq, Clone, Copy)] +pub enum MediaSource { + Unknown, + UserFile, + /// Eg. the web audio API in Javascript. + UserDeviceApi, +} + +/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. +pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { + let mut uuid_idempotency_token = None; + let mut model_weight_token = None; + let mut file_bytes = None; + let mut file_name = None; + let mut media_source = None; + + while let Ok(Some(mut field)) = multipart_payload.try_next().await { + let mut field_name = None; + let mut field_filename = None; + + if let content_disposition = field.content_disposition() { + field_name = content_disposition.get_name() + .map(|s| s.to_string()); + field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. + .map(|s| s.to_string()); + } + + match field_name.as_deref() { + Some("uuid_idempotency_token") => { + uuid_idempotency_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading idempotency token: {:}", e); + e + })?; + }, + Some("dataset_token") => { + model_weight_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading dataset token: {:}", e); + e + })?; + }, + Some("file") => { + file_name = field_filename.clone(); + file_bytes = checked_read_multipart_bytes(&mut field).await + .map_err(|e| { + warn!("Error reading audio upload: {:}", e); + e + })?; + }, + Some("source") => { + media_source = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading source: {:}", e); + e + })?; + }, + _ => continue, + } + } + + let media_source = match media_source.as_deref() { + Some("device") => MediaSource::UserDeviceApi, + Some("file") => MediaSource::UserFile, + _ => { + if file_name.as_deref() == Some("blob") { + MediaSource::UserDeviceApi + } else { + MediaSource::Unknown + } + }, + }; + + Ok(UploadSampleRequest { + uuid_idempotency_token, + dataset_token, + file_name, + file_bytes, + media_source, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index b9e2bd05c2..ba1f88eb2a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -3,4 +3,5 @@ pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; pub mod list_availible_weights; -pub mod create_weight; \ No newline at end of file +pub mod create_weight; +pub mod drain_multipart_weights_request; \ No newline at end of file From 3cc1ea693564915b3febde0682c08997939198cf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 1 Dec 2023 01:36:33 -0500 Subject: [PATCH 047/136] capture stderr for svc and rvc --- _sql/actions/generic_inference_jobs.sql | 7 ++++++ .../job/job_types/vc/rvc_v2/process_job.rs | 22 ++++++++++++++++++- .../vc/rvc_v2/rvc_v2_inference_command.rs | 10 ++++++++- .../job_types/vc/so_vits_svc/process_job.rs | 22 ++++++++++++++++++- .../so_vits_svc_inference_command.rs | 10 ++++++++- 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/_sql/actions/generic_inference_jobs.sql b/_sql/actions/generic_inference_jobs.sql index d22d74a190..e0ec788b3b 100644 --- a/_sql/actions/generic_inference_jobs.sql +++ b/_sql/actions/generic_inference_jobs.sql @@ -19,7 +19,14 @@ set status = 'dead' where status IN ('pending', 'started', 'attempt_failed') and maybe_model_type IN ('sad_talker', 'so_vits_svc'); +-- Tacotron 2 TTS update generic_inference_jobs set status = 'dead' where status IN ('pending', 'started', 'attempt_failed') and maybe_model_type IN ('tacotron2'); + +-- Sad Talker Face Animation +update generic_inference_jobs +set status = 'dead' +where status IN ('pending', 'started', 'attempt_failed') + and maybe_model_type IN ('sad_talker'); diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 16a8992dfd..74496b6abc 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -1,3 +1,4 @@ +use std::fs::read_to_string; use std::path::PathBuf; use std::time::Instant; @@ -220,6 +221,8 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result) -> Result) -> Result { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + //} + } + safe_delete_temp_file(&input_wav_path); safe_delete_temp_file(&output_audio_fs_path); safe_delete_temp_directory(&work_temp_dir); - return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status))); + + return Err(error); } // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs index bb3be2e01f..b6ff6cc243 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs @@ -1,13 +1,14 @@ use std::collections::HashSet; use std::env; use std::ffi::OsString; +use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::anyhow; use log::info; use once_cell::sync::Lazy; -use subprocess::{Popen, PopenConfig}; +use subprocess::{Popen, PopenConfig, Redirection}; use errors::AnyhowResult; use filesys::path_to_string::path_to_string; @@ -80,6 +81,8 @@ pub struct InferenceArgs, Q: AsRef> { /// --output_audio_filename: output path of wav file. pub output_path: P, + pub stderr_output_file: P, + /// --f0-method: f0 prediction method pub maybe_override_f0_method: Option, @@ -267,6 +270,11 @@ impl RvcV2InferenceCommand { let mut config = PopenConfig::default(); + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + if !env_vars.is_empty() { config.env = Some(env_vars); } diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 5a54968f7a..f5d7fa1b89 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -1,3 +1,4 @@ +use std::fs::read_to_string; use std::time::Instant; use anyhow::anyhow; @@ -148,6 +149,8 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result) -> Result) -> Result { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + //} + } + safe_delete_temp_file(&input_wav_path); safe_delete_temp_file(&output_audio_fs_path); safe_delete_temp_directory(&work_temp_dir); - return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status))); + + return Err(error); } // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs index e99384933c..cbeca1deb3 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs @@ -1,13 +1,14 @@ use std::collections::HashSet; use std::env; use std::ffi::OsString; +use std::fs::File; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::anyhow; use log::info; use once_cell::sync::Lazy; -use subprocess::{Popen, PopenConfig}; +use subprocess::{Popen, PopenConfig, Redirection}; use errors::AnyhowResult; use filesys::path_to_string::path_to_string; @@ -90,6 +91,8 @@ pub struct InferenceArgs> { /// --output_path: output path of converting model to onnx (which we use to test validity) pub output_path: P, + pub stderr_output_file: P, + /// --auto-predict-f0: turn on or off fundamental frequency auto prediction /// This sounds better when left off, but it defaults to *ON* if not specified. pub auto_predict_f0: bool, @@ -366,6 +369,11 @@ impl SoVitsSvcInferenceCommand { let mut config = PopenConfig::default(); + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + if !env_vars.is_empty() { config.env = Some(env_vars); } From b0a9342194b00e1776b27c9b565b4fcc94a2e18a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 1 Dec 2023 07:09:23 -0500 Subject: [PATCH 048/136] updated queue stats --- .../src/http_server/endpoints/stats/get_unified_queue_stats.rs | 3 +++ .../src/http_server/endpoints/stats/result_transformer.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs index 1af0ae3d2b..0ed6929ef8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs @@ -46,6 +46,7 @@ pub struct ByQueueStats { pub pending_rvc_jobs: u64, pub pending_svc_jobs: u64, pub pending_tacotron2_jobs: u64, + pub pending_voice_designer: u64, } #[derive(Debug)] @@ -97,6 +98,7 @@ pub async fn get_unified_queue_stats_handler( pending_rvc_jobs: 10_000, pending_svc_jobs: 10_000, pending_tacotron2_jobs: 10_000, + pending_voice_designer: 10_000, } }, legacy_tts: LegacyQueueDetails { @@ -197,6 +199,7 @@ pub async fn get_unified_queue_stats_handler( pending_rvc_jobs: cacheable_stats_result.queues.rvc_v2, pending_svc_jobs: cacheable_stats_result.queues.so_vits_svc, pending_tacotron2_jobs: cacheable_stats_result.queues.tacotron2, + pending_voice_designer: cacheable_stats_result.queues.vall_e_x, } }, legacy_tts: LegacyQueueDetails { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs index cf68587ad3..5e70d335c1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs @@ -27,6 +27,7 @@ pub struct Queues { pub sad_talker: u64, pub so_vits_svc: u64, pub tacotron2: u64, // Tacotron2 (as generic_inference_jobs; NB: included in `total_generic`) + pub vall_e_x: u64, // Counts both model weight calculation and inference. } pub fn database_result_to_cacheable(database_records: Vec) -> CacheableQueueStats { @@ -45,6 +46,7 @@ pub fn database_result_to_cacheable(database_records: Vec) -> Cac let sad_talker = unwrap(queue_lengths.get("sad_talker")); let so_vits_svc = unwrap(queue_lengths.get("so_vits_svc")); let tacotron2 = unwrap(queue_lengths.get("tacotron2")); + let vall_e_x = unwrap(queue_lengths.get("vall_e_x")); let total_generic = rvc_v2 + so_vits_svc; @@ -57,6 +59,7 @@ pub fn database_result_to_cacheable(database_records: Vec) -> Cac sad_talker, so_vits_svc, tacotron2, + vall_e_x, }, } } From 1d937ab98133d87f235153c3d2d88c9407f24c71 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 1 Dec 2023 15:22:32 -0500 Subject: [PATCH 049/136] Initial weights_endpoints need testing right now. --- crates/service/web/storyteller_web/src/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index db590ab3da..015192966c 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1239,7 +1239,7 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - // .route("/upload", web::post().to(upload_weights_handler)) + // .route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") From 8072d135962b52505cadf4b17e94b77ad2446355 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Sun, 3 Dec 2023 01:35:04 -0500 Subject: [PATCH 050/136] Fixed Delete and Updates for Weights --- .../queries/model_weights/update_weight.rs | 16 ++++++---------- .../endpoints/weights/delete_weight.rs | 17 +++++++++++------ .../src/http_server/endpoints/weights/mod.rs | 4 ++-- .../endpoints/weights/update_weight.rs | 19 +------------------ .../service/web/storyteller_web/src/routes.rs | 6 ++++-- 5 files changed, 24 insertions(+), 38 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 2b1d4a9c0f..0d15d4e38f 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -18,12 +18,8 @@ pub struct UpdateWeightArgs<'a> { } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; - let visbility: &str = match args.creator_set_visibility { - Some(visibility) => visibility.to_str(), - None => "", - }; let query_result = sqlx::query!( r#" @@ -37,11 +33,11 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { version = version + 1 WHERE token = ? "#, - args.title.as_deref().unwrap_or(""), - args.description_markdown.as_deref().unwrap_or(""), - args.maybe_thumbnail_token.as_deref().unwrap_or(""), - args.description_rendered_html.as_deref().unwrap_or(""), - visbility, + args.title.as_deref(), + args.description_markdown.as_deref(), + args.maybe_thumbnail_token.as_deref(), + args.description_rendered_html.as_deref(), + args.creator_set_visibility.as_deref(), args.weight_token.as_str() ) .execute(args.mysql_pool).await; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index 91dfcbe94c..f8f547b3b3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -23,11 +23,15 @@ use enums::common::visibility::Visibility; use tokens::tokens::users::UserToken; +#[derive(Deserialize)] +pub struct DeleteWeightPathInfo { + weight_token: String, +} + #[derive(Deserialize)] pub struct DeleteWeightRequest { set_delete: bool, - as_mod: bool, - weight_token: String, + as_mod: bool } // =============== Error Response =============== @@ -64,7 +68,8 @@ impl fmt::Display for DeleteWeightError { pub async fn delete_weight_handler( http_request: HttpRequest, - path: Path, + path: Path, + request: web::Json, server_state: web::Data> ) -> Result{ let maybe_user_session = server_state @@ -112,9 +117,9 @@ pub async fn delete_weight_handler( return Err(DeleteWeightError::NotAuthorized); } - let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(path.as_mod)); - - let query_result = if path.set_delete { + let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(request.as_mod)); + + let query_result = if request.set_delete { match delete_role { DeleteRole::ErrorDoNotDelete => { warn!("user is not allowed to delete weights: {}", user_session.user_token); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index ba1f88eb2a..f457767f25 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -3,5 +3,5 @@ pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; pub mod list_availible_weights; -pub mod create_weight; -pub mod drain_multipart_weights_request; \ No newline at end of file +// pub mod create_weight; +// pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 4ab26cab50..9f159ab417 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -19,7 +19,6 @@ use crate::server_state::ServerState; /// TODO will eventually be polymorphic #[derive(Deserialize)] pub struct UpdateWeightRequest { - pub weight_token: String, pub title: Option, pub thumbnail_token: Option, pub description_markdown: Option, @@ -127,25 +126,9 @@ pub async fn update_weight_handler( return Err(UpdateWeightError::NotAuthorized); } - let mut weight_type = None; - let mut weight_category = None; - - if let Some(wt) = request.weight_type.as_deref() { - weight_type = Some(wt.to_string()); - } - - if let Some(wc) = request.weight_category.as_deref() { - weight_category = Some(wc.to_string()); - } - - let mut maybe_mod_user_token = None; - - if is_mod { - maybe_mod_user_token = Some(user_session.user_token.clone()); - } let query_result = update_weights(UpdateWeightArgs { - weight_token: &ModelWeightToken::new(weight_token.clone()), + weight_token: &ModelWeightToken::new(path.weight_token.clone()), mysql_pool: &server_state.mysql_pool, title: request.title.as_deref(), maybe_thumbnail_token: request.thumbnail_token.as_deref(), diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 015192966c..4415eb75e8 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -152,6 +152,9 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; +use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; + pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where @@ -1223,7 +1226,6 @@ fn add_voice_designer_routes (app:App)-> App ) } - // ==================== Weights ROUTES ==================== fn add_weights_routes(app: App) -> App where @@ -1239,7 +1241,7 @@ fn add_weights_routes(app: App) -> App app.service( web ::scope("/v1/weights") - // .route("/upload", web::post().to(upload_weights_handler)) + //.route("/upload", web::post().to(upload_weights_handler)) .service( web ::resource("/weight/{weight_token}") From 9c8ee8dbbbd3d7a3230e55dcaf7b45228f644cd2 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Sun, 3 Dec 2023 22:55:56 -0500 Subject: [PATCH 051/136] Ok list_weights_by_user works properly now. --- .../model_weights/list_weights_by_user.rs | 6 +- .../src/queries/model_weights/tests.rs | 4 ++ .../endpoints/weights/list_weights_by_user.rs | 56 +++++++++++++++---- .../service/web/storyteller_web/src/routes.rs | 4 +- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs index 49d5e52155..e0ef62baee 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs @@ -77,11 +77,7 @@ pub async fn list_weights_by_creator_username( let raw_weights: Vec = get_raw_weights_by_creator_username(&mut connection, creator_username, can_see_deleted).await?; let weights_records: Vec = map_to_weights(raw_weights).await; - let filtered_weights: Vec = weights_records.into_iter().filter(|weight| { - weight.creator_username == creator_username && weight.creator_set_visibility == Visibility::Public && can_see_deleted - }).collect(); - - Ok(filtered_weights) + Ok(weights_records) } async fn get_raw_weights_by_creator_username( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index dca6d0cab6..9b054e4b30 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -292,6 +292,10 @@ mod tests { seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,5,&creator_username).await?; let weights_by_username = list_weights_by_creator_username(&pool, &creator_username, can_see_deleted).await?; + for weight in weights_by_username.iter() { + // print weight + println!("weight: {:?}", weight.creator_username); + } assert_eq!(weights_by_username.len(), 5); Ok(()) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index 21ac39c7a8..c478ea68d9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use log::warn; use enums::common::visibility::Visibility; -use mysql_queries::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; +use mysql_queries::queries::model_weights::list_weights_by_user::{list_weights_by_creator_username, WeightsJoinUserRecord}; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; @@ -19,11 +19,23 @@ use crate::server_state::ServerState; pub struct Weight { weight_token: ModelWeightToken, title: String, - + creator: UserDetailsLight, - creator_set_visibility: Visibility, + maybe_thumbnail_token: Option, + + description_markdown: String, + description_rendered_html: String, + + file_size_bytes: i32, + file_checksum_sha2: String, + cached_user_ratings_total_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_negative_count: u32, + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + created_at: DateTime, updated_at: DateTime, } @@ -96,17 +108,16 @@ pub async fn list_weights_by_user_handler( warn!("Error querying for weights: {:?}", e); ListWeightsByUserError::ServerError }); + let weights = match query_results { Ok(weights) => weights, Err(e) => { warn!("Error querying for weights: {:?}", e); - - return Err(ListWeightsByUserError::ServerError); } }; - - let weights = weights.into_iter().map(|weight| { + + let weights:Vec = weights.into_iter().map(|weight| { Weight { weight_token: weight.token, title: weight.title, @@ -116,16 +127,41 @@ pub async fn list_weights_by_user_handler( &weight.creator_display_name, &weight.creator_email_gravatar_hash, ), + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, creator_set_visibility: weight.creator_set_visibility, created_at: weight.created_at, updated_at: weight.updated_at, } }).collect(); - - let response = ListWeightsByUserSuccessResponse { + + let final_weights:Vec; + + // if it's not the user ... then only show public weights else show private and public + if (creator_user_token != user_session.user_token) { + final_weights = weights.into_iter().filter(|weight| { + weight.creator_set_visibility == Visibility::Public + }).collect(); + + } + else { + final_weights = weights; + } + + + let response: ListWeightsByUserSuccessResponse = ListWeightsByUserSuccessResponse { success: true, - weights, + weights: final_weights, }; + let body = serde_json::to_string(&response) .map_err(|e| ListWeightsByUserError::ServerError)?; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4415eb75e8..4f05c00c9a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1249,10 +1249,12 @@ fn add_weights_routes(app: App) -> App .route(web::post().to(update_weight_handler)) .route(web::delete().to(delete_weight_handler)) ) - // .route("/by_user/{username}", web::get().to(get_weights_by_user_handler)) + .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) // .route( // "/by_user_and_type/{username}/{type}", // web::get().to(get_weights_by_user_and_type_handler) // ) ) } +// use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +// use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; \ No newline at end of file From e4756a4f2c2c26bd05773be15c8e426b8931ba74 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 01:52:14 -0500 Subject: [PATCH 052/136] Added more data seeding to the database for the model_weights table. --- .../src/seeding/model_weights.rs | 89 ++++++++++++++++++- .../endpoints/weights/update_weight.rs | 1 - .../service/web/storyteller_web/src/routes.rs | 10 +-- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index de9bcd90a9..07c8f2f155 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -1,10 +1,13 @@ +use std::vec; + +use buckets::public::media_files::original_file; use log::info; use tokens::tokens::users::UserToken; use sqlx::{ MySql, Pool }; use enums::by_table::model_weights::{ - weights_types::WeightsType, - weights_category::WeightsCategory, + weights_types::{WeightsType, self}, + weights_category::{WeightsCategory, self}, }; use errors::{ anyhow, AnyhowResult }; @@ -15,6 +18,86 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +pub async fn seed_weights_for_paging( + mysql_pool: &Pool) -> AnyhowResult<()> +{ + + let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { + None => { + return Err(anyhow!("could not find user hanashi")); + } + Some(token) => token, + }; + + // create a loop that loops from 1 to 100 + for i in 1..=100 { + // create a new weight + let model_weight_token = ModelWeightToken(format!("item:{}", i)); + let title = format!("Title {}", i); + let description = format!("Description {}", i); + let description_rendered_html = format!("

{}

", description); + let original_filename = format!("filename{}.txt", i); + let original_download_url = format!("http://example.com/download{}", i); + let thumbnail_token = format!("Thumbnail {}", i); + let private_bucket_hash = format!("bucket_hash{}", i); + let private_bucket_prefix = format!("_fake{}", i); + let private_bucket_extension = format!("rvc{}", i); + let cached_user_ratings_total_count = i; + let cached_user_ratings_positive_count = i; + let cached_user_ratings_negative_count = i; + let cached_user_ratings_ratio = i as u32 / 100; + let version = i as i32; + + let weights_category = match i { + 1..=20 => WeightsCategory::VoiceConversion, + 21..=40 => WeightsCategory::TextToSpeech, + 41..=60 => WeightsCategory::ImageGeneration, + 61..=80 => WeightsCategory::ImageGeneration, + 81..=100 => WeightsCategory::ImageGeneration, + _ => WeightsCategory::ImageGeneration + }; + + let weights_types = match i { + 1..=20 => WeightsType::RvcV2, + 21..=40 => WeightsType::HifiganTacotron2, + 41..=60 => WeightsType::StableDiffusion15, + 61..=80 => WeightsType::StableDiffusionXL, + 81..=100 => WeightsType::LoRA, + _ => WeightsType::LoRA + }; + + let args = CreateModelWeightsArgs { + token: &model_weight_token, // replace with actual ModelWeightToken + weights_type: weights_types, // replace with actual WeightsType + weights_category: weights_category, // replace with actual WeightsCategory + title: title, + maybe_thumbnail_token: Some(thumbnail_token), + description_markdown: description, + description_rendered_html: description_rendered_html, + creator_user_token: Some(&user_token), // replace with actual UserToken + creator_ip_address: "192.168.1.1", + creator_set_visibility: Visibility::Public, + maybe_last_update_user_token: Some(user_token.to_string()), + original_download_url: Some(original_download_url), + original_filename: Some(original_filename), + file_size_bytes: 1024, + file_checksum_sha2: "checksum1".to_string(), + private_bucket_hash: "bucket_hash1".to_string(), + maybe_private_bucket_prefix: Some("_fake".to_string()), + maybe_private_bucket_extension: Some("rvc".to_string()), + cached_user_ratings_total_count: cached_user_ratings_total_count, + cached_user_ratings_positive_count: cached_user_ratings_positive_count, + cached_user_ratings_negative_count: cached_user_ratings_ratio, + maybe_cached_user_ratings_ratio: Some(0.9), + version: version, + mysql_pool: &mysql_pool, // replace with actual MySqlPool + }; + + create_weight(args).await?; + } + Ok(()) + +} pub async fn seed_weights_for_user_token( mysql_pool: &Pool, user_token: UserToken @@ -330,6 +413,6 @@ pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { // adds this to HANASHI get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await?; seed_weights_for_user_token(mysql_pool, user_token).await?; - + seed_weights_for_paging(mysql_pool).await?; Ok(()) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 9f159ab417..0e287729a3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -126,7 +126,6 @@ pub async fn update_weight_handler( return Err(UpdateWeightError::NotAuthorized); } - let query_result = update_weights(UpdateWeightArgs { weight_token: &ModelWeightToken::new(path.weight_token.clone()), mysql_pool: &server_state.mysql_pool, diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4f05c00c9a..6c06107519 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1250,11 +1250,9 @@ fn add_weights_routes(app: App) -> App .route(web::delete().to(delete_weight_handler)) ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) - // .route( - // "/by_user_and_type/{username}/{type}", - // web::get().to(get_weights_by_user_and_type_handler) - // ) + .route( + "/by_user_and_type/{username}/{type}", + web::get().to(list_availible_weights_handler) + ) ) } -// use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; -// use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; \ No newline at end of file From 241f648baf724e8cc976bdd33dc6c5010e5490f8 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 03:38:51 -0500 Subject: [PATCH 053/136] Add type filtering in there and category filtering --- .../list_weights_query_builder.rs | 8 +- .../weights/list_availible_weights.rs | 205 +++++++++++------- 2 files changed, 125 insertions(+), 88 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs index 69ccf3617a..e1d79b89ab 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs @@ -133,13 +133,13 @@ impl ListWeightsQueryBuilder { self } - pub fn weights_type(mut self, weights_type: WeightsType) -> Self { - self.weights_type = Some(weights_type); + pub fn weights_type(mut self, weights_type: Option) -> Self { + self.weights_type = weights_type; self } - pub fn weights_category(mut self, weights_category: WeightsCategory) -> Self { - self.weights_category = Some(weights_category); + pub fn weights_category(mut self, weights_category: Option) -> Self { + self.weights_category = weights_category; self } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index fa64de0d58..26257fc60a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -35,11 +35,17 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } +#[derive(Deserialize)] +pub struct ListWeightsByPathInfo { + pub username: String, + pub weights_type: Option, + pub weights_category: Option, +} + #[derive(Serialize)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, - pub weights_type: WeightsType, pub weights_category: WeightsCategory, @@ -73,31 +79,31 @@ pub struct ModelWeightForList { #[derive(Debug)] pub enum ListWeightError { - NotAuthorized, - ServerError, + NotAuthorized, + ServerError, } impl std::fmt::Display for ListWeightError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } } impl ResponseError for ListWeightError { - fn status_code(&self) -> StatusCode { - match *self { - ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } } - } } pub async fn list_availible_weights_handler( http_request: HttpRequest, + path: web::Path, query: web::Query, server_state: web::Data> ) -> Result { - let maybe_user_session = server_state.session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { @@ -123,103 +129,134 @@ pub async fn list_availible_weights_handler( let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) .map_err(|e| { warn!("crypto error: {:?}", e); ListWeightError::ServerError - })?; - Some(cursor) + })?; + Some(cursor) } else { - None + None }; - let include_user_hidden = is_mod; + let include_user_hidden = is_mod; - let mut query_builder = ListWeightsQueryBuilder::new() - .sort_ascending(sort_ascending) - .scope_creator_username(None) - .include_user_hidden(include_user_hidden) - .include_user_deleted_results(is_mod) - .include_mod_deleted_results(is_mod) - .limit(limit) - .cursor_is_reversed(cursor_is_reversed) - .offset(cursor); + let mut weights_category: Option = None; + let mut weights_type: Option = None; - let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + if let Some(weights_category_string) = path.weights_category.as_ref() { + let result = WeightsCategory::from_str(&weights_category_string); + match result { + Ok(category) => { + weights_category = Some(category) + }, + Err(e) => { + warn!("invalid weights_category: {:?}", e); + weights_category = None + } + } + } + + if let Some(weights_type_string) = path.weights_type.as_ref() { + let result = WeightsType::from_str(&weights_type_string); + match result { + Ok(wtype) => { + weights_type = Some(wtype) + }, + Err(e) => { + warn!("invalid weights_type: {:?}", e); + weights_type = None + } + } + } + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .weights_category(weights_category) + .weights_type(weights_type) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - let weights_page = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListWeightError::ServerError); - } - }; + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; - let cursor_next = if let Some(id) = weights_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; - let cursor_previous = if let Some(id) = weights_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; - // generate parse a response - let response = ListAvailibleWeightsSuccessResponse { - success: true, - weights: weights_page.weights.into_iter() - .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - weights_type: weights.weights_type, - weights_category: weights.weights_category, + // generate parse a response + let response = ListAvailibleWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, - maybe_thumbnail_token:weights.maybe_thumbnail_token, - description_markdown: weights.description_markdown, - description_rendered_html: weights.description_rendered_html, + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, - creator_user_token: weights.creator_user_token, - creator_set_visibility: weights.creator_set_visibility, + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, - file_size_bytes:weights.file_size_bytes, - file_checksum_sha2: weights.file_checksum_sha2, + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, - cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, - cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, - maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - version: weights.version, + version: weights.version, - created_at: weights.created_at, - updated_at: weights.updated_at, + created_at: weights.created_at, + updated_at: weights.updated_at, - creator_username: weights.creator_username, - creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash - }).collect::>(), - cursor_next, - cursor_previous, - }; - - let body = serde_json::to_string(&response) + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash + }).collect::>(), + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) .map_err(|e| ListWeightError::ServerError)?; From 4bf1a25fbb4ec89c98be5f6d8c409ff1bd7a4f31 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 2 Dec 2023 23:50:13 -0500 Subject: [PATCH 054/136] Kill inference jobs (mod endpoint) --- .../web/kill_generic_inference_jobs.rs | 157 ++++++++++++++++++ .../src/queries/generic_inference/web/mod.rs | 1 + .../inference_job/kill_inference_jobs.rs | 144 ++++++++++++++++ .../endpoints/inference_job/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 6 + 5 files changed, 309 insertions(+) create mode 100644 crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/kill_inference_jobs.rs diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs new file mode 100644 index 0000000000..4928fcd6f4 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs @@ -0,0 +1,157 @@ +use std::collections::HashSet; + +use anyhow::anyhow; +use sqlx::{MySql, MySqlPool, QueryBuilder}; + +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use enums::common::job_status::JobStatus; +use errors::AnyhowResult; + +/// Only certain job statuses should be modified. +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub enum KillableStatus { + Failed, + Pending, + Started, +} + +/// Target everything, by job category, or by model type. +#[derive(Clone)] +pub enum KillableTarget { + AllJobs, + Category(InferenceCategory), + ModelType(InferenceModelType), +} + +pub struct KillGenericInferenceJobsArgs<'a> { + pub job_statuses: HashSet, + pub target: KillableTarget, + pub maybe_priority_or_lower: Option, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn kill_generic_inference_jobs(args: KillGenericInferenceJobsArgs<'_>) -> AnyhowResult<()> { + let mut query = query_builder( + args.job_statuses, + args.target, + args.maybe_priority_or_lower + ); + + let query = query.build(); + + let result = query.execute(args.mysql_pool).await; + + match result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(_r) => Ok(()), + } +} + +fn query_builder( + job_statuses: HashSet, + target: KillableTarget, + maybe_priority_or_lower: Option, +) -> QueryBuilder<'static, MySql> { + + // NB: Query cannot be statically checked by sqlx + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +UPDATE generic_inference_jobs +SET status='cancelled_by_system' +WHERE status IN( + "# + ); + + let job_statuses = job_statuses.into_iter() + .map(|status| match status { + KillableStatus::Failed => JobStatus::AttemptFailed.to_str(), + KillableStatus::Pending => JobStatus::Pending.to_str(), + KillableStatus::Started => JobStatus::Started.to_str(), + }) + .collect::>(); + + // NB: Syntax will be wrong if list has zero length + let mut separated = query_builder.separated(", "); + for value_type in job_statuses.into_iter() { + separated.push_bind(value_type); + } + separated.push_unseparated(") "); + + if let Some(priority) = maybe_priority_or_lower { + query_builder.push(format!(" AND priority <= {priority} ")); + } + + match target { + KillableTarget::AllJobs => { + // No need to modify query. + } + KillableTarget::Category(category) => { + query_builder.push(" AND inference_category = "); + query_builder.push_bind(category); + } + KillableTarget::ModelType(model_type) => { + query_builder.push(" AND maybe_model_type = "); + query_builder.push_bind(model_type); + } + } + + query_builder +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use regex::Regex; + + use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; + use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; + + use crate::queries::generic_inference::web::kill_generic_inference_jobs::{KillableStatus, KillableTarget, query_builder}; + + fn cleanup_query(query: &str) -> String { + let regex = Regex::new(r#"\s+"#).unwrap(); + let result = regex.replace_all(query, " ").trim().to_string(); + result + } + + #[test] + fn test_statuses() { + let statuses = HashSet::from([KillableStatus::Failed]); + let builder = query_builder(statuses, KillableTarget::AllJobs, None); + let query = cleanup_query(&builder.into_sql()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?)".to_string()); + + let statuses = HashSet::from([KillableStatus::Failed, KillableStatus::Pending, KillableStatus::Started]); + let builder = query_builder(statuses, KillableTarget::AllJobs, None); + let query = cleanup_query(&builder.into_sql()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?, ?, ?)".to_string()); + } + + #[test] + fn test_priority() { + let statuses = HashSet::from([KillableStatus::Failed]); + let builder = query_builder(statuses, KillableTarget::AllJobs, Some(10)); + let query = cleanup_query(&builder.into_sql()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND priority <= 10".to_string()); + } + + #[test] + fn test_category_target() { + let statuses = HashSet::from([KillableStatus::Failed]); + let builder = query_builder(statuses, KillableTarget::Category(InferenceCategory::LipsyncAnimation), None); + let query = cleanup_query(&builder.into_sql()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND inference_category = ?".to_string()); + } + + #[test] + fn test_model_target() { + let statuses = HashSet::from([KillableStatus::Failed]); + let builder = query_builder(statuses, KillableTarget::ModelType(InferenceModelType::RvcV2), None); + let query = cleanup_query(&builder.into_sql()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND maybe_model_type = ?".to_string()); + } +} diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs index 71a2786c0b..53a89e6a78 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs @@ -1,3 +1,4 @@ pub mod get_inference_job_status; pub mod get_pending_inference_job_count; pub mod insert_generic_inference_job; +pub mod kill_generic_inference_jobs; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/kill_inference_jobs.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/kill_inference_jobs.rs new file mode 100644 index 0000000000..76957e221c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/kill_inference_jobs.rs @@ -0,0 +1,144 @@ +use std::collections::HashSet; +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use log::warn; + +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::generic_inference::web::kill_generic_inference_jobs; +use mysql_queries::queries::generic_inference::web::kill_generic_inference_jobs::{kill_generic_inference_jobs, KillGenericInferenceJobsArgs}; + +use crate::server_state::ServerState; + +/// Only certain job statuses should be modified. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KillableStatus { + Failed, + Pending, + Started, +} + +/// Target everything, by job category, or by model type. +#[derive(Clone, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KillableTarget { + AllJobs, + Category(InferenceCategory), + ModelType(InferenceModelType), +} + +// NB: ONLY MODERATORS CAN EDIT CATEGORIES. +// These are not sparse updates! +#[derive(Deserialize)] +pub struct KillInferenceJobsRequest { + pub job_statuses: HashSet, + pub target: KillableTarget, + pub maybe_priority_or_lower: Option, +} + +#[derive(Serialize)] +pub struct KillInferenceJobsResponse { + pub success: bool, +} + +#[derive(Debug, Serialize)] +pub enum KillInferenceJobsError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for KillInferenceJobsError { + fn status_code(&self) -> StatusCode { + match *self { + KillInferenceJobsError::BadInput(_) => StatusCode::BAD_REQUEST, + KillInferenceJobsError::NotFound => StatusCode::NOT_FOUND, + KillInferenceJobsError::NotAuthorized => StatusCode::UNAUTHORIZED, + KillInferenceJobsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for KillInferenceJobsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn kill_generic_inference_jobs_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + KillInferenceJobsError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(KillInferenceJobsError::NotAuthorized); + } + }; + + // TODO: We don't have a permission for this, so use this as a proxy permission + if !user_session.can_ban_users { + warn!("no permission to edit categories"); + return Err(KillInferenceJobsError::NotAuthorized); + } + + let job_statuses = request.job_statuses.iter() + .map(|status| match status { + KillableStatus::Failed => kill_generic_inference_jobs::KillableStatus::Failed, + KillableStatus::Pending => kill_generic_inference_jobs::KillableStatus::Pending, + KillableStatus::Started => kill_generic_inference_jobs::KillableStatus::Started, + }) + .collect::>(); + + let target = match request.target { + KillableTarget::AllJobs => kill_generic_inference_jobs::KillableTarget::AllJobs, + KillableTarget::Category(category) => kill_generic_inference_jobs::KillableTarget::Category(category), + KillableTarget::ModelType(model_type) => kill_generic_inference_jobs::KillableTarget::ModelType(model_type), + }; + + kill_generic_inference_jobs(KillGenericInferenceJobsArgs { + job_statuses, + target, + maybe_priority_or_lower: request.maybe_priority_or_lower, + mysql_pool: &server_state.mysql_pool, + }).await.map_err(|err| { + warn!("Query Error: {:?}", err); + KillInferenceJobsError::ServerError + })?; + + let response = KillInferenceJobsResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| KillInferenceJobsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs index 2ee8605d27..e39b062b03 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs @@ -1,3 +1,4 @@ pub mod get_inference_job_status; pub mod get_pending_inference_job_count; +pub mod kill_inference_jobs; pub mod terminate_inference_job_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9b6f920061..071b76c46d 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -37,6 +37,7 @@ use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_re use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; use crate::http_server::endpoints::inference_job::get_pending_inference_job_count::get_pending_inference_job_count_handler; +use crate::http_server::endpoints::inference_job::kill_inference_jobs::kill_generic_inference_jobs_handler; use crate::http_server::endpoints::inference_job::terminate_inference_job_handler::terminate_inference_job_handler; use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler; use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler; @@ -355,6 +356,11 @@ fn add_moderator_routes (app: App) -> App .route(web::get().to(get_w2l_inference_queue_count_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/kill_generic") + .route(web::post().to(kill_generic_inference_jobs_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::scope("/pending") From ae5eea53fe67f3a5b619f080a23cdddc42662b09 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 03:57:40 -0500 Subject: [PATCH 055/136] Something wrong with the query parameters not passing off the limits etc --- crates/service/web/storyteller_web/src/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 6c06107519..33130372c3 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1251,7 +1251,7 @@ fn add_weights_routes(app: App) -> App ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) .route( - "/by_user_and_type/{username}/{type}", + "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) ) From 28770ef736fba36f2bf4952e1bdf93fa4db500e2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 4 Dec 2023 05:00:46 -0500 Subject: [PATCH 056/136] fix query issue --- .../generic_inference/web/kill_generic_inference_jobs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs index 4928fcd6f4..5ac8ae9155 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/kill_generic_inference_jobs.rs @@ -81,7 +81,7 @@ WHERE status IN( separated.push_unseparated(") "); if let Some(priority) = maybe_priority_or_lower { - query_builder.push(format!(" AND priority <= {priority} ")); + query_builder.push(format!(" AND priority_level <= {priority} ")); } match target { @@ -136,7 +136,7 @@ mod tests { let statuses = HashSet::from([KillableStatus::Failed]); let builder = query_builder(statuses, KillableTarget::AllJobs, Some(10)); let query = cleanup_query(&builder.into_sql()); - assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND priority <= 10".to_string()); + assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND priority_level <= 10".to_string()); } #[test] From b55435d99287ca011f94a731954449069749b037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:30:06 +0000 Subject: [PATCH 057/136] Bump rustix from 0.38.25 to 0.38.26 Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.38.25 to 0.38.26. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Commits](https://github.com/bytecodealliance/rustix/compare/v0.38.25...v0.38.26) --- updated-dependencies: - dependency-name: rustix dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 124 +++++++++++++++++++++++++++-------- my-workspace-hack/Cargo.toml | 8 +-- 2 files changed, 99 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..62ac6ff06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,7 +521,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -531,7 +531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2196,12 +2196,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2220,7 +2220,7 @@ checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ "cfg-if 1.0.0", "home", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2714,7 +2714,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3079,7 +3079,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3600,7 +3600,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3736,7 +3736,7 @@ dependencies = [ "url", "void", "winapi 0.3.9", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4131,7 +4131,7 @@ dependencies = [ "libc", "redox_syscall 0.4.1", "smallvec 1.11.2", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -4841,7 +4841,7 @@ dependencies = [ "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4951,15 +4951,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -5061,7 +5061,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -5514,7 +5514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6231,7 +6231,7 @@ dependencies = [ "fastrand 2.0.1", "redox_syscall 0.4.1", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6405,7 +6405,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.5", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -7419,7 +7419,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -7428,7 +7428,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -7437,13 +7446,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -7452,42 +7476,84 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -7495,7 +7561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if 1.0.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] diff --git a/my-workspace-hack/Cargo.toml b/my-workspace-hack/Cargo.toml index 2f062e5a69..22f41a903d 100644 --- a/my-workspace-hack/Cargo.toml +++ b/my-workspace-hack/Cargo.toml @@ -142,22 +142,22 @@ url = { version = "2.4.1", features = ["serde"] } [target.x86_64-unknown-linux-gnu.dependencies] openssl = { version = "0.10.59", features = ["vendored"] } openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-unknown-linux-gnu.build-dependencies] openssl = { version = "0.10.59", features = ["vendored"] } openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } [target.x86_64-apple-darwin.dependencies] -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-apple-darwin.build-dependencies] -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } From ac13be29009ae7ecc390948ee1cec4da2d634c9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:30:26 +0000 Subject: [PATCH 058/136] Bump lettre from 0.11.1 to 0.11.2 Bumps [lettre](https://github.com/lettre/lettre) from 0.11.1 to 0.11.2. - [Release notes](https://github.com/lettre/lettre/releases) - [Changelog](https://github.com/lettre/lettre/blob/master/CHANGELOG.md) - [Commits](https://github.com/lettre/lettre/compare/v0.11.1...v0.11.2) --- updated-dependencies: - dependency-name: lettre dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 18 ++++-------------- crates/lib/email_sender/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..ffe82d3765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2907,16 +2907,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -3215,9 +3205,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "lettre" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a466bc111374ccf4d90877dba636924a2185e67e5be4b35d32043199365097b2" +checksum = "a48c2e9831b370bc2d7233c2620298c45f3a158ed6b4b8d7416b2ada5a268fd8" dependencies = [ "base64 0.21.5", "chumsky", @@ -3227,7 +3217,7 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna 0.4.0", + "idna", "mime", "native-tls", "nom", @@ -6935,7 +6925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", "serde", ] diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index e640833ea8..323b40abd4 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -11,5 +11,5 @@ path = "src/lib.rs" [dependencies] errors = { path = "../errors" } -lettre = "0.11.0" +lettre = "0.11.2" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } From 8c7b10015039937d53f25a0c44ec004fa79f7663 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 18:06:11 -0500 Subject: [PATCH 059/136] Adding new seed then have to switch tests back. --- crates/cli/dev_database_seed/src/main.rs | 1 - .../src/seeding/model_weights.rs | 392 +++++++++++++++--- 2 files changed, 327 insertions(+), 66 deletions(-) diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index ed29579867..0288c5aca8 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -61,7 +61,6 @@ pub async fn main() -> AnyhowResult<()> { seed_voice_conversion(&pool).await?; seed_weights(&pool).await?; - seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 07c8f2f155..2de742f766 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -18,36 +18,79 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; -pub async fn seed_weights_for_paging( - mysql_pool: &Pool) -> AnyhowResult<()> -{ +pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { + let sd1_5_markdown_description = r#" +# Dragonfruit AI Models Update and Workflow - let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { - None => { - return Err(anyhow!("could not find user hanashi")); - } - Some(token) => token, - }; +## Last Update for Dragonfruit Models +- **Note:** This might be the last update for a while, focusing next on a realistic merge (WIP). +- **Future Plans:** Possible update to Z.5D v2 after completing the realistic merge. + +## New Models Announcement +- **Upcoming Models:** Separate from the Dragonfruit branch, so stay tuned! + +## DragonfruitGT v1 (Z v2) +- **Focus:** More fantasy, richer color range, improved landscapes, more masculine men. +- **Alternative:** Use DragonfruitOG v2.0 for thicker lines, simpler colors. + +## GT v1.0 (Z v2.0) Features +- Improved backgrounds +- Deeper contrast and color +- Bolder lines and more detail + +### Settings for GT +- Steps: 30-50 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +## Workflow for Images +1. **BerryMix VAE:** Ensure correct color. +2. **Adetailer Extension:** Improve faces, bodies, and hands. Also available on Civitai and Huggingface. +3. **Hires Fix:** Denoising strength 0.3 - 0.35, using upscaler "x_NMKD-Superscale-SP_178000_G". +4. **Img2img Transfer:** Use denoising strength 0.3-0.35 with the same prompt and settings. Adjust height after hires. +5. **Second Img2img Transfer:** Low denoising strength 0.1-3.0. Use SD upscale script with "RealESRGAN_x4plus_anime_6B" sampler for size increase. + +## DragonfruitZ.5D v1.0 +- **Style:** 2.5D model, merging DragonfruitZ with realistic elements. +- **Features:** Skin color variety, masculine men, landscapes, fantasy, NSFW content, detailed faces. + +### Settings for Z.5D +- Steps: 30 +- Clip skip: 2 +- Upscaler: Euler a +- Extensions: Adetailer, VAE: berrysmix + +## DragonfruitOG v2.0 +- **Refinement:** Of the original Dragonfruit model. +- **Style:** Bold, clean lines and bold colors with decreased detail for simplicity. +- **Improvements:** Teeth, skin color variety, landscapes, fantasy elements, NSFW content. + +## DragonfruitZ v1.0 +- **Alternative Version:** More fantasy and color range, better landscapes, more masculine men. +- **Suggestion:** Use Dragonfruit (Unisex) v2 for thicker lines and simpler colors. + +## Dragonfruit OG v1.0 +- **Focus:** More unisex, suitable for creating male characters. +- **Personal Note:** First merge, aimed at 2D, flat, colorful, thick lined styles. + +### Personal Settings +- Steps: 20-30 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +*Thank you for your continued support and feedback!* +"#; + +let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + let user_token = user_token; // create a loop that loops from 1 to 100 for i in 1..=100 { // create a new weight - let model_weight_token = ModelWeightToken(format!("item:{}", i)); - let title = format!("Title {}", i); - let description = format!("Description {}", i); - let description_rendered_html = format!("

{}

", description); - let original_filename = format!("filename{}.txt", i); - let original_download_url = format!("http://example.com/download{}", i); - let thumbnail_token = format!("Thumbnail {}", i); - let private_bucket_hash = format!("bucket_hash{}", i); - let private_bucket_prefix = format!("_fake{}", i); - let private_bucket_extension = format!("rvc{}", i); - let cached_user_ratings_total_count = i; - let cached_user_ratings_positive_count = i; - let cached_user_ratings_negative_count = i; - let cached_user_ratings_ratio = i as u32 / 100; - let version = i as i32; - + let weights_category = match i { 1..=20 => WeightsCategory::VoiceConversion, 21..=40 => WeightsCategory::TextToSpeech, @@ -66,37 +109,250 @@ pub async fn seed_weights_for_paging( _ => WeightsType::LoRA }; - let args = CreateModelWeightsArgs { - token: &model_weight_token, // replace with actual ModelWeightToken - weights_type: weights_types, // replace with actual WeightsType - weights_category: weights_category, // replace with actual WeightsCategory - title: title, - maybe_thumbnail_token: Some(thumbnail_token), - description_markdown: description, - description_rendered_html: description_rendered_html, - creator_user_token: Some(&user_token), // replace with actual UserToken - creator_ip_address: "192.168.1.1", - creator_set_visibility: Visibility::Public, - maybe_last_update_user_token: Some(user_token.to_string()), - original_download_url: Some(original_download_url), - original_filename: Some(original_filename), - file_size_bytes: 1024, - file_checksum_sha2: "checksum1".to_string(), - private_bucket_hash: "bucket_hash1".to_string(), - maybe_private_bucket_prefix: Some("_fake".to_string()), - maybe_private_bucket_extension: Some("rvc".to_string()), - cached_user_ratings_total_count: cached_user_ratings_total_count, - cached_user_ratings_positive_count: cached_user_ratings_positive_count, - cached_user_ratings_negative_count: cached_user_ratings_ratio, - maybe_cached_user_ratings_ratio: Some(0.9), - version: version, - mysql_pool: &mysql_pool, // replace with actual MySqlPool - }; + let mut model_weight_token; + let mut title; + let mut description; + let mut description_rendered_html; + let mut original_filename; + let mut original_download_url; + let mut thumbnail_token; + + let mut private_bucket_hash:String = "".to_string(); + let mut private_bucket_prefix; + let mut private_bucket_extension; - create_weight(args).await?; + let mut cached_user_ratings_total_count; + let mut cached_user_ratings_positive_count; + let mut cached_user_ratings_negative_count; + let mut cached_user_ratings_ratio; + let mut version; + + match i { + 1..=20 => { + // voice conversion model + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding RVCv2 model {}", i); + }, + 21..=40 =>{ + // TTS + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + let tts_markdown_description = r#" + # HifiGAN Tacotron2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Goku MOS 4 + "#; + + description_rendered_html = tts_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("tt2"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding TT2 model {}", i); + }, + 41..=60 => { + // SD 1.5 + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = sd1_5_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; //sd1_5_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sd15"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SD15 model {}", i); + }, + 61..=80 => { + // SD XL + let sdxl_markdown_description = r#" + # Special Edition Release Announcement + + ## What's New? + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + + ## Tips for Use + 1. **Simple Prompts:** Avoid adding words like "detailed, realistic..." or specifying a specific artist. Simply describe what you want to see. + 2. **CFG Scale:** A range between 3 and 8.5 is effective (recommended: 7). + 3. **Minimum Steps:** Ensure at least 36 steps for optimal results. + 4. **Compatibility:** + - Works excellently with Hires fix. + - Great results with unaestheticXLv31 embedding, although sometimes better without it. + 5. **Sampling Method:** Either DPM++ 2M SDE Karras or DPM++ 2M Karras. + + Have fun! 😊 + "#; + let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("SDXL_Niji_Special Edition: {}", i); + description = format!("Description Number {}:", i); + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sdxl"); + + description_rendered_html = sdxl_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SDXL model {}", i); + }, + 81..=100 => { + // LoRA + let lora_markdown_description = r#" + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - LoRA Gawr Gura + "#; + + let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg"; + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura LoRA: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = lora_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; // lora_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("loRA"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding LoRA model {}", i); + }, + _ => { + // We went out of range so have more gwar gura + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + + println!("Out of range"); + } + } + + let args = CreateModelWeightsArgs { + token: &model_weight_token, // replace with actual ModelWeightToken + weights_type: weights_types, // replace with actual WeightsType + weights_category: weights_category, // replace with actual WeightsCategory + title: title, + maybe_thumbnail_token: Some(thumbnail_token.to_string()), + description_markdown: description, + description_rendered_html: description_rendered_html.to_string(), + creator_user_token: Some(&user_token), // replace with actual UserToken + creator_ip_address: "192.168.1.1", + creator_set_visibility: Visibility::Public, + maybe_last_update_user_token: Some(user_token.to_string()), + original_download_url: Some(original_download_url), + original_filename: Some(original_filename), + file_size_bytes: 1024, + file_checksum_sha2: "checksum1".to_string(), + private_bucket_hash: "bucket_hash1".to_string(), + maybe_private_bucket_prefix: Some(private_bucket_prefix), + maybe_private_bucket_extension: Some(private_bucket_extension), + cached_user_ratings_total_count: cached_user_ratings_total_count, + cached_user_ratings_positive_count: cached_user_ratings_positive_count, + cached_user_ratings_negative_count: cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32), + version: version, + mysql_pool: &mysql_pool, // replace with actual MySqlPool + }; + + create_weight(args).await; } Ok(()) - } pub async fn seed_weights_for_user_token( mysql_pool: &Pool, @@ -248,16 +504,9 @@ pub async fn seed_weights_for_user_token( } Ok(()) } -pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { - info!("Seeding weights..."); - let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { - None => { - return Err(anyhow!("could not find user hanashi")); - } - Some(token) => token, - }; +pub async fn original_seed_weights(mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { let model_weight_token1 = ModelWeightToken("1".to_string()); let creator_token1 = UserToken("creatorToken1".to_string()); @@ -410,9 +659,22 @@ pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { create_weight(model_weights_arg).await?; } - // adds this to HANASHI - get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await?; - seed_weights_for_user_token(mysql_pool, user_token).await?; - seed_weights_for_paging(mysql_pool).await?; + Ok(()) + +} +pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { + info!("Seeding weights..."); + + let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { + None => { + return Err(anyhow!("could not find user hanashi")); + } + Some(token) => token, + }; + + //original_seed_weights(mysql_pool,user_token).await?; + //seed_weights_for_user_token(mysql_pool, user_token).await?; + seed_weights_for_paging(mysql_pool,user_token).await?; + Ok(()) } From 91d0a1da8f02692b4f1bae743cd7ca8834c253fc Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 18:21:43 -0500 Subject: [PATCH 060/136] Tests pass again --- .../list_weights_query_builder.rs | 4 +-- .../src/queries/model_weights/tests.rs | 36 +++++++++---------- .../service/web/storyteller_web/src/routes.rs | 1 + 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs index e1d79b89ab..d1d18302b9 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs @@ -505,7 +505,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_type() { - let query_builder = ListWeightsQueryBuilder::new().weights_type(WeightsType::RvcV2); + let query_builder = ListWeightsQueryBuilder::new().weights_type(Some(WeightsType::RvcV2)); assert_eq!( &query_builder.build_predicates(), " WHERE model_weights.weights_type = ? \ @@ -520,7 +520,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_category() { let query_builder = ListWeightsQueryBuilder::new().weights_category( - WeightsCategory::VoiceConversion + Some(WeightsCategory::VoiceConversion) ); println!("Query HERE! {}", &query_builder.build_predicates()); assert_eq!( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 9b054e4b30..52dc725ed6 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -360,8 +360,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); @@ -372,8 +372,8 @@ mod tests { assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) .offset(Some(10)) @@ -384,8 +384,8 @@ mod tests { assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) .offset(Some(10)) @@ -409,8 +409,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -429,8 +429,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -460,8 +460,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -478,8 +478,8 @@ mod tests { let qb = ListWeightsQueryBuilder::new() .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .limit(10) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false) @@ -576,8 +576,8 @@ mod tests { // try a query with all the filters let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::Tacotron2) - .weights_category(WeightsCategory::TextToSpeech) + .weights_type(Some(WeightsType::Tacotron2)) + .weights_category(Some(WeightsCategory::TextToSpeech)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); @@ -587,8 +587,8 @@ mod tests { assert_eq!(result.weights.len(), 2); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 33130372c3..91efc45771 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1254,5 +1254,6 @@ fn add_weights_routes(app: App) -> App "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) + .route("/list/{username}/{weights_category}/{weights_type}",list_weights_handler) ) } From 2e3ff8a73024e441b90e595fec5992bb0d2edaa3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 4 Dec 2023 20:34:39 -0500 Subject: [PATCH 061/136] list media files for user --- .../media_files/list_media_files_for_user.rs | 203 +++++++++++++++++ .../src/queries/media_files/mod.rs | 11 +- .../media_files/list_media_files_for_user.rs | 215 ++++++++++++++++++ .../http_server/endpoints/media_files/mod.rs | 3 +- .../service/web/storyteller_web/src/routes.rs | 5 + 5 files changed, 431 insertions(+), 6 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs new file mode 100644 index 0000000000..1c7bcdee62 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -0,0 +1,203 @@ +use chrono::{DateTime, Utc}; +use sqlx::{MySql, MySqlPool, QueryBuilder}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; + +pub struct MediaFileListPage { + pub records: Vec, + + pub sort_ascending: bool, + + /// ID of the first record. + pub first_id: Option, + + /// ID of the last record. + pub last_id: Option, +} + +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Clone, Copy)] +pub enum ViewAs { + Author, + Moderator, + AnotherUser, +} + +pub struct ListMediaFileForUserArgs<'a> { + pub username: &'a str, + pub limit: usize, + pub maybe_filter_media_type: Option, + pub maybe_offset: Option, + pub cursor_is_reversed: bool, + pub view_as: ViewAs, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult { + + let mut query = query_builder( + args.maybe_filter_media_type, + args.username, + args.limit, + args.maybe_offset, + args.cursor_is_reversed, + args.view_as, + ); + + let query = query.build_query_as::(); + + let results = query.fetch_all(args.mysql_pool).await?; + + let first_id = results.first() + .map(|raw_result| raw_result.id); + + let last_id = results.last() + .map(|raw_result| raw_result.id); + + let results = results.into_iter() + .map(|record| { + MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + Ok(MediaFileListPage { + records: results, + sort_ascending: !args.cursor_is_reversed, + first_id, + last_id, + }) +} + +fn query_builder<'a>( + maybe_filter_media_type: Option, + username: &'a str, + limit: usize, + maybe_offset: Option, + cursor_is_reversed: bool, + view_as: ViewAs, +) -> QueryBuilder<'a, MySql> { + + // NB: Query cannot be statically checked by sqlx + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +SELECT + m.id, + m.token as `token: tokens::tokens::media_files::MediaFileToken`, + + m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + m.origin_product_category as `origin_product_category: enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory`, + + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users AS u + ON m.maybe_creator_user_token = u.token + +WHERE u.username = ? +AND m.user_deleted_at IS NULL +AND m.mod_deleted_at IS NULL + "# + ); + + query_builder.push_bind(username); + + if let Some(media_type) = maybe_filter_media_type { + query_builder.push(" AND m.media_file_type = ? "); + query_builder.push_bind(media_type); + } + + match view_as { + ViewAs::Author => {} + ViewAs::Moderator => {} + ViewAs::AnotherUser => { + query_builder.push(" AND m.creator_set_visibility = 'public' "); + } + } + + if let Some(offset) = maybe_offset { + if cursor_is_reversed { + query_builder.push(format!(" AND m.id < {offset} ")); + } else { + query_builder.push(format!(" AND m.id > {offset} ")); + } + } + + if cursor_is_reversed { + query_builder.push(" ORDER BY m.id ASC "); + } else { + query_builder.push(" ORDER BY m.id DESC "); + } + + query_builder.push(format!(" LIMIT {limit} ")); + + query_builder +} + +#[derive(sqlx::FromRow)] +struct MediaFileListItemInternal { + id: i64, + token: MediaFileToken, + + origin_category: MediaFileOriginCategory, + origin_product_category: MediaFileOriginProductCategory, + maybe_origin_model_type: Option, + maybe_origin_model_token: Option, + + media_type: MediaFileType, + public_bucket_directory_hash: String, + maybe_public_bucket_prefix: Option, + maybe_public_bucket_extension: Option, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 3225ced58e..619f018ce5 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -1,7 +1,8 @@ -pub mod get_media_file; -pub mod insert_media_file_from_face_animation; -pub mod insert_media_file_from_zero_shot_tts; -pub mod insert_media_file_from_cli_tool; -pub mod insert_media_file_from_voice_conversion; pub mod delete_media_file; +pub mod get_media_file; +pub mod insert_media_file_from_cli_tool; +pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_voice_conversion; +pub mod insert_media_file_from_zero_shot_tts; +pub mod list_media_files_for_user; pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs new file mode 100644 index 0000000000..35ae3f074c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -0,0 +1,215 @@ +use std::sync::Arc; + +use actix_web::{HttpMessage, HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Query}; +use chrono::{DateTime, Utc}; +use log::{info, warn}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct PathInfo { + username: String, +} + +#[derive(Deserialize)] +pub struct QueryParams { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, + pub filter_media_type: Option, +} + +#[derive(Serialize)] +pub struct SuccessResponse { + pub success: bool, + pub results: Vec, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize)] +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ErrorResponse { + ServerError, +} + +impl ResponseError for ErrorResponse { + fn status_code(&self) -> StatusCode { + match *self { + ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ErrorResponse::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +impl std::fmt::Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_media_files_for_user_handler( + http_request: HttpRequest, + path: Path, + query: Query, + server_state: web::Data> +) -> Result +{ + info!("Fetching inference results for user: {}", &path.username); + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ErrorResponse::ServerError + })?; + + let mut is_author = false; + let mut is_mod = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_author = session.username == path.username; + is_mod = session.can_ban_users; + }, + }; + + // TODO(bt,2023-12-04): Enforce real maximums and defaults + let limit = query.limit.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor as usize) + } else { + None + }; + + let view_as = if is_author { + ViewAs::Author + } else if is_mod { + ViewAs::Moderator + } else { + ViewAs::AnotherUser + }; + + let query_results = list_media_files_for_user(ListMediaFileForUserArgs { + username: &path.username, + limit, + maybe_filter_media_type: query.filter_media_type, + maybe_offset: cursor, + cursor_is_reversed, + view_as, + mysql_pool: &server_state.mysql_pool, + }).await; + + + let results_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ErrorResponse::ServerError); + } + }; + + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let results = results_page.records.into_iter() + .map(|record| MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + }) + .collect::>(); + + let response = SuccessResponse { + success: true, + results, + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ErrorResponse::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 014fa68854..6af4273b7a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; -pub mod delete_media_file; +pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..9aafd5340c 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -44,6 +44,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -1006,6 +1007,10 @@ fn add_media_file_routes (app: App) -> App .route(web::delete().to(delete_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/user/{username}") + .route(web::get().to(list_media_files_for_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } From 130e0daa084ddb2c5068c95ffdefadcfd83c85d7 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 4 Dec 2023 22:49:33 -0500 Subject: [PATCH 062/136] paging works correctly. --- .../src/queries/model_weights/tests.rs | 389 +++++++++++++----- 1 file changed, 290 insertions(+), 99 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 2a24c7fda3..917fdd98b3 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -3,8 +3,9 @@ mod tests { use std::result; use anyhow::Ok; - use enums::by_table::model_weights::{weights_types, weights_category}; - use sqlx::{MySqlPool, Acquire}; + use chrono::offset; + use enums::by_table::model_weights::{ weights_types, weights_category }; + use sqlx::{ MySqlPool, Acquire }; use rand::Rng; use tokio; @@ -24,7 +25,12 @@ mod tests { use crate::queries::model_weights::create_weight::create_weight; use crate::queries::model_weights::get_weight::get_weight_by_token; - use crate::queries::model_weights::delete_weights::{ delete_weights_as_user, delete_weights_as_mod, undelete_weights_as_mod , undelete_weights_as_user}; + use crate::queries::model_weights::delete_weights::{ + delete_weights_as_user, + delete_weights_as_mod, + undelete_weights_as_mod, + undelete_weights_as_user, + }; use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; @@ -45,23 +51,19 @@ mod tests { pub async fn delete_all_weights_for_table(mysql_pool: &MySqlPool) -> AnyhowResult<()> { // write a query that deletes all weights - let _r = sqlx::query("DELETE FROM model_weights") - .execute(mysql_pool) - .await?; + let _r = sqlx::query("DELETE FROM model_weights").execute(mysql_pool).await?; Ok(()) } #[tokio::test] #[serial] async fn test_create_weights() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); let random_number: u32 = rng.gen(); let model_weight_token1 = ModelWeightToken(random_number.to_string()); - let creator_token1 = UserToken("creatorToken!1".to_string()); let args = CreateModelWeightsArgs { @@ -131,7 +133,6 @@ mod tests { #[tokio::test] #[serial] async fn test_delete_and_undelete_weights_user() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); @@ -170,7 +171,7 @@ mod tests { create_weight(args).await?; delete_weights_as_user(&model_weight_token1, &pool).await?; - + let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -261,7 +262,6 @@ mod tests { } } - undelete_weights_as_mod(&model_weight_token1, &pool).await?; let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -285,13 +285,22 @@ mod tests { #[serial] async fn list_all_weights_by_user() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); let can_see_deleted = true; - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,5,&creator_username).await?; + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 5, + &creator_username + ).await?; - let weights_by_username = list_weights_by_creator_username(&pool, &creator_username, can_see_deleted).await?; + let weights_by_username = list_weights_by_creator_username( + &pool, + &creator_username, + can_see_deleted + ).await?; for weight in weights_by_username.iter() { // print weight println!("weight: {:?}", weight.creator_username); @@ -301,23 +310,29 @@ mod tests { Ok(()) } - async fn seed_weights_with_category_and_type(weights_type:WeightsType,weights_category:WeightsCategory,number_of_items:u32,creator_username:&str) -> AnyhowResult<()> { - + async fn seed_weights_with_category_and_type( + weights_type: WeightsType, + weights_category: WeightsCategory, + number_of_items: u32, + creator_username: &str + ) -> AnyhowResult<()> { let pool = setup().await; - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..number_of_items { - let mut rng = rand::thread_rng(); let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -346,7 +361,6 @@ mod tests { } Ok(()) - } // Tests paging for the list weights query builder @@ -356,40 +370,45 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,20,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 20, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); - + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + let result = qb.perform_query_for_page(&pool).await?; - + // let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; @@ -398,27 +417,31 @@ mod tests { Ok(()) } - #[tokio::test] #[serial] async fn list_weights_query_build_test_asc_desc_cursor_reverse() -> AnyhowResult<()> { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true) - .cursor_is_reversed(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .cursor_is_reversed(false); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -428,19 +451,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false) - .cursor_is_reversed(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false) + .cursor_is_reversed(true); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -456,18 +478,23 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -477,19 +504,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -503,11 +529,14 @@ mod tests { #[serial] async fn list_weights_query_build_test() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..6 { @@ -519,7 +548,7 @@ mod tests { WeightsType::RvcV2 } else if i % 5 == 3 { WeightsType::StableDiffusionXL - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsType::SoVitsSvc } else { WeightsType::StableDiffusion15 @@ -533,7 +562,7 @@ mod tests { WeightsCategory::VoiceConversion } else if i % 5 == 3 { WeightsCategory::ImageGeneration - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsCategory::VoiceConversion } else { WeightsCategory::ImageGeneration @@ -543,10 +572,10 @@ mod tests { let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -581,18 +610,18 @@ mod tests { .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); - let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 2); let qb = ListWeightsQueryBuilder::new() - .weights_type(Some(WeightsType::LoRA)) - .weights_category(Some(WeightsCategory::ImageGeneration)) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 1); assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); assert_eq!(result.weights[0].weights_type, WeightsType::LoRA); @@ -600,6 +629,168 @@ mod tests { Ok(()) } - -} + #[tokio::test] + #[serial] + async fn list_weights_query_builder_page_type_and_category() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + // using the seeded weights from the previous test + let creator_username = "hanashi".to_string(); + for i in 0..4 { + let weights_types = vec![ + WeightsType::HifiganTacotron2, + WeightsType::LoRA, + WeightsType::RvcV2, + WeightsType::StableDiffusionXL + ]; + + let weights_type = weights_types[i as usize]; + + let qb = ListWeightsQueryBuilder::new() + .weights_type(Some(weights_type)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + println!("weights_type: {:?}", weights_type); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 20, "Should be 20 weights"); + assert_eq!( + result.weights[0].weights_type, + weights_type, + "Should be the same weights type" + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_weight_query_builder_all() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .limit(11); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 11); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_limits() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), 10); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + let result = qb.perform_query_for_page(&pool).await?; + + // by default 25 + assert_eq!(result.weights.len(), 25); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + // over query limit + let qb: ListWeightsQueryBuilder = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(80); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 60); + // pick a random value in here + assert_eq!(result.weights[9].weights_category, WeightsCategory::ImageGeneration); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn page_across_a_category() -> AnyhowResult<()> { + // paging across a category and it works + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + // how we implement paging. + let page:u16 = 0; + let page_size: u16 = 20; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .sort_ascending(true) + .limit(page_size); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let page = 1; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .sort_ascending(true) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + let page = 2; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + Ok(()) + } +} From b9f604dd1875dc4ffac0e7920fffdff26f0c5892 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 4 Dec 2023 23:11:41 -0500 Subject: [PATCH 063/136] reusable pagination types --- .../src/http_server/common_responses/mod.rs | 2 + .../common_responses/pagination_cursors.rs | 53 +++++++++++++++++++ .../common_responses/pagination_page.rs | 33 ++++++++++++ .../media_files/list_media_files_for_user.rs | 11 ++-- 4 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs index 1aa45ef781..d84a319a04 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs @@ -1 +1,3 @@ +pub mod pagination_cursors; +pub mod pagination_page; pub mod user_details_lite; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs new file mode 100644 index 0000000000..c801faea59 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs @@ -0,0 +1,53 @@ +/// Pagination by cursors +/// These types of pagination are for "infinite scrolling", which do not reveal a number of pages. +/// This is good so that investors and competitors cannot reveal how many database records we have. +/// This is typically used for "discovery" type pages, not user profiles. +/// +/// See `PaginationPage` for the other type of pagination. +#[derive(Serialize)] +pub struct PaginationCursors { + + /// The "next" cursor. + /// This is an opaque (typically even encrypted) handle. + pub maybe_next: Option, + + /// The "previous" cursor. + /// This is an opaque (typically even encrypted) handle. + pub maybe_previous: Option, + + /// Details whether we're walking forward or backward. + pub cursor_is_reversed: bool, +} + +#[cfg(test)] +mod tests { + use crate::http_server::common_responses::pagination_cursors::PaginationCursors; + + #[test] + fn test_serialize_empty_json() { + // Make sure this interface doesn't change. + let pagination = PaginationCursors { + maybe_next: None, + maybe_previous: None, + cursor_is_reversed: false, + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"maybe_next\":null,\"maybe_previous\":null,\"cursor_is_reversed\":false}") + } + + #[test] + fn test_serialize_full_json() { + // Make sure this interface doesn't change. + let pagination = PaginationCursors { + maybe_next: Some("foo".to_string()), + maybe_previous: Some("bar".to_string()), + cursor_is_reversed: true, + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"maybe_next\":\"foo\",\"maybe_previous\":\"bar\",\"cursor_is_reversed\":true}") + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs new file mode 100644 index 0000000000..3e64af9626 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs @@ -0,0 +1,33 @@ +/// Pagination by page id +/// This type of pagination is by "page id" and "page count". +/// This should never be used to walk the entire public database as competitors and investors +/// can use it to glean information about the scale of our service. +/// This is best used for user profiles and scoped down results. +/// +/// See `PaginationCursors` for the other type of pagination. +#[derive(Serialize)] +pub struct PaginationPage { + /// The current page number + pub current: u64, + + /// How many pages there are in total + pub total_page_count: u64, +} + +#[cfg(test)] +mod tests { + use crate::http_server::common_responses::pagination_page::PaginationPage; + + #[test] + fn test_serialize() { + let pagination = PaginationPage { + current: 5, + total_page_count: 123, + + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"current\":5,\"total_page_count\":123}") + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs index 35ae3f074c..9cfe696a27 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -15,6 +15,7 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs}; use tokens::tokens::media_files::MediaFileToken; +use crate::http_server::common_responses::pagination_cursors::PaginationCursors; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -36,8 +37,7 @@ pub struct QueryParams { pub struct SuccessResponse { pub success: bool, pub results: Vec, - pub cursor_next: Option, - pub cursor_previous: Option, + pub pagination: PaginationCursors, } #[derive(Serialize)] @@ -202,8 +202,11 @@ pub async fn list_media_files_for_user_handler( let response = SuccessResponse { success: true, results, - cursor_next, - cursor_previous, + pagination: PaginationCursors { + maybe_next: cursor_next, + maybe_previous: cursor_previous, + cursor_is_reversed, + } }; let body = serde_json::to_string(&response) From f99b5a92b81ccb6e9a6ce0f08fb819e67d41e81d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 13:44:50 -0500 Subject: [PATCH 064/136] Added weights paging to michael's spec missing the fields. --- .../endpoints/weights/list_availible_weights.rs | 17 +++++++++++------ .../service/web/storyteller_web/src/routes.rs | 1 - 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 26257fc60a..7587931d3b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -22,22 +22,23 @@ use crate::server_state::ServerState; #[derive(Deserialize)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, - pub limit: Option, - pub cursor: Option, - pub cursor_is_reversed: Option, + pub page_size: u16, + pub page_index : u16, } #[derive(Serialize)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, + pub page_size: u16, + pub page_index : u16, pub cursor_next: Option, pub cursor_previous: Option, } #[derive(Deserialize)] pub struct ListWeightsByPathInfo { - pub username: String, + pub username: Option, pub weights_type: Option, pub weights_category: Option, } @@ -75,6 +76,11 @@ pub struct ModelWeightForList { pub creator_username: String, pub creator_display_name: String, pub creator_email_gravatar_hash: String, + + // additional fields to be added when tables are around + pub likes: u32, + pub bookmarks: bool, + } #[derive(Debug)] @@ -123,7 +129,7 @@ pub async fn list_availible_weights_handler( } }; - let limit = query.limit.unwrap_or(25); + let limit = query.page_size.unwrap_or(10); let sort_ascending = query.sort_ascending.unwrap_or(false); let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); @@ -236,7 +242,6 @@ pub async fn list_availible_weights_handler( file_size_bytes:weights.file_size_bytes, file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9f202a55d7..0ef5dbd81a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1260,6 +1260,5 @@ fn add_weights_routes(app: App) -> App "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", web::get().to(list_availible_weights_handler) ) - .route("/list/{username}/{weights_category}/{weights_type}",list_weights_handler) ) } From 49934107405a04e9e7a9e6ae4bbd4abc99649998 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 14:54:32 -0500 Subject: [PATCH 065/136] Built needs a simple endpoint query test --- .../weights/list_availible_weights.rs | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 7587931d3b..044b05b113 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -5,7 +5,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use chrono::{ DateTime, Utc }; use log::{ info, warn }; - +use rand::Rng; use enums::common::visibility::Visibility; use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; @@ -129,21 +129,9 @@ pub async fn list_availible_weights_handler( } }; - let limit = query.page_size.unwrap_or(10); + let limit = query.page_size; let sort_ascending = query.sort_ascending.unwrap_or(false); - let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); - - let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; let include_user_hidden = is_mod; @@ -185,12 +173,10 @@ pub async fn list_availible_weights_handler( .include_user_deleted_results(is_mod) .include_mod_deleted_results(is_mod) .limit(limit) - .cursor_is_reversed(cursor_is_reversed) - .offset(cursor); + .offset(Some((query.page_index as u64) * (query.page_size as u64))); let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - let weights_page = match query_results { Ok(results) => results, Err(e) => { @@ -221,6 +207,9 @@ pub async fn list_availible_weights_handler( None }; + let mut rng = rand::thread_rng(); + let random_float = rng.gen_range(0.0..1.0); + let random_bool = random_float >= 0.5; // generate parse a response let response = ListAvailibleWeightsSuccessResponse { @@ -242,7 +231,6 @@ pub async fn list_availible_weights_handler( file_size_bytes:weights.file_size_bytes, file_checksum_sha2: weights.file_checksum_sha2, - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, @@ -255,10 +243,15 @@ pub async fn list_availible_weights_handler( creator_username: weights.creator_username, creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash + creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + + bookmarks: random_bool, + likes: rng.gen_range(0..1000), }).collect::>(), cursor_next, cursor_previous, + page_index: query.page_index, + page_size: query.page_size }; let body = serde_json::to_string(&response) From 36f3003576ca86b88c0ab30fd99af3672ed3f8a1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 17:04:11 -0500 Subject: [PATCH 066/136] Fix QueryBuilder with respect to column names and type juggling --- .../media_files/media_file_origin_category.rs | 2 + .../media_file_origin_model_type.rs | 2 + .../media_file_origin_product_category.rs | 1 + .../by_table/media_files/media_file_type.rs | 1 + crates/_types/enums/src/common/visibility.rs | 1 + crates/_types/enums/src/lib.rs | 1 + .../enums/src/macros/impl_mysql_from_row.rs | 63 ++++++++++++++++++ crates/_types/enums/src/macros/mod.rs | 1 + crates/_types/enums/src/traits/mod.rs | 1 + .../_types/enums/src/traits/mysql_from_row.rs | 11 ++++ .../media_files/list_media_files_for_user.rs | 65 +++++++++++++++---- 11 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 crates/_types/enums/src/macros/impl_mysql_from_row.rs create mode 100644 crates/_types/enums/src/traits/mod.rs create mode 100644 crates/_types/enums/src/traits/mysql_from_row.rs diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs index 5db6daa07e..39c87a33e0 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; + use serde::Deserialize; use serde::Serialize; #[cfg(test)] @@ -29,6 +30,7 @@ pub enum MediaFileOriginCategory { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginCategory); impl_mysql_enum_coders!(MediaFileOriginCategory); +impl_mysql_from_row!(MediaFileOriginCategory); /// NB: Legacy API for older code. impl MediaFileOriginCategory { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index 72683e99ae..b38f90a6dc 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -24,6 +24,7 @@ pub enum MediaFileOriginModelType { /// so-vits-svc voice conversion models #[serde(rename = "so_vits_svc")] SoVitsSvc, + #[serde(rename = "vall_e_x")] VallEX, } @@ -31,6 +32,7 @@ pub enum MediaFileOriginModelType { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginModelType); impl_mysql_enum_coders!(MediaFileOriginModelType); +impl_mysql_from_row!(MediaFileOriginModelType); /// NB: Legacy API for older code. impl MediaFileOriginModelType { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 8bc2a4b81a..2d27b3824d 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,6 +39,7 @@ pub enum MediaFileOriginProductCategory { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginProductCategory); impl_mysql_enum_coders!(MediaFileOriginProductCategory); +impl_mysql_from_row!(MediaFileOriginProductCategory); /// NB: Legacy API for older code. impl MediaFileOriginProductCategory { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_type.rs index 15bb135098..08fe71f572 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_type.rs @@ -25,6 +25,7 @@ pub enum MediaFileType { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileType); impl_mysql_enum_coders!(MediaFileType); +impl_mysql_from_row!(MediaFileType); /// NB: Legacy API for older code. impl MediaFileType { diff --git a/crates/_types/enums/src/common/visibility.rs b/crates/_types/enums/src/common/visibility.rs index e5a8494005..f7240ca332 100644 --- a/crates/_types/enums/src/common/visibility.rs +++ b/crates/_types/enums/src/common/visibility.rs @@ -38,6 +38,7 @@ pub enum Visibility { impl_enum_display_and_debug_using_to_str!(Visibility); +impl_mysql_from_row!(Visibility); // For reference, here's what the serde implementation might be if manually written. // This may be useful for designing composite types in the future: diff --git a/crates/_types/enums/src/lib.rs b/crates/_types/enums/src/lib.rs index 8a725a12ed..e558ab9d23 100644 --- a/crates/_types/enums/src/lib.rs +++ b/crates/_types/enums/src/lib.rs @@ -32,3 +32,4 @@ mod macros; pub mod by_table; pub mod common; +pub mod traits; diff --git a/crates/_types/enums/src/macros/impl_mysql_from_row.rs b/crates/_types/enums/src/macros/impl_mysql_from_row.rs new file mode 100644 index 0000000000..91380fbf65 --- /dev/null +++ b/crates/_types/enums/src/macros/impl_mysql_from_row.rs @@ -0,0 +1,63 @@ + +/// Implement `MySqlFromRow` on a type. +macro_rules! impl_mysql_from_row { + ($t:ident) => { + + // Try to convert a MySQL row and named field into the value type (for non-nullable fields). + impl crate::traits::mysql_from_row::MySqlFromRow<$t> for $t { + fn try_from_mysql_row(row: &sqlx_mysql::MySqlRow, field_name: &str) -> Result<$t, sqlx::Error> { + use sqlx::{Error, Row}; + + // NB(bt,2023-12-05): For now only string encodings are considered. + // We may want to revisit in the future if we deal with binary data. + let value : String = row.try_get(field_name)?; + + let output_type = Self::from_str(&value) + .map_err(|err| { + Error::ColumnDecode { + index: format!("mysql_from_row failure on field {}: {:?}", field_name, err), + source: format!("mysql_from_row failure on field {}: {:?}", field_name, err).into(), + } + })?; + + Ok(output_type) + } + + // Try to convert a MySQL row and named field into the value type (for nullable fields). + fn try_from_mysql_row_nullable(row: &sqlx_mysql::MySqlRow, field_name: &str) -> Result, sqlx::Error> { + use sqlx::{Error, Row}; + + // NB(bt,2023-12-05): For now only string encodings are considered. + // We may want to revisit in the future if we deal with binary data. + // NB(2): Nullable fields decode as Option. + let maybe_value : Option = row.try_get(field_name)?; + + let value = match maybe_value { + Some(value) => value, + None => return Ok(None), + }; + + let output_type = Self::from_str(&value) + .map_err(|err| { + Error::ColumnDecode { + index: format!("mysql_from_row failure on field {}: {:?}", field_name, err), + source: format!("mysql_from_row failure on field {}: {:?}", field_name, err).into(), + } + })?; + + Ok(Some(output_type)) + } + } + + //// TODO: Tests. + //#[cfg(test)] + //#[test] + //fn test_name() { + // use strum::IntoEnumIterator; + // for variant in $t::iter() { + // assert_eq!(format!("{}", variant), variant.to_str()); + // } + //} + + } +} diff --git a/crates/_types/enums/src/macros/mod.rs b/crates/_types/enums/src/macros/mod.rs index 2f678901d2..93cba5b4dc 100644 --- a/crates/_types/enums/src/macros/mod.rs +++ b/crates/_types/enums/src/macros/mod.rs @@ -1,3 +1,4 @@ #[macro_use] pub mod impl_enum_display_and_debug_using_to_str; #[macro_use] pub mod impl_mysql_enum_coders; +#[macro_use] pub mod impl_mysql_from_row; #[macro_use] pub mod impl_sqlite_enum_coders; diff --git a/crates/_types/enums/src/traits/mod.rs b/crates/_types/enums/src/traits/mod.rs new file mode 100644 index 0000000000..ceb15f8868 --- /dev/null +++ b/crates/_types/enums/src/traits/mod.rs @@ -0,0 +1 @@ +pub mod mysql_from_row; \ No newline at end of file diff --git a/crates/_types/enums/src/traits/mysql_from_row.rs b/crates/_types/enums/src/traits/mysql_from_row.rs new file mode 100644 index 0000000000..7d3c3cbb67 --- /dev/null +++ b/crates/_types/enums/src/traits/mysql_from_row.rs @@ -0,0 +1,11 @@ +use sqlx_mysql::MySqlRow; + +/// This is a trait (in conjunction with the macro implementation) that makes up for sqlx's +/// QueryBuilder not playing well with FromRow implementation or derivation of named types. +pub trait MySqlFromRow { + + /// Attempt to parse a named column from a MySqlRow into the type. + fn try_from_mysql_row(row: &MySqlRow, field_name: &str) -> Result; + + fn try_from_mysql_row_nullable(row: &MySqlRow, field_name: &str) -> Result, sqlx::Error>; +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index 1c7bcdee62..1f9480ee93 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -1,11 +1,13 @@ use chrono::{DateTime, Utc}; -use sqlx::{MySql, MySqlPool, QueryBuilder}; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; +use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; @@ -119,21 +121,21 @@ fn query_builder<'a>( r#" SELECT m.id, - m.token as `token: tokens::tokens::media_files::MediaFileToken`, + m.token, - m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, - m.origin_product_category as `origin_product_category: enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory`, + m.origin_category, + m.origin_product_category, m.maybe_origin_model_type, m.maybe_origin_model_token, - m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, - m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + m.creator_set_visibility, m.created_at, m.updated_at @@ -141,24 +143,30 @@ FROM media_files AS m LEFT OUTER JOIN users AS u ON m.maybe_creator_user_token = u.token -WHERE u.username = ? -AND m.user_deleted_at IS NULL -AND m.mod_deleted_at IS NULL +WHERE m.user_deleted_at IS NULL + AND m.mod_deleted_at IS NULL "# ); + query_builder.push(" AND u.username = "); query_builder.push_bind(username); if let Some(media_type) = maybe_filter_media_type { - query_builder.push(" AND m.media_file_type = ? "); - query_builder.push_bind(media_type); + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.media_file_type = "); } match view_as { ViewAs::Author => {} ViewAs::Moderator => {} ViewAs::AnotherUser => { - query_builder.push(" AND m.creator_set_visibility = 'public' "); + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.creator_set_visibility = "); + query_builder.push_bind(Visibility::Public.to_str()); } } @@ -181,7 +189,6 @@ AND m.mod_deleted_at IS NULL query_builder } -#[derive(sqlx::FromRow)] struct MediaFileListItemInternal { id: i64, token: MediaFileToken, @@ -201,3 +208,35 @@ struct MediaFileListItemInternal { created_at: DateTime, updated_at: DateTime, } + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for MediaFileListItemInternal { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + id: row.try_get("id")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, + creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) + } +} From e986a44e9daf942be923e2ef1276cd552114bd0d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:00:44 -0500 Subject: [PATCH 067/136] documentation system open api spec. --- Cargo.lock | 114 +++++++++++++ crates/_types/tokens/Cargo.toml | 3 + .../src/tokens/generic_inference_jobs.rs | 3 +- crates/service/web/storyteller_web/Cargo.toml | 6 + .../web/storyteller_web/src/api_doc.rs | 11 ++ .../web/storyteller_web/src/api_doc_server.rs | 151 ++++++++++++++++++ .../inference/enqueue_tts_request.rs | 23 ++- .../service/web/storyteller_web/src/routes.rs | 4 + 8 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/api_doc.rs create mode 100644 crates/service/web/storyteller_web/src/api_doc_server.rs diff --git a/Cargo.lock b/Cargo.lock index ae154198df..561b3a18b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,6 +2964,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", + "serde", ] [[package]] @@ -4393,6 +4394,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -4876,6 +4901,41 @@ dependencies = [ "quick-xml 0.30.0", ] +[[package]] +name = "rust-embed" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn 2.0.39", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada" +dependencies = [ + "sha2 0.10.8", + "walkdir", +] + [[package]] name = "rust-ini" version = "0.18.0" @@ -5411,6 +5471,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5894,6 +5963,8 @@ dependencies = [ "url_config", "user_input_common", "users_component", + "utoipa", + "utoipa-swagger-ui", "uuid 0.8.2", ] @@ -6387,6 +6458,8 @@ dependencies = [ "sqlx-core", "strum", "toml", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -7025,6 +7098,47 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utoipa" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "regex", + "syn 2.0.39", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" +dependencies = [ + "actix-web", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "utoipa", + "zip", +] + [[package]] name = "uuid" version = "0.8.2" diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..f6276b98ab 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -28,6 +28,9 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } + [dev-dependencies] # Enum utilities diff --git a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs index 1c455e457e..48c3a78260 100644 --- a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs +++ b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs @@ -4,9 +4,10 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for "generic" inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct InferenceJobToken(String); diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..8cedfffde2 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -13,6 +13,10 @@ publish = false name = "storyteller-web" path = "src/main.rs" +[[bin]] +name = "docs" +path = "src/api_doc_server.rs" + [dependencies] # Internal @@ -160,6 +164,8 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs new file mode 100644 index 0000000000..f76feb1e76 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -0,0 +1,11 @@ + +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; + +use utoipa::OpenApi; +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), + components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] +pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs new file mode 100644 index 0000000000..b9b4213409 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -0,0 +1,151 @@ +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +//#![forbid(warnings)] +#![allow(unreachable_patterns)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use utoipa_swagger_ui::SwaggerUi; + +use std::sync::Arc; +use std::time::Duration; + +use actix_cors::Cors; +use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::middleware::{DefaultHeaders, Logger}; +use anyhow::anyhow; +use elasticsearch::Elasticsearch; +use elasticsearch::http::transport::Transport; +use futures::Future; +use limitation::Limiter; +use log::{error, info}; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use r2d2_redis::RedisConnectionManager; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; +use tokio::runtime::Runtime; + + + +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; +use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; +use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; +use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; +use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; +use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; +use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; +use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; +use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; +use cloud_storage::bucket_client::BucketClient; +use config::common_env::CommonEnv; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; +use email_sender::smtp_email_sender::SmtpEmailSender; +use errors::AnyhowResult; +use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use mysql_queries::mediators::badge_granter::BadgeGranter; +use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use redis_caching::redis_ttl_cache::RedisTtlCache; +use reusable_types::server_environment::ServerEnvironment; +use twitch_common::twitch_secrets::TwitchSecrets; +use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; +use users_component::cookies::session::session_cookie_manager::SessionCookieManager; +use users_component::utils::session_checker::SessionChecker; + +use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; +use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; +use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; +use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::routes::add_routes; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; +use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; +use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; + +pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); + +pub mod billing; +pub mod configs; +pub mod cookies; +pub mod http_server; +pub mod memory_cache; +pub mod model; +pub mod routes; +pub mod server_state; +pub mod subscriptions; +pub mod threads; +pub mod user_avatars; +pub mod util; +pub mod validations; + +const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &str = "SECRET_KEY"; +const ENV_REGION_NAME : &str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; + +use std::{ + error::Error, + future::{self, Ready}, + net::Ipv4Addr, +}; + +pub mod api_doc; +use api_doc::ApiDoc; + +use futures::future::LocalBoxFuture; + +use utoipa::OpenApi; +#[actix_web::main] +async fn main() -> Result<(), impl Error> { + HttpServer::new(move || { + App::new() + .service( + SwaggerUi::new("/{_:.*}") + .url("/api-docs/openapi.json", ApiDoc::openapi()), + ) + }) + .bind(("localhost", 8989)).unwrap() + .run().await +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index d394edabcd..32ae10d58d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,8 +1,8 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] +use utoipa::ToSchema; -// TODO MOVE into own file. use std::fmt::Debug; use std::sync::Arc; @@ -42,7 +42,9 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; /// This is useful for catching the live logs or intercepting the job. const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; + #[derive(Deserialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, @@ -50,12 +52,14 @@ pub struct EnqueueTTSRequest { } #[derive(Serialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] +#[derive(ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, @@ -95,13 +99,26 @@ impl std::fmt::Display for EnqueueTTSRequestError { // Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions // insert generic inference job.rs // Need to convert it to generic inference job. +#[utoipa::path( + post, + path = "/inference/enqueue_tts/", + responses( + (status = 200, description = "Enqueue TTS generically", body = EnqueueTTSRequestSuccessResponse), + (status = 400, description = "Bad input", body = EnqueueTTSRequestError), + (status = 401, description = "Not authorized", body = EnqueueTTSRequestError), + (status = 429, description = "Rate limited", body = EnqueueTTSRequestError), + (status = 500, description = "Server error", body = EnqueueTTSRequestError) + ), + params( + ("request" = EnqueueTTSRequest, description = "Payload for TTS Request") + ) +)] pub async fn enqueue_tts_request( http_request: HttpRequest, request: web::Json, server_state: web::Data> ) -> Result { - println!("Received payload"); - + let mut maybe_user_token: Option = None; let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..d84410d643 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,6 +161,10 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { + use utoipa::OpenApi; + + + let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ From e32f8094e5edbe2d1c23f35bfa5e30ed2b36d7d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 01:29:52 -0500 Subject: [PATCH 068/136] global list of media files --- .../queries/media_files/list_media_files.rs | 262 ++++++++++++++++++ .../media_files/list_media_files_for_user.rs | 1 + .../src/queries/media_files/mod.rs | 1 + .../endpoints/media_files/list_media_files.rs | 214 ++++++++++++++ .../media_files/list_media_files_for_user.rs | 2 - .../http_server/endpoints/media_files/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 7 +- 7 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs new file mode 100644 index 0000000000..15571f0563 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs @@ -0,0 +1,262 @@ +use chrono::{DateTime, Utc}; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +pub struct MediaFileListPage { + pub records: Vec, + + pub sort_ascending: bool, + + /// ID of the first record. + pub first_id: Option, + + /// ID of the last record. + pub last_id: Option, +} + +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Clone, Copy)] +pub enum ViewAs { + Moderator, + NonModerator, +} + +pub struct ListMediaFilesArgs<'a> { + pub limit: usize, + pub maybe_filter_media_type: Option, + pub maybe_offset: Option, + pub cursor_is_reversed: bool, + pub view_as: ViewAs, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn list_media_files(args: ListMediaFilesArgs<'_>) -> AnyhowResult { + + let mut query = query_builder( + args.maybe_filter_media_type, + args.limit, + args.maybe_offset, + args.cursor_is_reversed, + args.view_as, + ); + + let query = query.build_query_as::(); + + let results = query.fetch_all(args.mysql_pool).await?; + + let first_id = results.first() + .map(|raw_result| raw_result.id); + + let last_id = results.last() + .map(|raw_result| raw_result.id); + + let results = results.into_iter() + .map(|record| { + MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_username: record.maybe_creator_username, + maybe_creator_display_name: record.maybe_creator_display_name, + maybe_creator_gravatar_hash: record.maybe_creator_gravatar_hash, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + Ok(MediaFileListPage { + records: results, + sort_ascending: !args.cursor_is_reversed, + first_id, + last_id, + }) +} + +fn query_builder<'a>( + maybe_filter_media_type: Option, + limit: usize, + maybe_offset: Option, + cursor_is_reversed: bool, + view_as: ViewAs, +) -> QueryBuilder<'a, MySql> { + + // NB: Query cannot be statically checked by sqlx + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +SELECT + m.id, + m.token, + + m.origin_category, + m.origin_product_category, + + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + m.media_type, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.maybe_creator_user_token, + u.username as maybe_creator_username, + u.display_name as maybe_creator_display_name, + u.email_gravatar_hash as maybe_creator_gravatar_hash, + + m.creator_set_visibility, + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users AS u + ON m.maybe_creator_user_token = u.token + +WHERE m.user_deleted_at IS NULL +AND m.mod_deleted_at IS NULL + "# + ); + + if let Some(media_type) = maybe_filter_media_type { + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.media_file_type = "); + query_builder.push_bind(media_type.to_str()); + } + + match view_as { + ViewAs::Moderator => {} + ViewAs::NonModerator=> { + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.creator_set_visibility = "); + query_builder.push_bind(Visibility::Public.to_str()); + } + } + + if let Some(offset) = maybe_offset { + if cursor_is_reversed { + query_builder.push(format!(" AND m.id < {offset} ")); + } else { + query_builder.push(format!(" AND m.id > {offset} ")); + } + } + + if cursor_is_reversed { + query_builder.push(" ORDER BY m.id ASC "); + } else { + query_builder.push(" ORDER BY m.id DESC "); + } + + query_builder.push(format!(" LIMIT {limit} ")); + + query_builder +} + +struct MediaFileListItemInternal { + id: i64, + token: MediaFileToken, + + origin_category: MediaFileOriginCategory, + origin_product_category: MediaFileOriginProductCategory, + maybe_origin_model_type: Option, + maybe_origin_model_token: Option, + + media_type: MediaFileType, + public_bucket_directory_hash: String, + maybe_public_bucket_prefix: Option, + maybe_public_bucket_extension: Option, + + maybe_creator_user_token: Option, + maybe_creator_username: Option, + maybe_creator_display_name: Option, + maybe_creator_gravatar_hash: Option, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for MediaFileListItemInternal { + fn from_row(row: &MySqlRow) -> Result { + let maybe_creator_user_token : Option = row.try_get("maybe_creator_user_token")?; + let maybe_creator_user_token = maybe_creator_user_token.map(|user_token| UserToken::new(user_token)); + + Ok(Self { + id: row.try_get("id")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, + maybe_creator_user_token, + maybe_creator_username: row.try_get("maybe_creator_username")?, + maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, + maybe_creator_gravatar_hash: row.try_get("maybe_creator_gravatar_hash")?, + creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index 1f9480ee93..87bcc96ee8 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -156,6 +156,7 @@ WHERE m.user_deleted_at IS NULL // Otherwise, it's calling the Display trait on the raw type which is resulting in an // incorrect binding and runtime error. query_builder.push(" AND m.media_file_type = "); + query_builder.push_bind(media_type.to_str()); } match view_as { diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 619f018ce5..0590f82159 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -4,5 +4,6 @@ pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; +pub mod list_media_files; pub mod list_media_files_for_user; pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs new file mode 100644 index 0000000000..df1e91d32d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs @@ -0,0 +1,214 @@ +use std::sync::Arc; + +use actix_web::{HttpMessage, HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Query}; +use chrono::{DateTime, Utc}; +use log::{info, warn}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use mysql_queries::queries::media_files::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::common_responses::pagination_cursors::PaginationCursors; +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct QueryParams { + pub sort_ascending: Option, + pub per_page: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, + pub filter_media_type: Option, +} + +#[derive(Serialize)] +pub struct SuccessResponse { + pub success: bool, + pub results: Vec, + pub pagination: PaginationCursors, +} + +#[derive(Serialize)] +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub maybe_creator: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ErrorResponse { + ServerError, +} + +impl ResponseError for ErrorResponse { + fn status_code(&self) -> StatusCode { + match *self { + ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ErrorResponse::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +impl std::fmt::Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_media_files_handler( + http_request: HttpRequest, + query: Query, + server_state: web::Data> +) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ErrorResponse::ServerError + })?; + + let mut is_mod = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_mod = session.can_ban_users; + }, + }; + + // TODO(bt,2023-12-04): Enforce real maximums and defaults + let limit = query.per_page.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor as usize) + } else { + None + }; + + let view_as = if is_mod { + ViewAs::Moderator + } else { + ViewAs::NonModerator + }; + + let query_results = list_media_files(ListMediaFilesArgs { + limit, + maybe_filter_media_type: query.filter_media_type, + maybe_offset: cursor, + cursor_is_reversed, + view_as, + mysql_pool: &server_state.mysql_pool, + }).await; + + + let results_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ErrorResponse::ServerError); + } + }; + + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let results = results_page.records.into_iter() + .map(|record| MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + maybe_creator: UserDetailsLight::from_optional_db_fields_owned( + record.maybe_creator_user_token, + record.maybe_creator_username, + record.maybe_creator_display_name, + record.maybe_creator_gravatar_hash, + ), + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + }) + .collect::>(); + + let response = SuccessResponse { + success: true, + results, + pagination: PaginationCursors { + maybe_next: cursor_next, + maybe_previous: cursor_previous, + cursor_is_reversed, + } + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ErrorResponse::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs index 9cfe696a27..bd3eda2b78 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -94,8 +94,6 @@ pub async fn list_media_files_for_user_handler( server_state: web::Data> ) -> Result { - info!("Fetching inference results for user: {}", &path.username); - let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 6af4273b7a..ab3552ac20 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,4 +1,5 @@ pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; +pub mod list_media_files; pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9aafd5340c..bae83ecc53 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -44,6 +44,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; @@ -1007,7 +1008,11 @@ fn add_media_file_routes (app: App) -> App .route(web::delete().to(delete_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(web::resource("/user/{username}") + .service(web::resource("/list") + .route(web::get().to(list_media_files_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/list/user/{username}") .route(web::get().to(list_media_files_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) From b5fcc3cdfac8493932780047495e809941d1393f Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:14:02 -0500 Subject: [PATCH 069/136] Added documentation about the documentation --- _develop/docs/README.md | 4 +++- _develop/docs/local_development/general_setup.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/_develop/docs/README.md b/_develop/docs/README.md index 9ba35171fa..8b55ea0f13 100644 --- a/_develop/docs/README.md +++ b/_develop/docs/README.md @@ -29,7 +29,9 @@ Web API Overview * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only * `w2l-download-job` - (*deprecated*) Async download of videos and images * `w2l-inference-job` - (*deprecated*) Async W2L inference - + * Documentation Server: + * `docs` - this will run a local swagger server with the endpoints that you have access to, http://127.0.0.1:8989/api-docs/openapi.json use this to port everything over to the post man. + ### PowerStream * [PowerStream](https://power.stream) (deprecated) is powered by the following applications: diff --git a/_develop/docs/local_development/general_setup.md b/_develop/docs/local_development/general_setup.md index 24d849051e..25c499e0c4 100644 --- a/_develop/docs/local_development/general_setup.md +++ b/_develop/docs/local_development/general_setup.md @@ -380,3 +380,16 @@ of this error message. You can essentially ignore it. ### [Fix] Can't connect to local MySQL after install on Ubuntu - If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux). + +### API Server +``` +cargo run --bin docs +``` +This setups an api server with swagger docs to show you responses and requests with each associated endpoint +It is run at http://127.0.0.1:8989/ +To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json +We use this: `https://github.com/juhaku/utoipa` +The documentation example is in: +`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs` +Write into the api doc to: +`crates/service/web/storyteller_web/src/api_doc.rs` \ No newline at end of file From 404853e129a6e21a3ff833df6df69a794a133d0b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 19:16:55 -0500 Subject: [PATCH 070/136] cleanup misleading old comment --- crates/_types/enums/Cargo.toml | 4 ---- crates/_types/tokens/Cargo.toml | 4 ---- crates/cli/dev_database_seed/Cargo.toml | 4 ---- crates/cli/dev_upload_media_file/Cargo.toml | 4 ---- crates/cli/elasticsearch-cli/Cargo.toml | 4 ---- crates/lib/mysql_queries/Cargo.toml | 4 ---- crates/lib/sqlite_queries/Cargo.toml | 5 ----- crates/lib/sqlx_mysql_helpers/Cargo.toml | 4 ---- crates/programs/aichatbot-sidecar/Cargo.toml | 5 ----- crates/service/job/analytics_job/Cargo.toml | 4 ---- crates/service/job/download_job/Cargo.toml | 4 ---- crates/service/job/email_sender_job/Cargo.toml | 4 ---- crates/service/job/inference_job/Cargo.toml | 4 ---- crates/service/job/tts_download_job/Cargo.toml | 4 ---- crates/service/job/tts_inference_job/Cargo.toml | 4 ---- crates/service/job/w2l_download_job/Cargo.toml | 4 ---- crates/service/job/w2l_inference_job/Cargo.toml | 4 ---- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 5 ----- crates/service/plugins/billing/Cargo.toml | 4 ---- crates/service/plugins/users/Cargo.toml | 4 ---- crates/service/web/storyteller_web/Cargo.toml | 4 ---- crates/service/web/websocket_gateway/Cargo.toml | 4 ---- 22 files changed, 91 deletions(-) diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 8493b4c80b..1cb6c94409 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -17,10 +17,6 @@ path = "src/lib.rs" serde = "1.0.125" serde_derive = "1.0.125" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] } sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..cc8825e725 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -20,10 +20,6 @@ serde_derive = "1.0.125" # Random rand = "0.8.5" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index b2a4dcf25e..d2ef105cce 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -36,10 +36,6 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 2b3c6b2626..c1f7ed8790 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -33,10 +33,6 @@ log = "0.4.14" # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index c3a234024c..9876936696 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -33,10 +33,6 @@ strum = { version = "0.25", features = ["derive"] } # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 02274f5412..bcdff745f8 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -39,10 +39,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" serial_test = "2.0.0" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 tokio = { version = "1.6.0", features = ["full"] } rand = "0.8" diff --git a/crates/lib/sqlite_queries/Cargo.toml b/crates/lib/sqlite_queries/Cargo.toml index 9427adfeb5..17f25f8d0b 100644 --- a/crates/lib/sqlite_queries/Cargo.toml +++ b/crates/lib/sqlite_queries/Cargo.toml @@ -28,11 +28,6 @@ log = "0.4.14" once_cell = "1.9.0" regex = "1.5.4" -# SQlite Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ sqlx = { version = "0.7.2", features = [ "sqlite", "runtime-tokio-native-tls", "chrono" ] } [dev-dependencies] diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index e7837fde69..647e980426 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -16,10 +16,6 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index b6a9068b0a..cebc56b8a1 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -42,11 +42,6 @@ rand = "0.8.3" regex = "1.5.4" strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc. -# SQlite Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ sqlx = { version = "0.7.2", features = [ "sqlite", "runtime-tokio-native-tls", "chrono" ] } # Async Runtime diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 30640d7446..6a803df2a4 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -32,10 +32,6 @@ hostname = "0.3.1" log = "0.4.14" rand = "0.8.5" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Runtime diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 7b69a8a6cc..6a4c49eb42 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -43,10 +43,6 @@ subprocess = "0.2.9" tempdir = "0.3.7" zip = "0.6.6" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 8a9324340d..b1385b31e8 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -56,10 +56,6 @@ once_cell = "1.18.0" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 36194c73a9..f82dbe6589 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -52,10 +52,6 @@ once_cell = "1.18.0" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index ccdd04bd2b..b57ea434e8 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -35,10 +35,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 846405c909..2cbff86bf7 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -41,10 +41,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index c71e96c24b..aa55cde374 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -35,10 +35,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 04d38e114d..6aefb7d7f6 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -33,10 +33,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 8de44b6db1..94e3520a39 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -47,11 +47,6 @@ futures = { version = "0.3.17", features = ["thread-pool"] } # feature for Threa futures-util = "0.3.29" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate - -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index fac7fcba86..5e879949eb 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -50,10 +50,6 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt # include support for async fn in traits. async-trait = "0.1.58" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index bee1b60deb..5d852fa380 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -58,10 +58,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # TODO: Shouldn't really be a dependency, but need to add caching for the DDoS attack. diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..eef8bc4162 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -97,10 +97,6 @@ futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older ve futures-core = "0.3.29" # NB: For middleware/futures upgrade pin-project-lite = "0.2.9" # NB: For middleware/futures upgrade -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index f1202bcdd1..85df5ba18d 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -47,10 +47,6 @@ futures = { version = "0.3.17", features = ["thread-pool"] } # feature for Threa futures-util = "0.3.29" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database From 4856ac28aa15a034a39f8e9f7d7deae89f619c11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:19:36 +0000 Subject: [PATCH 071/136] Bump chrono from 0.4.22 to 0.4.31 Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.22 to 0.4.31. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.22...v0.4.31) --- updated-dependencies: - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 15 ++++++++++----- crates/_types/reusable_types/Cargo.toml | 2 +- crates/lib/datetimes/Cargo.toml | 2 +- .../lib/deprecated/http_server_common/Cargo.toml | 2 +- crates/lib/elasticsearch_schema/Cargo.toml | 2 +- crates/lib/jobs_common/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/lib/tts_common/Cargo.toml | 2 +- crates/lib/web_scrapers/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- crates/service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- crates/service/job/tts_download_job/Cargo.toml | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- crates/service/web/websocket_gateway/Cargo.toml | 2 +- my-workspace-hack/Cargo.toml | 4 ++-- 24 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..b5ac403216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,6 +468,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -1037,18 +1043,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", - "winapi 0.3.9", + "windows-targets", ] [[package]] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 7b87cc85e7..0548cf530e 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # (none) # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } # Serialization serde = "1.0.125" diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 8cd99557fa..2961491dce 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,7 +13,7 @@ name = "datetimes" path = "src/lib.rs" [dependencies] -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } once_cell = "1.18.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 82f7101d72..91bda125b9 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -21,7 +21,7 @@ reusable_types = { path = "../../../_types/reusable_types" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/elasticsearch_schema/Cargo.toml b/crates/lib/elasticsearch_schema/Cargo.toml index 7926f55ac1..d2dd451dae 100644 --- a/crates/lib/elasticsearch_schema/Cargo.toml +++ b/crates/lib/elasticsearch_schema/Cargo.toml @@ -23,7 +23,7 @@ elasticsearch = "8.5.0-alpha.1" log = "0.4.14" # Serialization -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.108" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index ad3a0c9a3e..6f68d2eb5f 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -24,7 +24,7 @@ redis_common = { path = "../redis_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index bcdff745f8..2af35c4ffc 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -25,7 +25,7 @@ tokens = { path = "../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 6e42e04e6a..e424163e24 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -20,7 +20,7 @@ user_input_common = { path = "../user_input_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index a5e01837ce..f17589be4d 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -19,7 +19,7 @@ errors = { path = "../../lib/errors" } # External bytes = "1.3" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" strum = { version = "0.25", features = ["derive"] } # Fancy enum iteration, etc. diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 6a803df2a4..436d75347c 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -26,7 +26,7 @@ tokens = { path = "../../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 6a4c49eb42..76d139dcb8 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -35,7 +35,7 @@ subprocess_common = { path = "../../../lib/subprocess_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index b1385b31e8..9c4d7f9eec 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -46,7 +46,7 @@ tts_common = { path = "../../../lib/tts_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index f82dbe6589..53334a6465 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -43,7 +43,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index b57ea434e8..79741dc0d3 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 2cbff86bf7..02757a7af7 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -32,7 +32,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index aa55cde374..afb81aec18 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 6aefb7d7f6..a3aab860c6 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -26,7 +26,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 94e3520a39..00af98d691 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -28,7 +28,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 5e879949eb..fc99fded80 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -24,7 +24,7 @@ url_config = { path = "../../../lib/url_config" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" once_cell = "1.18.0" diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 5d852fa380..09915d397c 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -34,7 +34,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" lazy_static = "1.4.0" # Regex caching (TODO: Use OnceCell instead) log = "0.4.14" diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index abbd3f4e96..9c8de7f237 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" #once_cell = "1.9.0" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index eef8bc4162..f298a33609 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -54,7 +54,7 @@ users_component = { path = "../../plugins/users" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hex = "0.4.3" diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 85df5ba18d..5f051a7172 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -27,7 +27,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hostname = "0.3.1" diff --git a/my-workspace-hack/Cargo.toml b/my-workspace-hack/Cargo.toml index 2f062e5a69..e93823db80 100644 --- a/my-workspace-hack/Cargo.toml +++ b/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -88,7 +88,7 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } From 0033af73cef3a64c95ee95838b3bbf452257346e Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 20:26:07 -0500 Subject: [PATCH 072/136] Feedback addressed! --- Cargo.lock | 1 - crates/_types/tokens/Cargo.toml | 1 - .../web/storyteller_web/src/api_doc_server.rs | 82 +------------------ .../service/web/storyteller_web/src/routes.rs | 5 -- 4 files changed, 2 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 561b3a18b1..0a0f2d5d63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6459,7 +6459,6 @@ dependencies = [ "strum", "toml", "utoipa", - "utoipa-swagger-ui", ] [[package]] diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index f6276b98ab..7df9d68c0b 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -29,7 +29,6 @@ sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } -utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [dev-dependencies] diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index b9b4213409..e3fc865736 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -19,18 +19,9 @@ use utoipa_swagger_ui::SwaggerUi; -use std::sync::Arc; -use std::time::Duration; - -use actix_cors::Cors; -use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::{App, HttpServer, middleware, web}; use actix_web::middleware::{DefaultHeaders, Logger}; -use anyhow::anyhow; -use elasticsearch::Elasticsearch; -use elasticsearch::http::transport::Transport; -use futures::Future; -use limitation::Limiter; -use log::{error, info}; + use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; @@ -38,63 +29,10 @@ use sqlx::mysql::MySqlPoolOptions; use sqlx::MySqlPool; use tokio::runtime::Runtime; - - -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; -use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; -use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; -use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; -use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; -use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; -use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; -use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; -use config::common_env::CommonEnv; -use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; -use email_sender::smtp_email_sender::SmtpEmailSender; use errors::AnyhowResult; -use http_server_common::cors::{build_cors_config, build_production_cors_config}; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use mysql_queries::mediators::badge_granter::BadgeGranter; -use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use redis_caching::redis_ttl_cache::RedisTtlCache; -use reusable_types::server_environment::ServerEnvironment; -use twitch_common::twitch_secrets::TwitchSecrets; -use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; -use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; -use users_component::cookies::session::session_cookie_manager::SessionCookieManager; -use users_component::utils::session_checker::SessionChecker; -use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; -use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; -use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; -use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; -use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; -use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; -use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; -use crate::routes::add_routes; use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; -use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; -use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; -use crate::threads::poll_ip_banlist_thread::poll_ip_bans; -use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; -use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; -use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; - -pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); -pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); pub mod billing; pub mod configs; @@ -110,21 +48,6 @@ pub mod user_avatars; pub mod util; pub mod validations; -const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; - -// Buckets (shared config) -const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; -const ENV_SECRET_KEY : &str = "SECRET_KEY"; -const ENV_REGION_NAME : &str = "REGION_NAME"; - -// Buckets (private data) -const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; -// Buckets (public data) -const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; - -// Various bucket roots -const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; - use std::{ error::Error, future::{self, Ready}, @@ -135,7 +58,6 @@ pub mod api_doc; use api_doc::ApiDoc; use futures::future::LocalBoxFuture; - use utoipa::OpenApi; #[actix_web::main] async fn main() -> Result<(), impl Error> { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index d84410d643..48d3048c78 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,11 +161,6 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { - use utoipa::OpenApi; - - - - let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ app = add_w2l_routes(app); /* /w2l */ From 92e0b1e9f4d615bfd430ab0a6e0a31ad8d8ee98b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 20:40:09 -0500 Subject: [PATCH 073/136] Revert "Bump chrono from 0.4.22 to 0.4.31" This reverts commit 4856ac28aa15a034a39f8e9f7d7deae89f619c11. --- Cargo.lock | 15 +++++---------- crates/_types/reusable_types/Cargo.toml | 2 +- crates/lib/datetimes/Cargo.toml | 2 +- .../lib/deprecated/http_server_common/Cargo.toml | 2 +- crates/lib/elasticsearch_schema/Cargo.toml | 2 +- crates/lib/jobs_common/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/lib/tts_common/Cargo.toml | 2 +- crates/lib/web_scrapers/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- crates/service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- crates/service/job/tts_download_job/Cargo.toml | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- crates/service/web/websocket_gateway/Cargo.toml | 2 +- my-workspace-hack/Cargo.toml | 4 ++-- 24 files changed, 29 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a95e836b72..31a560b20c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,12 +468,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -1043,17 +1037,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", + "num-integer", "num-traits", "serde", + "time 0.1.45", "wasm-bindgen", - "windows-targets", + "winapi 0.3.9", ] [[package]] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 0548cf530e..7b87cc85e7 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # (none) # External -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } # Serialization serde = "1.0.125" diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 2961491dce..8cd99557fa 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,7 +13,7 @@ name = "datetimes" path = "src/lib.rs" [dependencies] -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } once_cell = "1.18.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 91bda125b9..82f7101d72 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -21,7 +21,7 @@ reusable_types = { path = "../../../_types/reusable_types" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/elasticsearch_schema/Cargo.toml b/crates/lib/elasticsearch_schema/Cargo.toml index d2dd451dae..7926f55ac1 100644 --- a/crates/lib/elasticsearch_schema/Cargo.toml +++ b/crates/lib/elasticsearch_schema/Cargo.toml @@ -23,7 +23,7 @@ elasticsearch = "8.5.0-alpha.1" log = "0.4.14" # Serialization -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.108" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index 6f68d2eb5f..ad3a0c9a3e 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -24,7 +24,7 @@ redis_common = { path = "../redis_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 2af35c4ffc..bcdff745f8 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -25,7 +25,7 @@ tokens = { path = "../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index e424163e24..6e42e04e6a 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -20,7 +20,7 @@ user_input_common = { path = "../user_input_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index f17589be4d..a5e01837ce 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -19,7 +19,7 @@ errors = { path = "../../lib/errors" } # External bytes = "1.3" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" strum = { version = "0.25", features = ["derive"] } # Fancy enum iteration, etc. diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 436d75347c..6a803df2a4 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -26,7 +26,7 @@ tokens = { path = "../../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 76d139dcb8..6a4c49eb42 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -35,7 +35,7 @@ subprocess_common = { path = "../../../lib/subprocess_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 9c4d7f9eec..b1385b31e8 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -46,7 +46,7 @@ tts_common = { path = "../../../lib/tts_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 53334a6465..f82dbe6589 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -43,7 +43,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 79741dc0d3..b57ea434e8 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 02757a7af7..2cbff86bf7 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -32,7 +32,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index afb81aec18..aa55cde374 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index a3aab860c6..6aefb7d7f6 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -26,7 +26,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 00af98d691..94e3520a39 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -28,7 +28,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index fc99fded80..5e879949eb 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -24,7 +24,7 @@ url_config = { path = "../../../lib/url_config" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" once_cell = "1.18.0" diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 09915d397c..5d852fa380 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -34,7 +34,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" lazy_static = "1.4.0" # Regex caching (TODO: Use OnceCell instead) log = "0.4.14" diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 9c8de7f237..abbd3f4e96 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } # External -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" #once_cell = "1.9.0" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 3713a67023..9eff06ef38 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -58,7 +58,7 @@ users_component = { path = "../../plugins/users" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hex = "0.4.3" diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 5f051a7172..85df5ba18d 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -27,7 +27,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.31", features = ["serde"] } +chrono = { version = "=0.4.22", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hostname = "0.3.1" diff --git a/my-workspace-hack/Cargo.toml b/my-workspace-hack/Cargo.toml index 072f62bcbf..22f41a903d 100644 --- a/my-workspace-hack/Cargo.toml +++ b/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.31", features = ["alloc", "serde"] } +chrono = { version = "0.4.22", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -88,7 +88,7 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.31", features = ["alloc", "serde"] } +chrono = { version = "0.4.22", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } From 092745166e68818b539ef65f49546268d91c34c2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 20:56:22 -0500 Subject: [PATCH 074/136] vendor `limitation` crate The limitation crate is four years old, stuck at 0.1.1. It looks unlikey to be updated by the maintainer. Upgrades to chrono and other libraries have broken limitation, so we're likely now on the hook for maintaining this. --- Cargo.lock | 614 ++++++++++++------ Cargo.toml | 1 + crates/service/web/storyteller_web/Cargo.toml | 2 +- .../service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/limitation/CHANGELOG.md | 13 + crates/vendor/limitation/CODE_OF_CONDUCT.md | 77 +++ crates/vendor/limitation/Cargo.toml | 28 + crates/vendor/limitation/LICENSE.txt | 373 +++++++++++ crates/vendor/limitation/Makefile.toml | 6 + crates/vendor/limitation/README.md | 298 +++++++++ crates/vendor/limitation/README.tpl | 156 +++++ crates/vendor/limitation/src/lib.rs | 377 +++++++++++ .../limitation/tests/version-numbers.rs | 13 + 13 files changed, 1774 insertions(+), 186 deletions(-) create mode 100644 crates/vendor/limitation/CHANGELOG.md create mode 100644 crates/vendor/limitation/CODE_OF_CONDUCT.md create mode 100644 crates/vendor/limitation/Cargo.toml create mode 100644 crates/vendor/limitation/LICENSE.txt create mode 100644 crates/vendor/limitation/Makefile.toml create mode 100644 crates/vendor/limitation/README.md create mode 100644 crates/vendor/limitation/README.tpl create mode 100644 crates/vendor/limitation/src/lib.rs create mode 100644 crates/vendor/limitation/tests/version-numbers.rs diff --git a/Cargo.lock b/Cargo.lock index 31a560b20c..ed3a73d44a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "smallvec 1.11.2", - "tokio", + "tokio 1.34.0", "tokio-util", ] @@ -39,7 +39,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio", + "tokio 1.34.0", "tokio-util", "tracing", ] @@ -78,7 +78,7 @@ dependencies = [ "log", "mime", "mime_guess", - "percent-encoding", + "percent-encoding 2.3.1", "pin-project-lite", ] @@ -110,12 +110,12 @@ dependencies = [ "language-tags", "local-channel", "mime", - "percent-encoding", + "percent-encoding 2.3.1", "pin-project-lite", "rand 0.8.5", "sha1 0.10.6", "smallvec 1.11.2", - "tokio", + "tokio 1.34.0", "tokio-util", "tracing", "zstd 0.12.4", @@ -127,7 +127,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote", + "quote 1.0.33", "syn 2.0.39", ] @@ -153,7 +153,7 @@ dependencies = [ "serde_json", "serde_plain", "tempfile", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -164,8 +164,8 @@ checksum = "0a0a77f836d869f700e5b47ac7c3c8b9c8bc82e4aec861954c6198abee3ebd4d" dependencies = [ "darling 0.20.3", "parse-size", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -190,7 +190,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "actix-macros", "futures-core", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -206,7 +206,7 @@ dependencies = [ "futures-util", "mio 0.8.9", "socket2 0.5.5", - "tokio", + "tokio 1.34.0", "tracing", ] @@ -268,7 +268,7 @@ dependencies = [ "smallvec 1.11.2", "socket2 0.5.5", "time 0.3.30", - "url", + "url 2.5.0", ] [[package]] @@ -285,7 +285,7 @@ dependencies = [ "bytestring", "futures-core", "pin-project-lite", - "tokio", + "tokio 1.34.0", "tokio-util", ] @@ -296,8 +296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -307,8 +307,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -416,8 +416,8 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9afe97decc070be1864d96ab118383abed1c58fc09696663666049431110c5" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -465,7 +465,7 @@ dependencies = [ "serde_json", "sqlx", "tokens", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -594,7 +594,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -618,7 +618,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -627,8 +627,8 @@ version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -675,7 +675,7 @@ dependencies = [ "native-tls", "serde", "serde_json", - "url", + "url 2.5.0", ] [[package]] @@ -709,7 +709,7 @@ dependencies = [ "serde", "thiserror", "time 0.3.30", - "url", + "url 2.5.0", ] [[package]] @@ -804,7 +804,7 @@ dependencies = [ "serde_derive", "serde_json", "sqlx", - "tokio", + "tokio 1.34.0", "toml", "url_config", ] @@ -1137,8 +1137,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -1157,7 +1157,7 @@ dependencies = [ "log", "my-workspace-hack", "rust-s3", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -1213,7 +1213,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.34.0", "tokio-util", ] @@ -1247,7 +1247,7 @@ dependencies = [ "anyhow", "easyenv", "my-workspace-hack", - "url", + "url 2.5.0", ] [[package]] @@ -1289,7 +1289,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ - "percent-encoding", + "percent-encoding 2.3.1", "time 0.3.30", "version_check", ] @@ -1381,6 +1381,17 @@ dependencies = [ "crossbeam-utils 0.8.16", ] +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1388,10 +1399,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch", + "crossbeam-epoch 0.9.15", "crossbeam-utils 0.8.16", ] +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", +] + [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -1405,6 +1431,17 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1480,7 +1517,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote", + "quote 1.0.33", "syn 2.0.39", ] @@ -1522,8 +1559,8 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "strsim 0.10.0", "syn 1.0.109", ] @@ -1536,8 +1573,8 @@ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "strsim 0.10.0", "syn 1.0.109", ] @@ -1550,8 +1587,8 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "strsim 0.10.0", "syn 2.0.39", ] @@ -1563,7 +1600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote", + "quote 1.0.33", "syn 1.0.109", ] @@ -1574,7 +1611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core 0.14.4", - "quote", + "quote 1.0.33", "syn 1.0.109", ] @@ -1585,7 +1622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", - "quote", + "quote 1.0.33", "syn 2.0.39", ] @@ -1688,8 +1725,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling 0.14.4", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -1710,8 +1747,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1749,7 +1786,7 @@ dependencies = [ "sqlx", "storyteller_root", "tokens", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -1770,7 +1807,7 @@ dependencies = [ "mysql_queries", "sqlx", "tokens", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -1783,8 +1820,8 @@ dependencies = [ "byteorder", "diesel_derives", "mysqlclient-sys", - "percent-encoding", - "url", + "percent-encoding 2.3.1", + "url 2.5.0", ] [[package]] @@ -1794,8 +1831,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ "diesel_table_macro_syntax", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -1870,8 +1907,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -1934,7 +1971,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", - "tokio", + "tokio 1.34.0", "zip", ] @@ -1980,7 +2017,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -2025,13 +2062,13 @@ dependencies = [ "bytes 1.5.0", "dyn-clone", "lazy_static", - "percent-encoding", + "percent-encoding 2.3.1", "reqwest", "rustc_version 0.2.3", "serde", "serde_json", "serde_with", - "url", + "url 2.5.0", "void", ] @@ -2057,7 +2094,7 @@ dependencies = [ "storyteller_root", "strum", "tokens", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -2132,7 +2169,7 @@ dependencies = [ "tempdir", "testing", "tokens", - "tokio", + "tokio 1.34.0", "tts_common", ] @@ -2341,7 +2378,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "percent-encoding", + "percent-encoding 2.3.1", ] [[package]] @@ -2469,8 +2506,8 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -2596,7 +2633,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", + "tokio 1.34.0", "tokio-util", "tracing", ] @@ -2737,8 +2774,8 @@ dependencies = [ "log", "mac", "markup5ever", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -2788,7 +2825,7 @@ dependencies = [ "serde_json", "serde_qs 0.8.5", "serde_urlencoded", - "url", + "url 2.5.0", ] [[package]] @@ -2851,7 +2888,7 @@ dependencies = [ "itoa 1.0.9", "pin-project-lite", "socket2 0.4.10", - "tokio", + "tokio 1.34.0", "tower-service", "tracing", "want", @@ -2866,7 +2903,7 @@ dependencies = [ "bytes 1.5.0", "hyper", "native-tls", - "tokio", + "tokio 1.34.0", "tokio-native-tls", ] @@ -2907,6 +2944,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.5.0" @@ -3025,7 +3073,7 @@ dependencies = [ "subprocess_common", "tempdir", "tokens", - "tokio", + "tokio 1.34.0", "tts_common", ] @@ -3073,6 +3121,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.10.5" @@ -3218,15 +3275,15 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna", + "idna 0.5.0", "mime", "native-tls", "nom", "once_cell", "quoted_printable", "socket2 0.5.5", - "tokio", - "url", + "tokio 1.34.0", + "url 2.5.0", ] [[package]] @@ -3284,13 +3341,13 @@ dependencies = [ [[package]] name = "limitation" version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e415413963930f7050f21994bfd336dc75ef8b5b3b142e99c55b72b61f726d5" dependencies = [ "chrono", "futures 0.1.31", "redis 0.13.0", "time 0.1.45", + "tokio 0.1.22", + "version-sync", ] [[package]] @@ -3426,14 +3483,20 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "maybe-async" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -3486,6 +3549,15 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -3639,8 +3711,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -3718,13 +3790,13 @@ dependencies = [ "syn 2.0.39", "time 0.3.30", "time-macros", - "tokio", + "tokio 1.34.0", "tokio-stream", "tracing", "twitch_oauth2", "unicode-bidi", "unicode-normalization", - "url", + "url 2.5.0", "void", "winapi 0.3.9", "windows-sys 0.48.0", @@ -3754,7 +3826,7 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio", + "tokio 1.34.0", "toml", ] @@ -3824,7 +3896,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "tokio", + "tokio 1.34.0", "uuid 0.8.2", ] @@ -4027,8 +4099,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -4178,6 +4250,12 @@ dependencies = [ "base64ct", ] +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -4241,8 +4319,8 @@ checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -4279,8 +4357,8 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -4391,8 +4469,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", "version_check", ] @@ -4403,11 +4481,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "version_check", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -4426,6 +4513,17 @@ dependencies = [ "cc", ] +[[package]] +name = "pulldown-cmark" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b74cc784b038a9921fd1a48310cc2e238101aa8ae0b94201e2d85121dd68b5" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4467,13 +4565,22 @@ dependencies = [ "memchr", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.69", ] [[package]] @@ -4618,7 +4725,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-deque", + "crossbeam-deque 0.8.3", "crossbeam-utils 0.8.16", ] @@ -4642,7 +4749,7 @@ dependencies = [ "dtoa 0.4.8", "futures 0.1.31", "itoa 0.4.8", - "percent-encoding", + "percent-encoding 2.3.1", "sha1 0.6.1", "tokio-codec", "tokio-executor", @@ -4650,7 +4757,7 @@ dependencies = [ "tokio-sync", "tokio-tcp", "tokio-uds", - "url", + "url 2.5.0", ] [[package]] @@ -4663,9 +4770,9 @@ dependencies = [ "combine 4.6.6", "dtoa 0.4.8", "itoa 0.4.8", - "percent-encoding", + "percent-encoding 2.3.1", "sha1 0.6.1", - "url", + "url 2.5.0", ] [[package]] @@ -4679,13 +4786,13 @@ dependencies = [ "combine 4.6.6", "futures-util", "itoa 1.0.9", - "percent-encoding", + "percent-encoding 2.3.1", "pin-project-lite", "ryu", "sha1 0.6.1", - "tokio", + "tokio 1.34.0", "tokio-util", - "url", + "url 2.5.0", ] [[package]] @@ -4800,17 +4907,17 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding", + "percent-encoding 2.3.1", "pin-project-lite", "serde", "serde_json", "serde_urlencoded", "system-configuration", - "tokio", + "tokio 1.34.0", "tokio-native-tls", "tokio-util", "tower-service", - "url", + "url 2.5.0", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", @@ -4908,8 +5015,8 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "rust-embed-utils", "shellexpand", "syn 2.0.39", @@ -4956,7 +5063,7 @@ dependencies = [ "maybe-async", "md5", "minidom", - "percent-encoding", + "percent-encoding 2.3.1", "quick-xml 0.26.0", "reqwest", "serde", @@ -4964,9 +5071,9 @@ dependencies = [ "sha2 0.10.8", "thiserror", "time 0.3.30", - "tokio", + "tokio 1.34.0", "tokio-stream", - "url", + "url 2.5.0", ] [[package]] @@ -5136,7 +5243,7 @@ dependencies = [ "clap 2.34.0", "errors", "my-workspace-hack", - "tokio", + "tokio 1.34.0", "web_scrapers", ] @@ -5225,7 +5332,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", ] [[package]] @@ -5240,6 +5347,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b46e1121e8180c12ff69a742aabc4f310542b6ccb69f1691689ac17fdf8618aa" + [[package]] name = "serde" version = "1.0.193" @@ -5255,8 +5368,8 @@ version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -5296,7 +5409,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" dependencies = [ - "percent-encoding", + "percent-encoding 2.3.1", "serde", "thiserror", ] @@ -5307,7 +5420,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cac3f1e2ca2fe333923a1ae72caca910b98ed0630bb35ef6f8c8517d6e81afa" dependencies = [ - "percent-encoding", + "percent-encoding 2.3.1", "serde", "thiserror", ] @@ -5318,8 +5431,8 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -5352,8 +5465,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -5377,8 +5490,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -5531,8 +5644,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -5646,7 +5759,7 @@ dependencies = [ "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue", + "crossbeam-queue 0.3.8", "dotenvy", "either", "event-listener", @@ -5663,7 +5776,7 @@ dependencies = [ "native-tls", "once_cell", "paste", - "percent-encoding", + "percent-encoding 2.3.1", "rustls 0.21.9", "rustls-pemfile", "serde", @@ -5672,10 +5785,10 @@ dependencies = [ "smallvec 1.11.2", "sqlformat", "thiserror", - "tokio", + "tokio 1.34.0", "tokio-stream", "tracing", - "url", + "url 2.5.0", "webpki-roots 0.25.3", ] @@ -5685,8 +5798,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "sqlx-core", "sqlx-macros-core", "syn 1.0.109", @@ -5704,8 +5817,8 @@ dependencies = [ "heck", "hex", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "serde", "serde_json", "sha2 0.10.8", @@ -5715,8 +5828,8 @@ dependencies = [ "sqlx-sqlite", "syn 1.0.109", "tempfile", - "tokio", - "url", + "tokio 1.34.0", + "url 2.5.0", ] [[package]] @@ -5748,7 +5861,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "once_cell", - "percent-encoding", + "percent-encoding 2.3.1", "rand 0.8.5", "rsa", "serde", @@ -5818,11 +5931,11 @@ dependencies = [ "futures-util", "libsqlite3-sys", "log", - "percent-encoding", + "percent-encoding 2.3.1", "serde", "sqlx-core", "tracing", - "url", + "url 2.5.0", "urlencoding", ] @@ -5925,7 +6038,7 @@ dependencies = [ "native-tls", "once_cell", "pin-project-lite", - "pulldown-cmark", + "pulldown-cmark 0.9.3", "r2d2_redis", "rand 0.8.5", "redis_caching", @@ -5945,7 +6058,7 @@ dependencies = [ "symphonia", "time 0.3.30", "tokens", - "tokio", + "tokio 1.34.0", "toml", "tts_common", "twitch_common", @@ -5989,8 +6102,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", ] [[package]] @@ -6032,8 +6145,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustversion", "syn 2.0.39", ] @@ -6229,14 +6342,25 @@ dependencies = [ "symphonia-metadata", ] +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "unicode-ident", ] @@ -6246,8 +6370,8 @@ version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "unicode-ident", ] @@ -6353,8 +6477,8 @@ version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -6451,6 +6575,30 @@ dependencies = [ "utoipa", ] +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + [[package]] name = "tokio" version = "1.34.0" @@ -6481,6 +6629,16 @@ dependencies = [ "tokio-io", ] +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + [[package]] name = "tokio-executor" version = "0.1.10" @@ -6491,6 +6649,17 @@ dependencies = [ "futures 0.1.31", ] +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + [[package]] name = "tokio-io" version = "0.1.13" @@ -6508,8 +6677,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -6520,7 +6689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -6549,7 +6718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.1", - "tokio", + "tokio 1.34.0", "webpki", ] @@ -6561,7 +6730,7 @@ checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -6588,6 +6757,35 @@ dependencies = [ "tokio-reactor", ] +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue 0.2.3", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", +] + [[package]] name = "tokio-tungstenite" version = "0.15.0" @@ -6598,13 +6796,28 @@ dependencies = [ "log", "pin-project", "rustls 0.19.1", - "tokio", + "tokio 1.34.0", "tokio-rustls", "tungstenite", "webpki", "webpki-roots 0.21.1", ] +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + [[package]] name = "tokio-uds" version = "0.2.7" @@ -6633,7 +6846,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.34.0", "tracing", ] @@ -6670,8 +6883,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] @@ -6718,7 +6931,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -6752,7 +6965,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio", + "tokio 1.34.0", "tts_common", ] @@ -6790,7 +7003,7 @@ dependencies = [ "rustls-native-certs", "sha-1", "thiserror", - "url", + "url 2.5.0", "utf-8", "webpki", ] @@ -6820,7 +7033,7 @@ dependencies = [ "my-workspace-hack", "mysql_queries", "native-tls", - "pulldown-cmark", + "pulldown-cmark 0.9.3", "r2d2_redis", "rand 0.8.5", "redis_common", @@ -6833,7 +7046,7 @@ dependencies = [ "sqlx", "time 0.3.30", "tokens", - "tokio", + "tokio 1.34.0", "tokio-tungstenite", "toml", "tts_common", @@ -6865,7 +7078,7 @@ dependencies = [ "thiserror", "twitch_oauth2", "typed-builder", - "url", + "url 2.5.0", "version_check", ] @@ -6901,7 +7114,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "url", + "url 2.5.0", ] [[package]] @@ -6910,8 +7123,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -6963,6 +7176,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -6990,6 +7209,17 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + [[package]] name = "url" version = "2.5.0" @@ -6997,8 +7227,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna", - "percent-encoding", + "idna 0.5.0", + "percent-encoding 2.3.1", "serde", ] @@ -7025,7 +7255,7 @@ version = "0.0.1" dependencies = [ "my-workspace-hack", "once_cell", - "pulldown-cmark", + "pulldown-cmark 0.9.3", "regex", "unicode-segmentation", ] @@ -7106,8 +7336,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" dependencies = [ "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "regex", "syn 2.0.39", ] @@ -7155,6 +7385,22 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "version-sync" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "844f3d3a2467f15cb999f5af7775f6e108ac546d4f42365832ed4c755404f806" +dependencies = [ + "itertools 0.8.2", + "proc-macro2 0.4.30", + "pulldown-cmark 0.4.1", + "regex", + "semver-parser 0.9.0", + "syn 0.15.44", + "toml", + "url 1.7.2", +] + [[package]] name = "version_check" version = "0.9.4" @@ -7195,7 +7441,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -7224,7 +7470,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio", + "tokio 1.34.0", ] [[package]] @@ -7289,8 +7535,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", "wasm-bindgen-shared", ] @@ -7313,7 +7559,7 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ - "quote", + "quote 1.0.33", "wasm-bindgen-macro-support", ] @@ -7323,8 +7569,8 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7453,7 +7699,7 @@ dependencies = [ "sha2 0.9.9", "sqlx", "time 0.3.30", - "tokio", + "tokio 1.34.0", "toml", "twitch_api2", "twitch_common", @@ -7692,8 +7938,8 @@ version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 2.0.39", ] diff --git a/Cargo.toml b/Cargo.toml index 4729fc4d0d..5b062feb02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ members = [ "crates/service/plugins/users", # Vendored code + "crates/vendor/limitation", "crates/vendor/newrelic-telemetry-sdk-rust", # Vendored code diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 9eff06ef38..ffe732cdc1 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -104,7 +104,7 @@ pin-project-lite = "0.2.9" # NB: For middleware/futures upgrade sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database -limitation = "0.1.1" # Redis rate limiter +limitation = { path = "../../../vendor/limitation" } r2d2_redis = "0.14.0" # Elasticsearch diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 85df5ba18d..9bba30b65e 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -50,7 +50,7 @@ futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older ve sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database -limitation = "0.1.1" # Redis rate limiter +limitation = { path = "../../../vendor/limitation" } r2d2_redis = "0.14.0" redis_async = { package = "redis", version = "0.21.5", features = ["tokio-comp"] } diff --git a/crates/vendor/limitation/CHANGELOG.md b/crates/vendor/limitation/CHANGELOG.md new file mode 100644 index 0000000000..660198b082 --- /dev/null +++ b/crates/vendor/limitation/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## Unreleased + +## 0.1.1 / 2019-10-20 + +### Improvements + +- Add `readme` to `Cargo.toml` (@fnichol) + +## 0.1.0 / 2019-10-20 + +- The initial release. diff --git a/crates/vendor/limitation/CODE_OF_CONDUCT.md b/crates/vendor/limitation/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..c063ff3dcc --- /dev/null +++ b/crates/vendor/limitation/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at fnichol@nichol.ca. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml new file mode 100644 index 0000000000..86fb33b371 --- /dev/null +++ b/crates/vendor/limitation/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "limitation" +version = "0.1.1" +authors = ["Fletcher Nichol "] +edition = "2018" +license = "MPL-2.0" +repository = "https://github.com/fnichol/limitation" +documentation = "https://docs.rs/limitation" +readme = "README.md" +keywords = ["rate-limit", "rate-limiting", "limiter", "redis"] +categories = ["asynchronous", "web-programming"] +description = """ +A rate limiter using a fixed window counter for arbitrary keys, backed by +Redis. +""" + +[badges] +cirrus-ci = { repository = "fnichol/limitation" } + +[dependencies] +chrono = "0.4.9" +futures = "0.1.29" +redis = "0.13.0" +time = "0.1.42" + +[dev-dependencies] +tokio = "0.1.22" +version-sync = "0.8.1" diff --git a/crates/vendor/limitation/LICENSE.txt b/crates/vendor/limitation/LICENSE.txt new file mode 100644 index 0000000000..a612ad9813 --- /dev/null +++ b/crates/vendor/limitation/LICENSE.txt @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/crates/vendor/limitation/Makefile.toml b/crates/vendor/limitation/Makefile.toml new file mode 100644 index 0000000000..fa7177bbcf --- /dev/null +++ b/crates/vendor/limitation/Makefile.toml @@ -0,0 +1,6 @@ +[tasks.member-readme] +dependencies = [ + "generate-readme", + "update-readme-toc", + "format-readme", +] diff --git a/crates/vendor/limitation/README.md b/crates/vendor/limitation/README.md new file mode 100644 index 0000000000..53f02caab3 --- /dev/null +++ b/crates/vendor/limitation/README.md @@ -0,0 +1,298 @@ +# limitation + +| | | +| --------------: | ------------------------------------------- | +| CI | [![CI Status][badge-ci-overall]][ci] | +| Latest Version | [![Latest version][badge-version]][crate] | +| Documentation | [![Documentation][badge-docs]][docs] | +| Crate Downloads | [![Crate downloads][badge-crate-dl]][crate] | +| License | [![Crate license][badge-license]][github] | + +**Table of Contents** + + + +- [About](#about) +- [Usage](#usage) + - [Quick Example](#quick-example) + - [The Limiter Builder](#the-limiter-builder) +- [Examples](#examples) +- [Related Projects and References](#related-projects-and-references) +- [Ideas and Future Work](#ideas-and-future-work) +- [CI Status](#ci-status) + - [Build (master branch)](#build-master-branch) + - [Test (master branch)](#test-master-branch) + - [Check (master branch)](#check-master-branch) +- [Code of Conduct](#code-of-conduct) +- [Issues](#issues) +- [Contributing](#contributing) +- [Release History](#release-history) +- [Authors](#authors) +- [License](#license) + + + +A rate limiter using a fixed window counter for arbitrary keys, backed by Redis. + +## About + +This library provides a fixed window counter for arbitrary keys backed by a +Redis instance. The period length and per-period maximum limit are configurable +on setup. The communication with the backend is performing in a non-blocking, +asynchronous fashion allowing the library to pair with other async frameworks +such as Tokio, Actix, etc. + +_Note_: Currently pre-_async/await_ Futures are used (that is Futures 0.1.x) but +that may be upgraded in the near future as the Rust 1.39.0 release is near. + +## Usage + +Add `limitation` to your `Cargo.toml`: + +```toml +[dependencies] +limitation = "0.1.1" +``` + +### Quick Example + +The primary type is the [`Limiter`] which uses a builder to construct itself. +Once built, use the [`count`] method with a key representing a user, a session, +an interaction, etc. Note that `count` is a Future and therefore has to be +driving to completion with a runtime. For example, to run one count on a key of +`"10.0.0.5"`: + +[`limiter`]: struct.Limiter.html +[`count`]: struct.Limiter.html#method.count + +```rust +use limitation::Limiter; +use futures::Future; + +// Build a Limiter with a default rate with a local running Redis instance +let limiter = Limiter::build("redis://127.0.0.1/").finish()?; +// The key to count and track +let key = "10.0.0.5"; + +// Start and run a Tokio runtime to drive the Future to completion +tokio::run( + limiter + // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` + // containing a Status if the limit has been exceeded + .count(key) + // This example deals with both limit exceeded and any Redis connection issues. In this + // case we'll print the error to the standard error stream to show the current limit + // status + .map_err(|err| eprintln!("err: {}", err)) + // In this case we are under the limit and can print out the limit status to the + // standard output stream + .and_then(|status| { + println!("ok: {:?}", status); + Ok(()) + }), +); +``` + +### The Limiter Builder + +The builder for the `Limiter` has 2 settings which can be customized to the use +case: + +- [`limit`]: The high water mark for number of requests in the period. The + default is `5000`. +- [`period`]: A `Duration` for the period window. The default is 60 minutes. + +```rust +use limitation::Limiter; +use std::time::Duration; + +let limiter = Limiter::build("redis://127.0.0.1/") + .limit(5) + .period(Duration::from_secs(10)) + .finish()?; +``` + +[`limit`]: struct.Builder.html#method.limit +[`period`]: struct.Builder.html#method.period + +## Examples + +A simple example that uses this library can be found in [limitation-example]. + +[limitation-example]: + https://github.com/fnichol/limitation/tree/master/limitation-example + +## Related Projects and References + +The primary inspiration for the implementation was a blog post called [An +alternative approach to building a simple API Rate limiter using NodeJS and +Redis][blog-post] by [Atul R](https://twitter.com/masteratul94). + +Other research and references used for this library: + +- +- +- +- + +[blog-post]: https://blog.atulr.com/rate-limiter/ + +## Ideas and Future Work + +These are some ideas and potential future work for this project. If you're +reading this then maybe you're curious or interesting in helping out? Great! Be +sure to check out the [Contributing] section and dig in! + +- Investigate and offer alternative rate-limiting algorithms, notably a Sliding + Window solution. +- Add async Redis connection pooling with the `bb8` and `bb8-redis` crates to + reduce connection establishment delays. +- Add a `status` method on `Limiter` which returns they key's `Status` without + counting a request. +- Add `RedisServer` support in an integration testing suite, similar to the + infrastructure in the [redis] crate. + +[contributing]: + https://github.com/fnichol/limitation/tree/master/limitation#contributing +[redis]: https://github.com/mitsuhiko/redis-rs/blob/master/tests/support/mod.rs + +## CI Status + +### Build (master branch) + +| Operating System | Stable Rust | Nightly Rust | MSRV | +| ---------------: | ----------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| FreeBSD | [![FreeBSD Stable Build Status][badge-stable_freebsd-build]][ci-master] | [![FreeBSD Nightly Build Status][badge-nightly_freebsd-build]][ci-master] | [![FreeBSD Oldest Build Status][badge-oldest_freebsd-build]][ci-master] | +| Linux | [![Linux Stable Build Status][badge-stable_linux-build]][ci-master] | [![Linux Nightly Build Status][badge-nightly_linux-build]][ci-master] | [![Linux Oldest Build Status][badge-oldest_linux-build]][ci-master] | +| macOS | [![macOS Stable Build Status][badge-stable_macos-build]][ci-master] | [![macOS Nightly Build Status][badge-nightly_macos-build]][ci-master] | [![macOS Oldest Build Status][badge-oldest_macos-build]][ci-master] | +| Windows | [![Windows Stable Build Status][badge-stable_windows-build]][ci-master] | [![Windows Nightly Build Status][badge-nightly_windows-build]][ci-master] | [![Windows Oldest Build Status][badge-oldest_windows-build]][ci-master] | + +### Test (master branch) + +| Operating System | Stable Rust | Nightly Rust | MSRV | +| ---------------: | --------------------------------------------------------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------- | +| FreeBSD | [![FreeBSD Stable Test Status][badge-stable_freebsd-test]][ci-master] | [![FreeBSD Nightly Test Status][badge-nightly_freebsd-test]][ci-master] | [![FreeBSD Oldest Test Status][badge-oldest_freebsd-test]][ci-master] | +| Linux | [![Linux Stable Test Status][badge-stable_linux-test]][ci-master] | [![Linux Nightly Test Status][badge-nightly_linux-test]][ci-master] | [![Linux Oldest Test Status][badge-oldest_linux-test]][ci-master] | +| macOS | [![macOS Stable Test Status][badge-stable_macos-test]][ci-master] | [![macOS Nightly Test Status][badge-nightly_macos-test]][ci-master] | [![macOS Oldest Test Status][badge-oldest_macos-test]][ci-master] | +| Windows | [![Windows Stable Test Status][badge-stable_windows-test]][ci-master] | [![Windows Nightly Test Status][badge-nightly_windows-test]][ci-master] | [![Windows Oldest Test Status][badge-oldest_windows-test]][ci-master] | + +### Check (master branch) + +| | Status | +| ------ | ------------------------------------------------- | +| Lint | [![Lint Status][badge-check-lint]][ci-master] | +| Format | [![Format Status][badge-check-format]][ci-master] | + +## Code of Conduct + +This project adheres to the Contributor Covenant [code of +conduct][code-of-conduct]. By participating, you are expected to uphold this +code. Please report unacceptable behavior to fnichol@nichol.ca. + +## Issues + +If you have any problems with or questions about this project, please contact us +through a [GitHub issue][issues]. + +## Contributing + +You are invited to contribute to new features, fixes, or updates, large or +small; we are always thrilled to receive pull requests, and do our best to +process them as fast as we can. + +Before you start to code, we recommend discussing your plans through a [GitHub +issue][issues], especially for more ambitious contributions. This gives other +contributors a chance to point you in the right direction, give you feedback on +your design, and help you find out if someone else is working on the same thing. + +## Release History + +See the [changelog] for a full release history. + +## Authors + +Created and maintained by [Fletcher Nichol][fnichol] (). + +## License + +Licensed under the Mozilla Public License Version 2.0 ([LICENSE.txt][license]). + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the MPL-2.0 license, shall be +licensed as above, without any additional terms or conditions. + +[badge-check-format]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=check&script=format +[badge-check-lint]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=check&script=lint +[badge-ci-overall]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square +[badge-crate-dl]: + https://img.shields.io/crates/d/limitation.svg?style=flat-square +[badge-docs]: https://docs.rs/limitation/badge.svg?style=flat-square +[badge-license]: + https://img.shields.io/crates/l/limitation.svg?style=flat-square +[badge-nightly_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_freebsd&script=build +[badge-nightly_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_freebsd&script=test +[badge-nightly_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_linux&script=build +[badge-nightly_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_linux&script=test +[badge-nightly_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_macos&script=build +[badge-nightly_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_macos&script=test +[badge-nightly_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_windows&script=build +[badge-nightly_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_windows&script=test +[badge-oldest_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_freebsd&script=build +[badge-oldest_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_freebsd&script=test +[badge-oldest_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_linux&script=build +[badge-oldest_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_linux&script=test +[badge-oldest_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_macos&script=build +[badge-oldest_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_macos&script=test +[badge-oldest_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_windows&script=build +[badge-oldest_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_windows&script=test +[badge-stable_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_freebsd&script=build +[badge-stable_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_freebsd&script=test +[badge-stable_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_linux&script=build +[badge-stable_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_linux&script=test +[badge-stable_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_macos&script=build +[badge-stable_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_macos&script=test +[badge-stable_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_windows&script=build +[badge-stable_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_windows&script=test +[badge-version]: + https://img.shields.io/crates/v/limitation.svg?style=flat-square +[changelog]: + https://github.com/fnichol/limitation/blob/master/limitation/CHANGELOG.md +[ci]: https://cirrus-ci.com/github/fnichol/limitation +[ci-master]: https://cirrus-ci.com/github/fnichol/limitation/master +[code-of-conduct]: + https://github.com/fnichol/limitation/blob/master/limitation/CODE_OF_CONDUCT.md +[commonmark]: https://commonmark.org/ +[crate]: https://crates.io/crates/limitation +[docs]: https://docs.rs/limitation +[fnichol]: https://github.com/fnichol +[github]: https://github.com/fnichol/limitation +[issues]: https://github.com/fnichol/limitation/issues +[license]: + https://github.com/fnichol/limitation/blob/master/limitation/LICENSE.txt diff --git a/crates/vendor/limitation/README.tpl b/crates/vendor/limitation/README.tpl new file mode 100644 index 0000000000..ddc6547369 --- /dev/null +++ b/crates/vendor/limitation/README.tpl @@ -0,0 +1,156 @@ +# {{crate}} + +| | | +| --------------: | ------------------------------------------- | +| CI | [![CI Status][badge-ci-overall]][ci] | +| Latest Version | [![Latest version][badge-version]][crate] | +| Documentation | [![Documentation][badge-docs]][docs] | +| Crate Downloads | [![Crate downloads][badge-crate-dl]][crate] | +| License | [![Crate license][badge-license]][github] | + +**Table of Contents** + + + +{{readme}} + +## CI Status + +### Build (master branch) + +| Operating System | Stable Rust | Nightly Rust | MSRV | +| ---------------: | ----------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| FreeBSD | [![FreeBSD Stable Build Status][badge-stable_freebsd-build]][ci-master] | [![FreeBSD Nightly Build Status][badge-nightly_freebsd-build]][ci-master] | [![FreeBSD Oldest Build Status][badge-oldest_freebsd-build]][ci-master] | +| Linux | [![Linux Stable Build Status][badge-stable_linux-build]][ci-master] | [![Linux Nightly Build Status][badge-nightly_linux-build]][ci-master] | [![Linux Oldest Build Status][badge-oldest_linux-build]][ci-master] | +| macOS | [![macOS Stable Build Status][badge-stable_macos-build]][ci-master] | [![macOS Nightly Build Status][badge-nightly_macos-build]][ci-master] | [![macOS Oldest Build Status][badge-oldest_macos-build]][ci-master] | +| Windows | [![Windows Stable Build Status][badge-stable_windows-build]][ci-master] | [![Windows Nightly Build Status][badge-nightly_windows-build]][ci-master] | [![Windows Oldest Build Status][badge-oldest_windows-build]][ci-master] | + +### Test (master branch) + +| Operating System | Stable Rust | Nightly Rust | MSRV | +| ---------------: | --------------------------------------------------------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------- | +| FreeBSD | [![FreeBSD Stable Test Status][badge-stable_freebsd-test]][ci-master] | [![FreeBSD Nightly Test Status][badge-nightly_freebsd-test]][ci-master] | [![FreeBSD Oldest Test Status][badge-oldest_freebsd-test]][ci-master] | +| Linux | [![Linux Stable Test Status][badge-stable_linux-test]][ci-master] | [![Linux Nightly Test Status][badge-nightly_linux-test]][ci-master] | [![Linux Oldest Test Status][badge-oldest_linux-test]][ci-master] | +| macOS | [![macOS Stable Test Status][badge-stable_macos-test]][ci-master] | [![macOS Nightly Test Status][badge-nightly_macos-test]][ci-master] | [![macOS Oldest Test Status][badge-oldest_macos-test]][ci-master] | +| Windows | [![Windows Stable Test Status][badge-stable_windows-test]][ci-master] | [![Windows Nightly Test Status][badge-nightly_windows-test]][ci-master] | [![Windows Oldest Test Status][badge-oldest_windows-test]][ci-master] | + +### Check (master branch) + +| | Status | +| ------ | ------------------------------------------------- | +| Lint | [![Lint Status][badge-check-lint]][ci-master] | +| Format | [![Format Status][badge-check-format]][ci-master] | + +## Code of Conduct + +This project adheres to the Contributor Covenant [code of +conduct][code-of-conduct]. By participating, you are expected to uphold this +code. Please report unacceptable behavior to fnichol@nichol.ca. + +## Issues + +If you have any problems with or questions about this project, please contact us +through a [GitHub issue][issues]. + +## Contributing + +You are invited to contribute to new features, fixes, or updates, large or +small; we are always thrilled to receive pull requests, and do our best to +process them as fast as we can. + +Before you start to code, we recommend discussing your plans through a [GitHub +issue][issues], especially for more ambitious contributions. This gives other +contributors a chance to point you in the right direction, give you feedback on +your design, and help you find out if someone else is working on the same thing. + +## Release History + +See the [changelog] for a full release history. + +## Authors + +Created and maintained by [Fletcher Nichol][fnichol] (). + +## License + +Licensed under the Mozilla Public License Version 2.0 ([LICENSE.txt][license]). + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the MPL-2.0 license, shall be +licensed as above, without any additional terms or conditions. + +[badge-check-format]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=check&script=format +[badge-check-lint]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=check&script=lint +[badge-ci-overall]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square +[badge-crate-dl]: + https://img.shields.io/crates/d/limitation.svg?style=flat-square +[badge-docs]: https://docs.rs/limitation/badge.svg?style=flat-square +[badge-license]: + https://img.shields.io/crates/l/limitation.svg?style=flat-square +[badge-nightly_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_freebsd&script=build +[badge-nightly_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_freebsd&script=test +[badge-nightly_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_linux&script=build +[badge-nightly_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_linux&script=test +[badge-nightly_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_macos&script=build +[badge-nightly_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_macos&script=test +[badge-nightly_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_windows&script=build +[badge-nightly_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_nightly_windows&script=test +[badge-oldest_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_freebsd&script=build +[badge-oldest_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_freebsd&script=test +[badge-oldest_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_linux&script=build +[badge-oldest_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_linux&script=test +[badge-oldest_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_macos&script=build +[badge-oldest_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_macos&script=test +[badge-oldest_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_windows&script=build +[badge-oldest_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_1.35.0_windows&script=test +[badge-stable_freebsd-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_freebsd&script=build +[badge-stable_freebsd-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_freebsd&script=test +[badge-stable_linux-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_linux&script=build +[badge-stable_linux-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_linux&script=test +[badge-stable_macos-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_macos&script=build +[badge-stable_macos-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_macos&script=test +[badge-stable_windows-build]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_windows&script=build +[badge-stable_windows-test]: + https://img.shields.io/cirrus/github/fnichol/limitation.svg?style=flat-square&task=test_stable_windows&script=test +[badge-version]: + https://img.shields.io/crates/v/limitation.svg?style=flat-square +[changelog]: + https://github.com/fnichol/limitation/blob/master/limitation/CHANGELOG.md +[ci]: https://cirrus-ci.com/github/fnichol/limitation +[ci-master]: https://cirrus-ci.com/github/fnichol/limitation/master +[code-of-conduct]: + https://github.com/fnichol/limitation/blob/master/limitation/CODE_OF_CONDUCT.md +[commonmark]: https://commonmark.org/ +[crate]: https://crates.io/crates/limitation +[docs]: https://docs.rs/limitation +[fnichol]: https://github.com/fnichol +[github]: https://github.com/fnichol/limitation +[issues]: https://github.com/fnichol/limitation/issues +[license]: + https://github.com/fnichol/limitation/blob/master/limitation/LICENSE.txt diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs new file mode 100644 index 0000000000..e738ee4246 --- /dev/null +++ b/crates/vendor/limitation/src/lib.rs @@ -0,0 +1,377 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A rate limiter using a fixed window counter for arbitrary keys, backed by Redis. +//! +//! # About +//! +//! This library provides a fixed window counter for arbitrary keys backed by a Redis instance. The +//! period length and per-period maximum limit are configurable on setup. The communication with +//! the backend is performing in a non-blocking, asynchronous fashion allowing the library to pair +//! with other async frameworks such as Tokio, Actix, etc. +//! +//! *Note*: Currently pre-*async/await* Futures are used (that is Futures 0.1.x) but that may be +//! upgraded in the near future as the Rust 1.39.0 release is near. +//! +//! # Usage +//! +//! Add `limitation` to your `Cargo.toml`: +//! +//! ```toml +//! [dependencies] +//! limitation = "0.1.1" +//! ``` +//! +//! ## Quick Example +//! +//! The primary type is the [`Limiter`] which uses a builder to construct itself. Once built, use +//! the [`count`] method with a key representing a user, a session, an interaction, etc. Note that +//! `count` is a Future and therefore has to be driving to completion with a runtime. For example, +//! to run one count on a key of `"10.0.0.5"`: +//! +//! [`Limiter`]: struct.Limiter.html +//! [`count`]: struct.Limiter.html#method.count +//! +//! ```no_run +//! use limitation::Limiter; +//! use futures::Future; +//! +//! // Build a Limiter with a default rate with a local running Redis instance +//! let limiter = Limiter::build("redis://127.0.0.1/").finish()?; +//! // The key to count and track +//! let key = "10.0.0.5"; +//! +//! // Start and run a Tokio runtime to drive the Future to completion +//! tokio::run( +//! limiter +//! // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` +//! // containing a Status if the limit has been exceeded +//! .count(key) +//! // This example deals with both limit exceeded and any Redis connection issues. In this +//! // case we'll print the error to the standard error stream to show the current limit +//! // status +//! .map_err(|err| eprintln!("err: {}", err)) +//! // In this case we are under the limit and can print out the limit status to the +//! // standard output stream +//! .and_then(|status| { +//! println!("ok: {:?}", status); +//! Ok(()) +//! }), +//! ); +//! # Ok::<(), limitation::Error>(()) +//! ``` +//! +//! ## The Limiter Builder +//! +//! The builder for the `Limiter` has 2 settings which can be customized to the use case: +//! +//! - [`limit`]: The high water mark for number of requests in the period. The default is `5000`. +//! - [`period`]: A `Duration` for the period window. The default is 60 minutes. +//! +//! ```no_run +//! use limitation::Limiter; +//! use std::time::Duration; +//! +//! let limiter = Limiter::build("redis://127.0.0.1/") +//! .limit(5) +//! .period(Duration::from_secs(10)) +//! .finish()?; +//! # Ok::<(), limitation::Error>(()) +//! ``` +//! +//! [`limit`]: struct.Builder.html#method.limit +//! [`period`]: struct.Builder.html#method.period +//! +//! # Examples +//! +//! A simple example that uses this library can be found in [limitation-example]. +//! +//! [limitation-example]: https://github.com/fnichol/limitation/tree/master/limitation-example +//! +//! # Related Projects and References +//! +//! The primary inspiration for the implementation was a blog post called [An alternative approach +//! to building a simple API Rate limiter using NodeJS and Redis][blog-post] by [Atul +//! R](https://twitter.com/masteratul94). +//! +//! Other research and references used for this library: +//! +//! - +//! - +//! - +//! - +//! +//! [blog-post]: https://blog.atulr.com/rate-limiter/ +//! +//! # Ideas and Future Work +//! +//! These are some ideas and potential future work for this project. If you're reading this then +//! maybe you're curious or interesting in helping out? Great! Be sure to check out the +//! [Contributing] section and dig in! +//! +//! - Investigate and offer alternative rate-limiting algorithms, notably a Sliding Window +//! solution. +//! - Add async Redis connection pooling with the `bb8` and `bb8-redis` crates to reduce +//! connection establishment delays. +//! - Add a `status` method on `Limiter` which returns they key's `Status` without counting a +//! request. +//! - Add `RedisServer` support in an integration testing suite, similar to the infrastructure in +//! the [redis] crate. +//! +//! [Contributing]: https://github.com/fnichol/limitation/tree/master/limitation#contributing +//! [redis]: https://github.com/mitsuhiko/redis-rs/blob/master/tests/support/mod.rs + +#![doc(html_root_url = "https://docs.rs/limitation/0.1.1")] +#![deny(missing_docs)] + +use chrono::SubsecRound; +use futures::Future; +use redis::Client; +use std::convert::TryInto; +use std::error; +use std::fmt; +use std::ops::Add; +use std::time::Duration; + +/// The default limit of requests in a period +const DEFAULT_LIMIT: usize = 5000; +/// The default length of the period in seconds +const DEFAULT_PERIOD_SECS: u64 = 60 * 60; + +/// A rate limiter using a fixed window counter, backed by Redis. +/// +/// The per-period limit and the period duration are customizable when building an instance. +/// +/// The [`count`] method is the primary unit of interaction which requires a key representing a +/// user, a session, an interaction, etc. This method returns a Future which needs a runtime to +/// drive the task to completion asynchronously. +/// +/// [`count`]: #method.count +#[derive(Clone, Debug)] +pub struct Limiter { + /// The Redis client + client: Client, + /// The per-period limit + limit: usize, + /// The period duration + period: Duration, +} + +impl Limiter { + /// Returns a builder for a `Limiter`. + /// + /// The [`finish`] method will build the final `Limiter` and perform a **synchronous** + /// connection test to the Redis backend. + /// + /// [`finish`]: struct.Builder.html#method.finish + pub fn build(redis_url: &str) -> Builder { + Builder { + redis_url, + limit: DEFAULT_LIMIT, + period: Duration::from_secs(DEFAULT_PERIOD_SECS), + } + } + + /// Counts a request on a key over a period and returns a [`Status`]. + /// + /// The `Status` type gives the caller a current state snapshot for the given key. If the limit + /// is exceeded a `Error::LimitExceeded` will be returned which also contains a `Status`. + /// Critically, the `Status` contains a time when the next period begins and the limit will + /// reset. The time is a "unix timestamp" in UTC time. + /// + /// # Errors + /// + /// Returns an `Err` if: + /// + /// - The limit has been exceeded in the current period + /// - A client error has occurred + /// - A time computation failed + /// + /// [`Status`]: struct.Status.html + pub fn count>(&self, key: K) -> impl Future { + let limit = self.limit; + + self.track(key).and_then(move |(count, reset_epoch_utc)| { + let status = build_status(count, limit, reset_epoch_utc); + + if count > limit { + Err(Error::LimitExceeded(status)) + } else { + Ok(status) + } + }) + } + + /// Tracks the given key in a period and returns the count and TTL for the key in seconds. + fn track>(&self, key: K) -> impl Future { + let key = key.into(); + let exipres = self.period.as_secs(); + + self.client + .get_async_connection() + .from_err() + .and_then(move |con| { + // The seed of this approach is outlined Atul R in a blog post about rate limiting + // using NodeJS and Redis. For more details, see + // https://blog.atulr.com/rate-limiter/ + let mut pipe = redis::pipe(); + pipe.atomic() + .cmd("SET") + .arg(&key) + .arg(0) + .arg("EX") + .arg(exipres) + .arg("NX") + .ignore() + .cmd("INCR") + .arg(&key) + .cmd("TTL") + .arg(&key); + + pipe.query_async(con) + .from_err() + .and_then(|(_, (count, ttl)): (_, (usize, u64))| { + Ok((count, epoch_utc_plus(Duration::from_secs(ttl))?)) + }) + }) + } +} + +/// A report for a given key containing the limit status. +/// +/// The status contains the following information: +/// +/// - [`limit`]: the maximum number of requests allowed in the current period +/// - [`remaining`]: how many requests are left in the current period +/// - [`reset_epoch_utc`]: a UNIX timestamp in UTC approximately when the next period will begin +/// +/// [`limit`]: #method.limit +/// [`remaining`]: #method.remaining +/// [`reset_epoch_utc`]: #method.reset_epoch_utc +#[derive(Clone, Debug)] +pub struct Status { + limit: usize, + remaining: usize, + reset_epoch_utc: usize, +} + +impl Status { + /// Returns the maximum number of requests permitted in the current period. + pub fn limit(&self) -> usize { + self.limit + } + + /// Returns the number of requests remaining in the current period. + pub fn remaining(&self) -> usize { + self.remaining + } + + /// Returns a UNIX timestamp in UTC approximately when the next period will begin. + pub fn reset_epoch_utc(&self) -> usize { + self.reset_epoch_utc + } +} + +/// A builder for a [`Limiter`]. +/// +/// [`Limiter`]: struct.Limiter.html +pub struct Builder<'a> { + redis_url: &'a str, + limit: usize, + period: Duration, +} + +impl Builder<'_> { + /// Sets a new maximum limit for the Limiter. + pub fn limit(&mut self, limit: usize) -> &mut Self { + self.limit = limit; + self + } + + /// Sets a new period duration for the Limiter. + pub fn period(&mut self, period: Duration) -> &mut Self { + self.period = period; + self + } + + /// Finializes and returns a `Limiter`. + /// + /// Note that this method will connect to the Redis server to test its connection which is a + /// **synchronous** operation. + /// + /// # Errors + /// + /// Returns an `Err` if the Redis client fails to be created or fails to connect. + pub fn finish(&self) -> Result { + Ok(Limiter { + client: Client::open(self.redis_url)?, + limit: self.limit, + period: self.period, + }) + } +} + +/// Error type for this crate. +#[derive(Debug)] +pub enum Error { + /// The Redis client failed to connect or run a query. + Client(redis::RedisError), + /// The limit is exceeded for a key. + LimitExceeded(Status), + /// A time conversion failed. + Time(time::OutOfRangeError), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::Client(ref err) => write!(f, "client error ({})", err), + Error::LimitExceeded(ref status) => write!(f, "rate limit exceeded ({:?})", status), + Error::Time(ref err) => write!(f, "time conversion error ({})", err), + } + } +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + Error::Client(ref err) => err.source(), + Error::LimitExceeded(_) => None, + Error::Time(ref err) => err.source(), + } + } +} + +impl From for Error { + fn from(err: redis::RedisError) -> Self { + Error::Client(err) + } +} + +impl From for Error { + fn from(err: time::OutOfRangeError) -> Self { + Error::Time(err) + } +} + +/// Builds a `Status`. +fn build_status(count: usize, limit: usize, reset_epoch_utc: usize) -> Status { + let remaining = if count >= limit { 0 } else { limit - count }; + + Status { + limit, + remaining, + reset_epoch_utc, + } +} + +/// Calculates a timestamp for "now plus a duration". +fn epoch_utc_plus(duration: Duration) -> Result { + Ok(chrono::Utc::now() + .add(chrono::Duration::from_std(duration)?) + .round_subsecs(0) + .timestamp() + .try_into() + .unwrap_or(0)) +} diff --git a/crates/vendor/limitation/tests/version-numbers.rs b/crates/vendor/limitation/tests/version-numbers.rs new file mode 100644 index 0000000000..e0b0acaaaa --- /dev/null +++ b/crates/vendor/limitation/tests/version-numbers.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#[test] +fn test_readme_deps() { + version_sync::assert_markdown_deps_updated!("README.md"); +} + +#[test] +fn test_html_root_url() { + version_sync::assert_html_root_url_updated!("src/lib.rs"); +} From 1a3b02b5d2635eac00e876093f2cb8046a1b08f9 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 21:46:53 -0500 Subject: [PATCH 075/136] Adding docs --- Cargo.lock | 3 +- crates/_types/enums/Cargo.toml | 1 + .../model_weights/weights_category.rs | 4 ++- .../by_table/model_weights/weights_types.rs | 3 +- .../web/storyteller_web/src/api_doc.rs | 9 +++--- .../web/storyteller_web/src/api_doc_server.rs | 1 - .../inference/enqueue_tts_request.rs | 9 ++---- .../weights/list_availible_weights.rs | 28 +++++++++++++++---- 8 files changed, 38 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a95e836b72..00a71c2794 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,7 +1053,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2178,6 +2178,7 @@ dependencies = [ "sqlx-mysql", "strum", "toml", + "utoipa", ] [[package]] diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 1cb6c94409..4479a39957 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -21,6 +21,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } [dev-dependencies] diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index c2bca7f24d..5818ec59ab 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -5,8 +5,10 @@ use strum::EnumCount; #[cfg(test)] use strum::EnumIter; +use utoipia::ToSchema; + #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { #[serde(rename = "image_generation")] ImageGeneration, diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index e7ff3c3c97..0ef556d927 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -4,10 +4,11 @@ use std::collections::BTreeSet; use strum::EnumCount; #[cfg(test)] use strum::EnumIter; +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f76feb1e76..f90d681efd 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,11 +1,12 @@ -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; -use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use crate::http_server::endpoints::*; +use tokens::tokens::*; use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] + + #[openapi(paths(crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights), + components(schemas(ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index e3fc865736..8f305b120e 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -56,7 +56,6 @@ use std::{ pub mod api_doc; use api_doc::ApiDoc; - use futures::future::LocalBoxFuture; use utoipa::OpenApi; #[actix_web::main] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 32ae10d58d..41adb52569 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -43,23 +43,20 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; -#[derive(Deserialize)] -#[derive(ToSchema)] +#[derive(Deserialize,ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, voice_token: String, } -#[derive(Serialize)] -#[derive(ToSchema)] +#[derive(Serialize,ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } -#[derive(Debug)] -#[derive(ToSchema)] +#[derive(Debug,ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index 044b05b113..f658f0a62a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -13,20 +13,23 @@ use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, }; + use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; -#[derive(Deserialize)] +use utoipia::Schema; + +#[derive(Deserialize,Schema)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, pub page_size: u16, pub page_index : u16, } -#[derive(Serialize)] +#[derive(Serialize,Schema)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, @@ -36,14 +39,14 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } -#[derive(Deserialize)] +#[derive(Deserialize,Schema)] pub struct ListWeightsByPathInfo { pub username: Option, pub weights_type: Option, pub weights_category: Option, } -#[derive(Serialize)] +#[derive(Serialize,Schema)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, @@ -83,7 +86,7 @@ pub struct ModelWeightForList { } -#[derive(Debug)] +#[derive(Debug,Schema)] pub enum ListWeightError { NotAuthorized, ServerError, @@ -104,6 +107,19 @@ impl ResponseError for ListWeightError { } } +#[utoipa::path( + get, + path = "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", + responses( + (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightError), + (status = 500, description = "Server error", body = ListWeightError), + ), + params( + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request") + ("path" = ListWeightsByPathInfo, description = "Path for Request") + ) +)] pub async fn list_availible_weights_handler( http_request: HttpRequest, path: web::Path, @@ -245,6 +261,7 @@ pub async fn list_availible_weights_handler( creator_display_name: weights.creator_display_name, creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + // TODO FIX THIS when we align. bookmarks: random_bool, likes: rng.gen_range(0..1000), }).collect::>(), @@ -257,6 +274,5 @@ pub async fn list_availible_weights_handler( let body = serde_json::to_string(&response) .map_err(|e| ListWeightError::ServerError)?; - Ok(HttpResponse::Ok().content_type("application/json").body(body)) } From fc122cb996254b2a7fcc815a3304ccf22a9fcdef Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 21:54:06 -0500 Subject: [PATCH 076/136] upgrade chrono with vendored limitation --- Cargo.lock | 15 ++++++++++----- crates/_types/reusable_types/Cargo.toml | 2 +- crates/lib/api_clients/fakeyou_client/Cargo.toml | 2 +- crates/lib/datetimes/Cargo.toml | 2 +- .../lib/deprecated/http_server_common/Cargo.toml | 2 +- crates/lib/elasticsearch_schema/Cargo.toml | 2 +- crates/lib/jobs_common/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/lib/sqlite_queries/Cargo.toml | 2 +- crates/lib/tts_common/Cargo.toml | 2 +- crates/lib/web_scrapers/Cargo.toml | 2 +- crates/programs/aichatbot-sidecar/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- crates/service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- crates/service/job/tts_download_job/Cargo.toml | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../http_server/web_utils/redis_rate_limiter.rs | 1 + crates/service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/limitation/src/lib.rs | 13 ++++++++++++- 28 files changed, 48 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed3a73d44a..bec222b29c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,6 +468,12 @@ dependencies = [ "tokio 1.34.0", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -1037,18 +1043,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", - "winapi 0.3.9", + "windows-targets 0.48.5", ] [[package]] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 7b87cc85e7..3d69e9c2ed 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # (none) # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } # Serialization serde = "1.0.125" diff --git a/crates/lib/api_clients/fakeyou_client/Cargo.toml b/crates/lib/api_clients/fakeyou_client/Cargo.toml index bfeea54bf1..ba496308e8 100644 --- a/crates/lib/api_clients/fakeyou_client/Cargo.toml +++ b/crates/lib/api_clients/fakeyou_client/Cargo.toml @@ -20,7 +20,7 @@ tokens = { path = "../../../_types/tokens" } # External bytes = "1.3" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.9.0" diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 8cd99557fa..f6abdd5a8b 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,7 +13,7 @@ name = "datetimes" path = "src/lib.rs" [dependencies] -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } once_cell = "1.18.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 82f7101d72..80a5a75d4c 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -21,7 +21,7 @@ reusable_types = { path = "../../../_types/reusable_types" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/elasticsearch_schema/Cargo.toml b/crates/lib/elasticsearch_schema/Cargo.toml index 7926f55ac1..9ddfa27ad5 100644 --- a/crates/lib/elasticsearch_schema/Cargo.toml +++ b/crates/lib/elasticsearch_schema/Cargo.toml @@ -23,7 +23,7 @@ elasticsearch = "8.5.0-alpha.1" log = "0.4.14" # Serialization -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.108" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index ad3a0c9a3e..c7ae49d883 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -24,7 +24,7 @@ redis_common = { path = "../redis_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index bcdff745f8..f690a27244 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -25,7 +25,7 @@ tokens = { path = "../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/lib/sqlite_queries/Cargo.toml b/crates/lib/sqlite_queries/Cargo.toml index 17f25f8d0b..3536eee1bc 100644 --- a/crates/lib/sqlite_queries/Cargo.toml +++ b/crates/lib/sqlite_queries/Cargo.toml @@ -20,7 +20,7 @@ errors = { path = "../../lib/errors" } tokens = { path = "../../_types/tokens" } # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 6e42e04e6a..8487151516 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -20,7 +20,7 @@ user_input_common = { path = "../user_input_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index a5e01837ce..7e5e5760eb 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -19,7 +19,7 @@ errors = { path = "../../lib/errors" } # External bytes = "1.3" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" strum = { version = "0.25", features = ["derive"] } # Fancy enum iteration, etc. diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index cebc56b8a1..6eea1f30e4 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -32,7 +32,7 @@ web_scrapers = { path = "../../lib/web_scrapers" } # External base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 6a803df2a4..d12b95a657 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -26,7 +26,7 @@ tokens = { path = "../../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 6a4c49eb42..01a63af0fb 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -35,7 +35,7 @@ subprocess_common = { path = "../../../lib/subprocess_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index b1385b31e8..7cb457f502 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -46,7 +46,7 @@ tts_common = { path = "../../../lib/tts_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index f82dbe6589..5ee58f1fca 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -43,7 +43,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index b57ea434e8..05ad634eb9 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 2cbff86bf7..84695a25d4 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -32,7 +32,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index aa55cde374..17d900b4c1 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 6aefb7d7f6..68d74191ce 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -26,7 +26,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 94e3520a39..2602ff7a8b 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -28,7 +28,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 5e879949eb..f7ccc93edb 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -24,7 +24,7 @@ url_config = { path = "../../../lib/url_config" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" once_cell = "1.18.0" diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 5d852fa380..2047df1610 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -34,7 +34,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" lazy_static = "1.4.0" # Regex caching (TODO: Use OnceCell instead) log = "0.4.14" diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index abbd3f4e96..faa0b178cd 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" #once_cell = "1.9.0" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index ffe732cdc1..857d55dd08 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -58,7 +58,7 @@ users_component = { path = "../../plugins/users" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hex = "0.4.3" diff --git a/crates/service/web/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs b/crates/service/web/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs index 6e5efee68f..8c9667621e 100644 --- a/crates/service/web/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs +++ b/crates/service/web/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs @@ -67,6 +67,7 @@ impl RedisRateLimiter { Err(err) => match err { Error::Client(_) => Ok(()), // NB: Fail open for failure to connect to Redis Error::Time(_) => Ok(()), // NB: Fail open for key parsing + Error::ChronoTime(_) => Ok(()), // NB: Fail open for key parsing Error::LimitExceeded(_) => Err(RateLimiterError::RateLimitExceededError), }, } diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 9bba30b65e..91d9fddd93 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -27,7 +27,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs index e738ee4246..649b30eb46 100644 --- a/crates/vendor/limitation/src/lib.rs +++ b/crates/vendor/limitation/src/lib.rs @@ -133,6 +133,7 @@ use std::error; use std::fmt; use std::ops::Add; use std::time::Duration; +use time; /// The default limit of requests in a period const DEFAULT_LIMIT: usize = 5000; @@ -321,6 +322,8 @@ pub enum Error { LimitExceeded(Status), /// A time conversion failed. Time(time::OutOfRangeError), + /// A time conversion failed (Chrono). + ChronoTime(chrono::OutOfRangeError), } impl fmt::Display for Error { @@ -329,6 +332,7 @@ impl fmt::Display for Error { Error::Client(ref err) => write!(f, "client error ({})", err), Error::LimitExceeded(ref status) => write!(f, "rate limit exceeded ({:?})", status), Error::Time(ref err) => write!(f, "time conversion error ({})", err), + Error::ChronoTime(ref err) => write!(f, "time conversion error ({})", err), } } } @@ -339,6 +343,7 @@ impl error::Error for Error { Error::Client(ref err) => err.source(), Error::LimitExceeded(_) => None, Error::Time(ref err) => err.source(), + Error::ChronoTime(ref err) => err.source(), } } } @@ -355,6 +360,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: chrono::OutOfRangeError) -> Self { + Error::ChronoTime(err) + } +} + /// Builds a `Status`. fn build_status(count: usize, limit: usize, reset_epoch_utc: usize) -> Status { let remaining = if count >= limit { 0 } else { limit - count }; @@ -367,7 +378,7 @@ fn build_status(count: usize, limit: usize, reset_epoch_utc: usize) -> Status { } /// Calculates a timestamp for "now plus a duration". -fn epoch_utc_plus(duration: Duration) -> Result { +fn epoch_utc_plus(duration: Duration) -> Result { Ok(chrono::Utc::now() .add(chrono::Duration::from_std(duration)?) .round_subsecs(0) From 32cfb10323b8f4bdc0a6373d9f46173f5207a475 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 00:55:28 -0500 Subject: [PATCH 077/136] added weights list documentation --- .../model_weights/weights_category.rs | 2 +- crates/_types/enums/src/common/visibility.rs | 5 ++-- .../_types/tokens/src/tokens/model_weights.rs | 3 +- crates/_types/tokens/src/tokens/users.rs | 4 +-- .../web/get_inference_job_status.rs | 1 + .../web/storyteller_web/src/api_doc.rs | 28 ++++++++++++++----- .../weights/list_availible_weights.rs | 14 +++++----- 7 files changed, 37 insertions(+), 20 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index 5818ec59ab..f6c4e6ed3f 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -5,8 +5,8 @@ use strum::EnumCount; #[cfg(test)] use strum::EnumIter; -use utoipia::ToSchema; +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { diff --git a/crates/_types/enums/src/common/visibility.rs b/crates/_types/enums/src/common/visibility.rs index f7240ca332..7e9c0341f3 100644 --- a/crates/_types/enums/src/common/visibility.rs +++ b/crates/_types/enums/src/common/visibility.rs @@ -15,9 +15,10 @@ use strum::EnumIter; /// See also: https://docs.rs/sqlx/0.4.0-beta.1/sqlx/trait.Type.html /// /// *DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY!* -/// + +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type)] +#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type,ToSchema)] #[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum Visibility { diff --git a/crates/_types/tokens/src/tokens/model_weights.rs b/crates/_types/tokens/src/tokens/model_weights.rs index ba29c1f8fb..a63f2d9028 100644 --- a/crates/_types/tokens/src/tokens/model_weights.rs +++ b/crates/_types/tokens/src/tokens/model_weights.rs @@ -5,8 +5,9 @@ use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for the "model_weights" table. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct ModelWeightToken(pub String); diff --git a/crates/_types/tokens/src/tokens/users.rs b/crates/_types/tokens/src/tokens/users.rs index 2e28fca9fe..4a678ebf9f 100644 --- a/crates/_types/tokens/src/tokens/users.rs +++ b/crates/_types/tokens/src/tokens/users.rs @@ -4,9 +4,9 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; - +use utoipa::ToSchema; /// The primary key for users. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct UserToken(pub String); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 615ef68ca9..9f6137cf01 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; + use chrono::{DateTime, Utc}; use log::warn; use sqlx::{MySql, MySqlPool}; diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f90d681efd..f86ca46b48 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,12 +1,26 @@ -use crate::http_server::endpoints::*; -use tokens::tokens::*; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; +use crate::http_server::endpoints::weights::list_availible_weights::*; +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::generic_inference_jobs::*; +use tokens::tokens::model_weights::*; +use tokens::tokens::users::*; +use enums::common::visibility::Visibility; +use chrono::{DateTime, Utc}; use utoipa::OpenApi; -#[derive(OpenApi)] -#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), - components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] - #[openapi(paths(crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights), - components(schemas(ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, + crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler + ),components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + + ModelWeightToken,UserToken,Visibility, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] pub struct ApiDoc; + diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs index f658f0a62a..5ababcdff9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs @@ -20,16 +20,16 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; -use utoipia::Schema; +use utoipa::ToSchema; -#[derive(Deserialize,Schema)] +#[derive(Deserialize,ToSchema)] pub struct ListAvailibleWeightsQuery { pub sort_ascending: Option, pub page_size: u16, pub page_index : u16, } -#[derive(Serialize,Schema)] +#[derive(Serialize,ToSchema)] pub struct ListAvailibleWeightsSuccessResponse { pub success: bool, pub weights: Vec, @@ -39,14 +39,14 @@ pub struct ListAvailibleWeightsSuccessResponse { pub cursor_previous: Option, } -#[derive(Deserialize,Schema)] +#[derive(Deserialize,ToSchema)] pub struct ListWeightsByPathInfo { pub username: Option, pub weights_type: Option, pub weights_category: Option, } -#[derive(Serialize,Schema)] +#[derive(Serialize,ToSchema)] pub struct ModelWeightForList { pub weight_token: ModelWeightToken, @@ -86,7 +86,7 @@ pub struct ModelWeightForList { } -#[derive(Debug,Schema)] +#[derive(Debug,ToSchema)] pub enum ListWeightError { NotAuthorized, ServerError, @@ -116,7 +116,7 @@ impl ResponseError for ListWeightError { (status = 500, description = "Server error", body = ListWeightError), ), params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request") + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), ("path" = ListWeightsByPathInfo, description = "Path for Request") ) )] From f76928d0019b0f15075e6e8338731624d50679a5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 01:28:54 -0500 Subject: [PATCH 078/136] Delete weight documentation added --- .../web/storyteller_web/src/api_doc.rs | 16 +++++++--- .../endpoints/weights/delete_weight.rs | 30 ++++++++++++++++--- .../web_utils/response_success_helpers.rs | 4 +-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f86ca46b48..9eb6412d18 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,7 +1,10 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; -use crate::http_server::endpoints::weights::list_availible_weights::*; + +use crate::http_server::endpoints::weights::list_availible_weights::*; +use crate::http_server::web_utils::response_success_helpers::simple_json_success::SimpleGenericJsonSuccess; +use crate::http_server::endpoints::weights::delete_weight::*; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -16,11 +19,16 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, - crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler - ),components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, + crate::http_server::endpoints::weights::delete_weight::delete_weight_handler + ),components(schemas( + SimpleGenericJsonSuccess, + EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, ModelWeightToken,UserToken,Visibility, ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList)))] + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError + )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index f8f547b3b3..a589b1eefe 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -10,7 +10,12 @@ use enums::by_table::audit_logs::audit_log_entity_action::AuditLogEntityAction:: use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; -use mysql_queries::queries::model_weights::delete_weights::{delete_weights_as_user,delete_weights_as_mod,undelete_weights_as_user,undelete_weights_as_mod}; +use mysql_queries::queries::model_weights::delete_weights::{ + delete_weights_as_user, + delete_weights_as_mod, + undelete_weights_as_user, + undelete_weights_as_mod +}; use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use tokens::tokens::model_weights::ModelWeightToken; @@ -22,21 +27,23 @@ use crate::util::delete_role_disambiguation::{delete_role_disambiguation, Delete use enums::common::visibility::Visibility; use tokens::tokens::users::UserToken; +use utoipa::ToSchema; -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct DeleteWeightPathInfo { weight_token: String, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct DeleteWeightRequest { set_delete: bool, as_mod: bool } + // =============== Error Response =============== -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize,ToSchema)] pub enum DeleteWeightError { BadInput(String), NotFound, @@ -66,6 +73,21 @@ impl fmt::Display for DeleteWeightError { } } + +#[utoipa::path( + get, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = DeleteWeightError), + (status = 401, description = "Not authorized", body = DeleteWeightError), + (status = 500, description = "Server error", body = DeleteWeightError), + ), + params( + ("request" = DeleteWeightRequest, description = "Payload for Request"), + ("path" = DeleteWeightPathInfo, description = "Path for Request") + ) +)] pub async fn delete_weight_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs index dae959059f..16908c9ac9 100644 --- a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs +++ b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs @@ -1,6 +1,6 @@ use actix_web::HttpResponse; - -#[derive(Serialize)] +use utoipa::ToSchema; +#[derive(Serialize,ToSchema)] pub struct SimpleGenericJsonSuccess { pub success: bool, } From 8a6678c7f52981c6da6c70dd0e27cea435917e29 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 6 Dec 2023 02:12:57 -0500 Subject: [PATCH 079/136] move workspace-hack under crates/ --- Cargo.toml | 4 ++-- crates/_types/buckets/Cargo.toml | 2 +- crates/_types/enums/Cargo.toml | 2 +- crates/_types/reusable_types/Cargo.toml | 2 +- crates/_types/tokens/Cargo.toml | 2 +- crates/cli/database_backup/Cargo.toml | 2 +- crates/cli/database_cleanup/Cargo.toml | 2 +- crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_upload_media_file/Cargo.toml | 2 +- crates/cli/elasticsearch-cli/Cargo.toml | 2 +- crates/cli/kubectl-pod-cleanup/Cargo.toml | 2 +- crates/cli/lru-delete-models/Cargo.toml | 2 +- crates/cli/scrape_test/Cargo.toml | 2 +- crates/lib/actix_helpers/Cargo.toml | 2 +- crates/lib/api_clients/cloud_storage/Cargo.toml | 2 +- crates/lib/api_clients/google_drive_common/Cargo.toml | 2 +- crates/lib/api_clients/twitch_common/Cargo.toml | 2 +- crates/lib/bootstrap/Cargo.toml | 2 +- crates/lib/caching/memory_caching/Cargo.toml | 2 +- crates/lib/caching/redis_caching/Cargo.toml | 2 +- crates/lib/collections/Cargo.toml | 2 +- crates/lib/concurrency/Cargo.toml | 2 +- crates/lib/cookies/Cargo.toml | 2 +- crates/lib/crockford/Cargo.toml | 2 +- crates/lib/datetimes/Cargo.toml | 2 +- crates/lib/deprecated/config/Cargo.toml | 2 +- crates/lib/deprecated/container_common/Cargo.toml | 2 +- crates/lib/deprecated/http_server_common/Cargo.toml | 2 +- crates/lib/easyenv/Cargo.toml | 2 +- crates/lib/elasticsearch_schema/Cargo.toml | 2 +- crates/lib/email_sender/Cargo.toml | 2 +- crates/lib/errors/Cargo.toml | 2 +- crates/lib/files/archives/Cargo.toml | 2 +- crates/lib/files/filesys/Cargo.toml | 2 +- crates/lib/files/hashing/Cargo.toml | 2 +- crates/lib/files/images/Cargo.toml | 2 +- crates/lib/files/media/Cargo.toml | 2 +- crates/lib/files/mimetypes/Cargo.toml | 2 +- crates/lib/idempotency/Cargo.toml | 2 +- crates/lib/jobs_common/Cargo.toml | 2 +- crates/lib/logging/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/lib/password/Cargo.toml | 2 +- crates/lib/redis_common/Cargo.toml | 2 +- crates/lib/server_environment/Cargo.toml | 2 +- crates/lib/sqlx_mysql_helpers/Cargo.toml | 2 +- crates/lib/storyteller_root/Cargo.toml | 2 +- crates/lib/subprocess_common/Cargo.toml | 2 +- crates/lib/testing/Cargo.toml | 2 +- crates/lib/tts_common/Cargo.toml | 2 +- crates/lib/url_config/Cargo.toml | 2 +- crates/lib/user_input_common/Cargo.toml | 2 +- crates/lib/web_scrapers/Cargo.toml | 2 +- .../my-workspace-hack}/.gitattributes | 0 {my-workspace-hack => crates/my-workspace-hack}/Cargo.toml | 0 {my-workspace-hack => crates/my-workspace-hack}/build.rs | 0 {my-workspace-hack => crates/my-workspace-hack}/src/lib.rs | 0 crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- crates/service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- crates/service/job/tts_download_job/Cargo.toml | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 2 +- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/plugins/inference_workloads/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- crates/service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 2 +- service_cpu.Dockerfile | 1 - service_gpu.Dockerfile | 1 - 75 files changed, 70 insertions(+), 72 deletions(-) rename {my-workspace-hack => crates/my-workspace-hack}/.gitattributes (100%) rename {my-workspace-hack => crates/my-workspace-hack}/Cargo.toml (100%) rename {my-workspace-hack => crates/my-workspace-hack}/build.rs (100%) rename {my-workspace-hack => crates/my-workspace-hack}/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 5b062feb02..7649065a0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,6 @@ members = [ "crates/vendor/limitation", "crates/vendor/newrelic-telemetry-sdk-rust", - # Vendored code - "my-workspace-hack", + # cargo-hakari workspace hack (compilation speedup hack) + "crates/my-workspace-hack", ] diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml index e3d3607ed1..80145886da 100644 --- a/crates/_types/buckets/Cargo.toml +++ b/crates/_types/buckets/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" # Internal crockford = { path = "../../lib/crockford" } errors = { path = "../../lib/errors" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External # (none yet) diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 1cb6c94409..6182365ff6 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1.0.125" sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] } sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 3d69e9c2ed..3fee6d8cfa 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -26,7 +26,7 @@ serde_derive = "1.0.125" # MySql Database sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # Test serialization diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index e85c19e259..c3c9ea1c73 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -22,7 +22,7 @@ rand = "0.8.5" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } diff --git a/crates/cli/database_backup/Cargo.toml b/crates/cli/database_backup/Cargo.toml index 3d14de4112..691890dfae 100644 --- a/crates/cli/database_backup/Cargo.toml +++ b/crates/cli/database_backup/Cargo.toml @@ -28,4 +28,4 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/database_cleanup/Cargo.toml b/crates/cli/database_cleanup/Cargo.toml index d23a8f7f81..1600fd11f6 100644 --- a/crates/cli/database_cleanup/Cargo.toml +++ b/crates/cli/database_cleanup/Cargo.toml @@ -28,4 +28,4 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index d2ef105cce..c07d0431ac 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -39,7 +39,7 @@ tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index c1f7ed8790..5831756152 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -34,7 +34,7 @@ log = "0.4.14" tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index 9876936696..f6ca014556 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -41,7 +41,7 @@ elasticsearch = "8.5.0-alpha.1" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/kubectl-pod-cleanup/Cargo.toml b/crates/cli/kubectl-pod-cleanup/Cargo.toml index 78beb7813b..a0bb79e162 100644 --- a/crates/cli/kubectl-pod-cleanup/Cargo.toml +++ b/crates/cli/kubectl-pod-cleanup/Cargo.toml @@ -23,7 +23,7 @@ log = "0.4.20" once_cell = "1.18.0" rand = "0.8.5" regex = "1.10.0" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # Async Runtime # NB: Not using Tokio here since everything is blocking so far. diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml index d70b183fed..f7af132bb6 100644 --- a/crates/cli/lru-delete-models/Cargo.toml +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -20,7 +20,7 @@ logging = { path = "../../lib/logging" } # External clap = { version = "4.4.7", features = ["derive"] } walkdir = "2.4.0" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index 658c01be7d..99753086ef 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -21,4 +21,4 @@ web_scrapers = { path = "../../lib/web_scrapers" } # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/actix_helpers/Cargo.toml b/crates/lib/actix_helpers/Cargo.toml index 66e3958e0c..89c0cbf5d9 100644 --- a/crates/lib/actix_helpers/Cargo.toml +++ b/crates/lib/actix_helpers/Cargo.toml @@ -34,7 +34,7 @@ futures-util = "0.3.29" # Serialization serde = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index 888ffabc67..ee5435b17a 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -28,4 +28,4 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/google_drive_common/Cargo.toml b/crates/lib/api_clients/google_drive_common/Cargo.toml index 56abcedc47..d15ae569df 100644 --- a/crates/lib/api_clients/google_drive_common/Cargo.toml +++ b/crates/lib/api_clients/google_drive_common/Cargo.toml @@ -22,5 +22,5 @@ subprocess_common = { path = "../../subprocess_common" } anyhow = "1.0.75" log = "0.4.14" tempdir = "0.3.7" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/twitch_common/Cargo.toml b/crates/lib/api_clients/twitch_common/Cargo.toml index 81fd70e80c..b377c34ee2 100644 --- a/crates/lib/api_clients/twitch_common/Cargo.toml +++ b/crates/lib/api_clients/twitch_common/Cargo.toml @@ -31,4 +31,4 @@ twitch_oauth2 = "0.6.0-rc.2" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/bootstrap/Cargo.toml b/crates/lib/bootstrap/Cargo.toml index 556e517ffe..3baecb1333 100644 --- a/crates/lib/bootstrap/Cargo.toml +++ b/crates/lib/bootstrap/Cargo.toml @@ -23,7 +23,7 @@ server_environment = { path = "../server_environment" } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/caching/memory_caching/Cargo.toml b/crates/lib/caching/memory_caching/Cargo.toml index 3f541a9647..06a8da7dfc 100644 --- a/crates/lib/caching/memory_caching/Cargo.toml +++ b/crates/lib/caching/memory_caching/Cargo.toml @@ -22,4 +22,4 @@ anyhow = "1.0.75" # In-memory caching lru_time_cache = "0.11.11" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/caching/redis_caching/Cargo.toml b/crates/lib/caching/redis_caching/Cargo.toml index f644278cab..9f31f5bdbd 100644 --- a/crates/lib/caching/redis_caching/Cargo.toml +++ b/crates/lib/caching/redis_caching/Cargo.toml @@ -23,7 +23,7 @@ r2d2_redis = "0.14.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml index a31edbb379..f74421ec5a 100644 --- a/crates/lib/collections/Cargo.toml +++ b/crates/lib/collections/Cargo.toml @@ -16,7 +16,7 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../errors" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External # KEEP THIS TO A MINIMUM OR NIL. diff --git a/crates/lib/concurrency/Cargo.toml b/crates/lib/concurrency/Cargo.toml index 0fe7b6d05f..c9511ffd58 100644 --- a/crates/lib/concurrency/Cargo.toml +++ b/crates/lib/concurrency/Cargo.toml @@ -14,7 +14,7 @@ path = "src/lib.rs" [dependencies] # Internal -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External # (None yet) diff --git a/crates/lib/cookies/Cargo.toml b/crates/lib/cookies/Cargo.toml index 792b27c484..8abda155b4 100644 --- a/crates/lib/cookies/Cargo.toml +++ b/crates/lib/cookies/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../errors" } hmac = "0.10.0" # jwt signing jwt = "0.13.0" sha2 = "0.9.5" # hmac signing -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/crockford/Cargo.toml b/crates/lib/crockford/Cargo.toml index e66d821a32..45fa6a9f04 100644 --- a/crates/lib/crockford/Cargo.toml +++ b/crates/lib/crockford/Cargo.toml @@ -16,7 +16,7 @@ path = "src/lib.rs" [dependencies] rand = "0.8.5" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index f6abdd5a8b..7b69245835 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -15,7 +15,7 @@ path = "src/lib.rs" [dependencies] chrono = { version = "0.4.22", features = ["serde"] } once_cell = "1.18.0" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # (none) \ No newline at end of file diff --git a/crates/lib/deprecated/config/Cargo.toml b/crates/lib/deprecated/config/Cargo.toml index f330624014..6142b3221e 100644 --- a/crates/lib/deprecated/config/Cargo.toml +++ b/crates/lib/deprecated/config/Cargo.toml @@ -21,4 +21,4 @@ easyenv = { path = "../../../lib/easyenv" } # External anyhow = "1.0.75" url = "2.4.1" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/container_common/Cargo.toml b/crates/lib/deprecated/container_common/Cargo.toml index ea97b1fc3d..7d0e9cb9dd 100644 --- a/crates/lib/deprecated/container_common/Cargo.toml +++ b/crates/lib/deprecated/container_common/Cargo.toml @@ -24,7 +24,7 @@ log = "0.4.14" # Serialization serde = { version = "1.0.125", features = ["derive"] } toml = "0.5.8" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 80a5a75d4c..4b7259c477 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -39,7 +39,7 @@ language-tags = "0.3.2" # Parsing single tags richly serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] actix-rt = "2.9.0" diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index 3af179ed9e..8bfc0d1044 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -20,7 +20,7 @@ storyteller_root = { path = "../storyteller_root" } dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/elasticsearch_schema/Cargo.toml b/crates/lib/elasticsearch_schema/Cargo.toml index 9ddfa27ad5..37dac8d081 100644 --- a/crates/lib/elasticsearch_schema/Cargo.toml +++ b/crates/lib/elasticsearch_schema/Cargo.toml @@ -26,7 +26,7 @@ log = "0.4.14" chrono = { version = "0.4.22", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index 323b40abd4..feb580c8c2 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -12,4 +12,4 @@ path = "src/lib.rs" [dependencies] errors = { path = "../errors" } lettre = "0.11.2" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/errors/Cargo.toml b/crates/lib/errors/Cargo.toml index eb50e9efbe..6268214f7d 100644 --- a/crates/lib/errors/Cargo.toml +++ b/crates/lib/errors/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # Consider the dependencies here wisely. # Ideally we have almost nothing as a dependency of this crate. anyhow = "1.0.75" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/archives/Cargo.toml b/crates/lib/files/archives/Cargo.toml index a0c6d9e2d5..d260c780c6 100644 --- a/crates/lib/files/archives/Cargo.toml +++ b/crates/lib/files/archives/Cargo.toml @@ -20,7 +20,7 @@ errors = { path = "../../errors" } # External zip = "0.6.6" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index 52dec1d28b..bbd7dbf27f 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -16,7 +16,7 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../../errors" } -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } log = "0.4.20" # External diff --git a/crates/lib/files/hashing/Cargo.toml b/crates/lib/files/hashing/Cargo.toml index 3099e9f2db..adb00e6168 100644 --- a/crates/lib/files/hashing/Cargo.toml +++ b/crates/lib/files/hashing/Cargo.toml @@ -22,7 +22,7 @@ bcrypt = "0.8" # password hashes data-encoding = "2.3.2" md-5 = "0.9.1" # DO NOT USE FOR SERIOUS CRYPTO (we use this for gravatar) ring = "0.17.0-alpha.10" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml index a3ee93dbaa..3327f009b6 100644 --- a/crates/lib/files/images/Cargo.toml +++ b/crates/lib/files/images/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../errors" } # External image = "0.24.1" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/files/media/Cargo.toml b/crates/lib/files/media/Cargo.toml index a099af5003..47d371d0b8 100644 --- a/crates/lib/files/media/Cargo.toml +++ b/crates/lib/files/media/Cargo.toml @@ -26,7 +26,7 @@ log = "0.4.14" # Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/ symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", rev = "e1a70095c4409478c1ded499fb8dd4afa540fb90", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/mimetypes/Cargo.toml b/crates/lib/files/mimetypes/Cargo.toml index 524b378157..6fe34f2034 100644 --- a/crates/lib/files/mimetypes/Cargo.toml +++ b/crates/lib/files/mimetypes/Cargo.toml @@ -16,7 +16,7 @@ path = "src/lib.rs" [dependencies] # External infer = "0.11" # Mimetype detection -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/idempotency/Cargo.toml b/crates/lib/idempotency/Cargo.toml index b3d8a69dc1..578404b0fb 100644 --- a/crates/lib/idempotency/Cargo.toml +++ b/crates/lib/idempotency/Cargo.toml @@ -19,7 +19,7 @@ path = "src/lib.rs" # External uuid = { version = "0.8.2", features = ["v4"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index c7ae49d883..1e6767ea01 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -27,4 +27,4 @@ anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index d5db48f134..bf708a7b84 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.20" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index f690a27244..4b09061ac0 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -44,5 +44,5 @@ tokio = { version = "1.6.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/password/Cargo.toml b/crates/lib/password/Cargo.toml index ecd994b18f..f8063637b3 100644 --- a/crates/lib/password/Cargo.toml +++ b/crates/lib/password/Cargo.toml @@ -19,7 +19,7 @@ errors = { path = "../errors" } # External bcrypt = "0.8" # password hashes -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } #hmac = "0.10.0" # jwt signing #jwt = "0.13.0" #sha2 = "0.9.5" # hmac signing diff --git a/crates/lib/redis_common/Cargo.toml b/crates/lib/redis_common/Cargo.toml index 8bfaee7bd7..28aa9fc780 100644 --- a/crates/lib/redis_common/Cargo.toml +++ b/crates/lib/redis_common/Cargo.toml @@ -27,4 +27,4 @@ serde_json = "1.0.108" # TODO: Get rid of in favor of std::time time = "0.3.30" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/server_environment/Cargo.toml b/crates/lib/server_environment/Cargo.toml index 6bc9d336d7..5ece74511a 100644 --- a/crates/lib/server_environment/Cargo.toml +++ b/crates/lib/server_environment/Cargo.toml @@ -15,7 +15,7 @@ path = "src/lib.rs" [dependencies] # External -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index 647e980426..57245fc712 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -18,6 +18,6 @@ errors = { path = "../errors" } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/storyteller_root/Cargo.toml b/crates/lib/storyteller_root/Cargo.toml index d5c5758e14..60143bba40 100644 --- a/crates/lib/storyteller_root/Cargo.toml +++ b/crates/lib/storyteller_root/Cargo.toml @@ -14,7 +14,7 @@ path = "src/lib.rs" [dependencies] # External -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] serial_test = "2.0.0" diff --git a/crates/lib/subprocess_common/Cargo.toml b/crates/lib/subprocess_common/Cargo.toml index 24548b670f..a9fd74418c 100644 --- a/crates/lib/subprocess_common/Cargo.toml +++ b/crates/lib/subprocess_common/Cargo.toml @@ -15,7 +15,7 @@ path = "src/lib.rs" [dependencies] # External -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # Serialization #serde = "1.0.125" diff --git a/crates/lib/testing/Cargo.toml b/crates/lib/testing/Cargo.toml index 91b58a9597..de25b327df 100644 --- a/crates/lib/testing/Cargo.toml +++ b/crates/lib/testing/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External # None yet diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 8487151516..985c8e11fc 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -29,4 +29,4 @@ unicode-segmentation = "1.10.1" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/url_config/Cargo.toml b/crates/lib/url_config/Cargo.toml index 295e2fc059..91ee274c39 100644 --- a/crates/lib/url_config/Cargo.toml +++ b/crates/lib/url_config/Cargo.toml @@ -21,4 +21,4 @@ reusable_types = { path = "../../_types/reusable_types" } # External actix-web = "4.4.0" anyhow = "1.0.75" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml index 2b89e8c730..d737ce2074 100644 --- a/crates/lib/user_input_common/Cargo.toml +++ b/crates/lib/user_input_common/Cargo.toml @@ -19,4 +19,4 @@ once_cell = "1.18.0" pulldown-cmark = "0.9.3" # Markdown regex = "1.5.4" unicode-segmentation = "1.10.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index 7e5e5760eb..c9f8bced1e 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -35,7 +35,7 @@ reqwest = { version = "0.11", features = ["json"] } # HTTP Client # Scraping rss = "2.0.2" scraper = "0.17.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/my-workspace-hack/.gitattributes b/crates/my-workspace-hack/.gitattributes similarity index 100% rename from my-workspace-hack/.gitattributes rename to crates/my-workspace-hack/.gitattributes diff --git a/my-workspace-hack/Cargo.toml b/crates/my-workspace-hack/Cargo.toml similarity index 100% rename from my-workspace-hack/Cargo.toml rename to crates/my-workspace-hack/Cargo.toml diff --git a/my-workspace-hack/build.rs b/crates/my-workspace-hack/build.rs similarity index 100% rename from my-workspace-hack/build.rs rename to crates/my-workspace-hack/build.rs diff --git a/my-workspace-hack/src/lib.rs b/crates/my-workspace-hack/src/lib.rs similarity index 100% rename from my-workspace-hack/src/lib.rs rename to crates/my-workspace-hack/src/lib.rs diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index d12b95a657..a1dac56f4d 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -42,4 +42,4 @@ tokio = { version = "1.6.0", features = ["macros"] } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 01a63af0fb..42bcfa7769 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -60,4 +60,4 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 7cb457f502..965444fc3a 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -82,4 +82,4 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 5ee58f1fca..d0008b9eac 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -78,4 +78,4 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 05ad634eb9..64fe625e3a 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -52,4 +52,4 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 84695a25d4..6ac55b3d1d 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -58,4 +58,4 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 17d900b4c1..2d3d54d8d0 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -52,4 +52,4 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 68d74191ce..84c844bd40 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -50,4 +50,4 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 2602ff7a8b..e9c1bb359a 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -81,7 +81,7 @@ toml = "0.5.8" # Text processing pulldown-cmark = "0.9.3" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index f7ccc93edb..9623a9f9dc 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -51,7 +51,7 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt async-trait = "0.1.58" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] mockall = "0.11.3" diff --git a/crates/service/plugins/inference_workloads/Cargo.toml b/crates/service/plugins/inference_workloads/Cargo.toml index 98243eb9e6..6d1f817577 100644 --- a/crates/service/plugins/inference_workloads/Cargo.toml +++ b/crates/service/plugins/inference_workloads/Cargo.toml @@ -15,4 +15,4 @@ path = "src/lib.rs" [dependencies] # External -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 2047df1610..ca9835c9ae 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -65,4 +65,4 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", r2d2_redis = "0.14.0" byteorder = "1.5.0" strum_macros = "0.25.3" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index faa0b178cd..5ed83e2aa9 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -45,4 +45,4 @@ actix-web = "4.4.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 857d55dd08..669066a316 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -159,7 +159,7 @@ pulldown-cmark = "0.9.3" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 91d9fddd93..2d29dcc45b 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -90,7 +90,7 @@ serde_json = "1.0.108" toml = "0.5.8" # Temporary (just to ask for oauth credentials during dev) -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 2e8dfb8c5e..d0e3422320 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -21,7 +21,7 @@ flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } tokio = { version = "1.1.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] rand = "0.8.5" diff --git a/service_cpu.Dockerfile b/service_cpu.Dockerfile index 4ddabe5b1e..d297b2b6d9 100644 --- a/service_cpu.Dockerfile +++ b/service_cpu.Dockerfile @@ -74,7 +74,6 @@ COPY Cargo.lock . COPY Cargo.toml . COPY .sqlx/ .sqlx COPY crates/ ./crates -COPY my-workspace-hack/ ./my-workspace-hack COPY includes/ ./includes COPY test_data/ ./test_data diff --git a/service_gpu.Dockerfile b/service_gpu.Dockerfile index a3dd798adb..bc2159be58 100644 --- a/service_gpu.Dockerfile +++ b/service_gpu.Dockerfile @@ -72,7 +72,6 @@ COPY Cargo.lock . COPY Cargo.toml . COPY .sqlx/ .sqlx COPY crates/ ./crates -COPY my-workspace-hack ./my-workspace-hack COPY includes/ ./includes COPY test_data/ ./test_data From f7d5ad496cafb910fa344cc3cca53c91612fd501 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 14:01:31 -0500 Subject: [PATCH 080/136] Update docs --- .../web/storyteller_web/src/api_doc.rs | 19 +- .../endpoints/weights/create_weight.rs | 296 ++++-------------- .../endpoints/weights/get_weight.rs | 7 +- .../endpoints/weights/update_weight.rs | 23 +- 4 files changed, 101 insertions(+), 244 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 9eb6412d18..33c7054cd3 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -20,15 +20,18 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, - crate::http_server::endpoints::weights::delete_weight::delete_weight_handler + crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, + crate::http_server::endpoints::weights::update_weight::update_weight_handler, + crate::http_server::endpoints::weights::get_weight::get_weight_handler, + crate::http_server::endpoints::weights::create_weight::create_weight_handler, ),components(schemas( - SimpleGenericJsonSuccess, - EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + SimpleGenericJsonSuccess, + EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, - ModelWeightToken,UserToken,Visibility, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, - DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError - )))] + ModelWeightToken,UserToken,Visibility, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError + )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 6b9194135f..5e502e6a57 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -28,8 +28,9 @@ use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_me use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; -#[derive(Serialize)] +#[derive(Serialize,ToSchema)] pub struct UploadWeightsSuccessResponse { pub success: bool, pub upload_token: ModelWeightToken, @@ -41,241 +42,80 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { ]) }); -pub async fn upload_weights_handler( - http_request: HttpRequest, - server_state: web::Data>, - mut multipart_payload: Multipart, -) -> Result { +// pub async fn create_weight_handler( +// http_request: HttpRequest, +// server_state: web::Data>, +// mut multipart_payload: Multipart, +// ) -> Result { - let response = handle_weight_upload( - &http_request, - &server_state, - multipart_payload, - &ALLOWED_MIME_TYPES).await?; - - // send a body with the required payload along side the multipart upload + - let weight_upload_token = response.to_model_token(); +// let weight_upload_token = response.to_model_token(); - let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { - success: true, - upload_token: weight_upload_token, - }; +// let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { +// success: true, +// upload_token: weight_upload_token, +// }; - let body = serde_json::to_string(&response) - .map_err(|e| UploadError::ServerError)?; +// let body = serde_json::to_string(&response) +// .map_err(|e| UploadError::ServerError)?; - return Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)); -} +// return Ok(HttpResponse::Ok() +// .content_type("application/json") +// .body(body)); +// } -pub enum SuccessCase { - WeightAlreadyUploaded { - existing_upload_token: ModelWeightToken, - }, - WeightSuccessfullyUploaded { - upload_token: ModelWeightToken, - } -} +// pub enum SuccessCase { +// WeightAlreadyUploaded { +// existing_upload_token: ModelWeightToken, +// }, +// WeightSuccessfullyUploaded { +// upload_token: ModelWeightToken, +// } +// } -impl SuccessCase { - pub fn to_model_token(self) -> ModelWeightToken { - match self { - SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, - SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, - } - } -} +// impl SuccessCase { +// pub fn to_model_token(self) -> ModelWeightToken { +// match self { +// SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, +// SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, +// } +// } +// } -pub async fn handle_weight_upload( - http_request: &HttpRequest, - server_state: &web::Data>, - mut multipart_payload: Multipart, - allowed_mimetypes: &HashSet<&'static str>, -) -> Result { - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - error!("MySql pool error: {:?}", err); - UploadError::ServerError - })?; +// let token = create_weight(CreateModelWeightsArgs { +// token: &ModelWeightToken::generate(), +// weights_type: WeightsType::LoRA, // to do parse multi part for this +// weights_category: WeightsCategory::ImageGeneration, +// title: upload_weights_request.file_name.as_deref(), +// maybe_thumbnail_token: &ThumbnailToken::generate(), +// description_markdown: "", +// description_rendered_html: "", +// creator_user_token: maybe_user_token.as_str(), +// creator_ip_address: &ip_address, +// creator_set_visibility, +// maybe_last_update_user_token: "", +// original_download_url: "", +// original_filename: upload_weights_request.file_name.as_deref(), +// file_size_bytes: file_size_bytes as i32, +// file_checksum_sha2: hash, +// private_bucket_hash: private_bucket_hash, +// maybe_private_bucket_prefix: None, +// maybe_private_bucket_extension: None, +// cached_user_ratings_total_count: 0, +// cached_user_ratings_positive_count: 0, +// cached_user_ratings_negative_count: 0, +// maybe_cached_user_ratings_ratio: None, +// version: 0, +// mysql_pool: &server_state.mysql_pool, +// }).await +// .map_err(|e| { +// error!("Error creating weights upload: {:?}", e); +// UploadError::ServerError +// })?; - // ==================== READ SESSION ==================== // +// info!("new weights upload id: {} token: {:?}", record_id, &token); - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) - .await - .map_err(|e| { - error!("Session checker error: {:?}", e); - UploadError::ServerError - })?; - - // ==================== RATE LIMIT ==================== // - - let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, - }; - - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - - // ==================== READ MULTIPART REQUEST ==================== // - - let upload_weights_request = drain_multipart_request(multipart_payload) - .await - .map_err(|e| { - // TODO: Error handling could be nicer. - UploadError::BadInput("bad request".to_string()) - })?; - - let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token - .ok_or(UploadError::BadInput("no uuid".to_string()))?; - - let maybe_existing_upload = - get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) - .await; - - match maybe_existing_upload { - Err(err) => { - error!("Error checking for previous upload: {:?}", err); - return Err(UploadError::ServerError); - } - Ok(Some(upload)) => { - // File already uploaded and frontend didn't protect us. - return Ok(SuccessCase::WeightAlreadyUploaded { - existing_upload_token: upload.token, - }); - } - Ok(None) => { - // Proceed. - } - } - - if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { - return Err(UploadError::BadInput(reason)); - } - - let creator_set_visibility = maybe_user_session - .as_ref() - .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. - .unwrap_or(Visibility::default()); - - let ip_address = get_request_ip(&http_request); - - let maybe_user_token = maybe_user_session - .map(|session| session.get_strongly_typed_user_token()); - - let maybe_file_size_bytes = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| bytes.len()); - - info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); - - let maybe_mimetype = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) - .flatten(); - - - let bytes = match upload_weights_request.file_bytes { - None => return Err(UploadError::BadInput("missing file contents".to_string())), - Some(bytes) => bytes, - }; - - let hash = sha256_hash_bytes(&bytes) - .map_err(|io_error| { - error!("Problem hashing bytes: {:?}", io_error); - UploadError::ServerError - })?; - - let file_size_bytes = bytes.len(); - - let mut maybe_duration_millis = None; - let mut maybe_codec_name = None; - let mut media_upload_type = None; - - if let Some(mimetype) = maybe_mimetype.as_deref() { - - if !allowed_mimetypes.contains(mimetype) { - // NB: Don't let our error message inject malicious strings - let filtered_mimetype = mimetype - .chars() - .filter(|c| c.is_ascii()) - .filter(|c| c.is_alphanumeric() || *c == '/') - .collect::(); - return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); - } - media_upload_type = match mimetype { - "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), - _ => None, - }; - } - - // This is fine for now we just don't expose the path! - let public_upload_path = WeightUploadOriginalFilePath::generate_new(); - - info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); - - // need to upload based on visibility - server_state.public_bucket_client.upload_file_with_content_type( - public_upload_path.get_full_object_path_str(), - bytes.as_ref(), - mime_type) - .await - .map_err(|e| { - warn!("Upload media bytes to bucket error: {:?}", e); - UploadError::ServerError - })?; - - let token = create_weight(CreateModelWeightsArgs { - token: &ModelWeightToken::generate(), - weights_type: WeightsType::LoRA, // to do parse multi part for this - weights_category: WeightsCategory::ImageGeneration, - title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: &ThumbnailToken::generate(), - description_markdown: "", - description_rendered_html: "", - creator_user_token: maybe_user_token.as_str(), - creator_ip_address: &ip_address, - creator_set_visibility, - maybe_last_update_user_token: "", - original_download_url: "", - original_filename: upload_weights_request.file_name.as_deref(), - file_size_bytes: file_size_bytes as i32, - file_checksum_sha2: hash, - private_bucket_hash: private_bucket_hash, - maybe_private_bucket_prefix: None, - maybe_private_bucket_extension: None, - cached_user_ratings_total_count: 0, - cached_user_ratings_positive_count: 0, - cached_user_ratings_negative_count: 0, - maybe_cached_user_ratings_ratio: None, - version: 0, - mysql_pool: &server_state.mysql_pool, - }).await - .map_err(|e| { - error!("Error creating weights upload: {:?}", e); - UploadError::ServerError - })?; - - info!("new weights upload id: {} token: {:?}", record_id, &token); - - Ok(SuccessCase::WeightSuccessfullyUploaded { - upload_token: token, - }) -} +// Ok() +// } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 9ca0ed207e..6bb712c677 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -18,7 +18,8 @@ use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use enums::by_table::model_weights::weights_types::WeightsType; use enums::by_table::model_weights::weights_category::WeightsCategory; -#[derive(Serialize, Clone)] +use utoipa::ToSchema +#[derive(Serialize, Clone,ToSchema)] pub struct GetWeightResponse { success: bool, weight_token: ModelWeightToken, @@ -44,12 +45,12 @@ pub struct GetWeightResponse { created_at: DateTime, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct GetWeightPathInfo { weight_token: String, } -#[derive(Debug)] +#[derive(Debug,ToSchema)] pub enum GetWeightError { NotAuthorized, NotFound, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 0e287729a3..47f087f23a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -15,9 +15,9 @@ use mysql_queries::queries::model_weights::update_weight::{ update_weights, Upda use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; - +use utopia::ToSchema; /// TODO will eventually be polymorphic -#[derive(Deserialize)] +#[derive(Deserialize, ToSchema)] pub struct UpdateWeightRequest { pub title: Option, pub thumbnail_token: Option, @@ -28,7 +28,7 @@ pub struct UpdateWeightRequest { pub visibility: Option, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub struct UpdateWeightResponse { pub success: bool, } @@ -41,7 +41,7 @@ pub struct UpdateWeightPathInfo { // =============== Error Response =============== -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub enum UpdateWeightError { BadInput(String), NotFound, @@ -72,7 +72,20 @@ impl fmt::Display for UpdateWeightError { } // =============== Handler =============== - +#[utoipa::path( + post, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = UpdateWeightError), + (status = 401, description = "Not authorized", body = UpdateWeightError), + (status = 500, description = "Server error", body = UpdateWeightError), + ), + params( + ("request" = UpdateWeightRequest, description = "Payload for Request"), + ("path" = UpdateWeightPathInfo, description = "Path for Request") + ) + )] pub async fn update_weight_handler( http_request: HttpRequest, path: Path, From 3a4ca7b54835de1998f8009305d5a58b9f0587e5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 16:14:08 -0500 Subject: [PATCH 081/136] weights doc complete. --- crates/_types/enums/Cargo.toml | 2 +- crates/_types/tokens/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../web/storyteller_web/src/api_doc.rs | 20 +- .../web/storyteller_web/src/api_doc_server.rs | 1 + .../endpoints/weights/delete_weight.rs | 2 +- .../endpoints/weights/get_weight.rs | 20 +- .../endpoints/weights/update_weight.rs | 4 +- .../endpoints/weights/upload_weight.rs | 271 ++++++++++++++++++ 9 files changed, 308 insertions(+), 16 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 4479a39957..71e7726e57 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -21,7 +21,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index e85c19e259..308f2e9a3f 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -24,7 +24,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index ffe732cdc1..d466063034 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -160,7 +160,7 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 33c7054cd3..9db38a306a 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,10 +1,11 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; - use crate::http_server::endpoints::weights::list_availible_weights::*; -use crate::http_server::web_utils::response_success_helpers::simple_json_success::SimpleGenericJsonSuccess; +use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; +use crate::http_server::endpoints::weights::get_weight::*; +use crate::http_server::endpoints::weights::update_weight::*; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -23,15 +24,20 @@ use utoipa::OpenApi; crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, - crate::http_server::endpoints::weights::create_weight::create_weight_handler, ),components(schemas( + // Tokens + UserToken,Visibility, + // SimpleGenericJsonSuccess, + // Inference EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, - - ModelWeightToken,UserToken,Visibility, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError,WeightsCategory,WeightsType, + //Model Weights + ModelWeightToken,WeightsCategory,WeightsType, + GetWeightPathInfo,GetWeightResponse,GetWeightError, + UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, + ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError, ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, - DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index 8f305b120e..e3fc865736 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -56,6 +56,7 @@ use std::{ pub mod api_doc; use api_doc::ApiDoc; + use futures::future::LocalBoxFuture; use utoipa::OpenApi; #[actix_web::main] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index a589b1eefe..7806fa8583 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -75,7 +75,7 @@ impl fmt::Display for DeleteWeightError { #[utoipa::path( - get, + delete, path = "/weight/{weight_token}", responses( (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 6bb712c677..1c0857b5b7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -18,8 +18,9 @@ use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use enums::by_table::model_weights::weights_types::WeightsType; use enums::by_table::model_weights::weights_category::WeightsCategory; -use utoipa::ToSchema -#[derive(Serialize, Clone,ToSchema)] +use utoipa::ToSchema; + +#[derive(Serialize, Clone, ToSchema)] pub struct GetWeightResponse { success: bool, weight_token: ModelWeightToken, @@ -72,7 +73,20 @@ impl ResponseError for GetWeightError { } } } - +#[utoipa::path( + get, + path = "/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = GetWeightResponse), + (status = 400, description = "Bad input", body = GetWeightError), + (status = 401, description = "Not authorized", body = GetWeightError), + (status = 500, description = "Server error", body = GetWeightError), + ), + params( + ("request" = UpdateWeightRequest, description = "Payload for Request"), + ("path" = GetWeightPathInfo, description = "Path for Request") + ) + )] pub async fn get_weight_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 47f087f23a..8ddddca33e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -15,7 +15,7 @@ use mysql_queries::queries::model_weights::update_weight::{ update_weights, Upda use tokens::tokens::model_weights::ModelWeightToken; use crate::server_state::ServerState; -use utopia::ToSchema; +use utoipa::ToSchema; /// TODO will eventually be polymorphic #[derive(Deserialize, ToSchema)] pub struct UpdateWeightRequest { @@ -34,7 +34,7 @@ pub struct UpdateWeightResponse { } /// For the URL PathInfo -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct UpdateWeightPathInfo { weight_token: String, } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs new file mode 100644 index 0000000000..2d66aa7684 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs @@ -0,0 +1,271 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use once_cell::sync::Lazy; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{error, info, warn}; + +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; + +pub async fn upload_weights_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, + ) -> Result { + + let response = handle_weight_upload( + &http_request, + &server_state, + multipart_payload, + &ALLOWED_MIME_TYPES).await?; + + // send a body with the required payload along side the multipart upload + + let weight_upload_token = response.to_model_token(); + + let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { + success: true, + upload_token: weight_upload_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); + } + + pub enum SuccessCase { + WeightAlreadyUploaded { + existing_upload_token: ModelWeightToken, + }, + WeightSuccessfullyUploaded { + upload_token: ModelWeightToken, + } + } + + impl SuccessCase { + pub fn to_model_token(self) -> ModelWeightToken { + match self { + SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, + SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, + } + } + } + + pub async fn handle_weight_upload( + http_request: &HttpRequest, + server_state: &web::Data>, + mut multipart_payload: Multipart, + allowed_mimetypes: &HashSet<&'static str>, + ) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + UploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + UploadError::ServerError + })?; + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { + return Err(UploadError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_weights_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + UploadError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token + .ok_or(UploadError::BadInput("no uuid".to_string()))?; + + let maybe_existing_upload = + get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + .await; + + match maybe_existing_upload { + Err(err) => { + error!("Error checking for previous upload: {:?}", err); + return Err(UploadError::ServerError); + } + Ok(Some(upload)) => { + // File already uploaded and frontend didn't protect us. + return Ok(SuccessCase::WeightAlreadyUploaded { + existing_upload_token: upload.token, + }); + } + Ok(None) => { + // Proceed. + } + } + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(UploadError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); + + let maybe_mimetype = upload_weights_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + + let bytes = match upload_weights_request.file_bytes { + None => return Err(UploadError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + UploadError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_upload_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !allowed_mimetypes.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + media_upload_type = match mimetype { + "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), + _ => None, + }; + } + + // This is fine for now we just don't expose the path! + let public_upload_path = WeightUploadOriginalFilePath::generate_new(); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + // need to upload based on visibility + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + UploadError::ServerError + })?; + + let token = create_weight(CreateModelWeightsArgs { + token: &ModelWeightToken::generate(), + weights_type: WeightsType::LoRA, // to do parse multi part for this + weights_category: WeightsCategory::ImageGeneration, + title: upload_weights_request.file_name.as_deref(), + maybe_thumbnail_token: &ThumbnailToken::generate(), + description_markdown: "", + description_rendered_html: "", + creator_user_token: maybe_user_token.as_str(), + creator_ip_address: &ip_address, + creator_set_visibility, + maybe_last_update_user_token: "", + original_download_url: "", + original_filename: upload_weights_request.file_name.as_deref(), + file_size_bytes: file_size_bytes as i32, + file_checksum_sha2: hash, + private_bucket_hash: private_bucket_hash, + maybe_private_bucket_prefix: None, + maybe_private_bucket_extension: None, + cached_user_ratings_total_count: 0, + cached_user_ratings_positive_count: 0, + cached_user_ratings_negative_count: 0, + maybe_cached_user_ratings_ratio: None, + version: 0, + mysql_pool: &server_state.mysql_pool, + }).await + .map_err(|e| { + error!("Error creating weights upload: {:?}", e); + UploadError::ServerError + })?; + + info!("new weights upload id: {} token: {:?}", record_id, &token); + + Ok(SuccessCase::WeightSuccessfullyUploaded { + upload_token: token, + }) + } + \ No newline at end of file From af6370eea1e490def21a783e29025774d0add58d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 19:53:52 -0500 Subject: [PATCH 082/136] Issue with the api doc not sure. --- .../web/storyteller_web/src/api_doc.rs | 8 +- .../endpoints/weights/create_weight.rs | 2 - .../endpoints/weights/get_weight.rs | 1 - .../weights/list_availible_weights.rs | 278 ------------------ .../src/http_server/endpoints/weights/mod.rs | 6 +- .../service/web/storyteller_web/src/routes.rs | 6 +- 6 files changed, 10 insertions(+), 291 deletions(-) delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 9db38a306a..40c351e55c 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,7 +1,7 @@ use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; -use crate::http_server::endpoints::weights::list_availible_weights::*; +use crate::http_server::endpoints::weights::list_available_weights::*; use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; use crate::http_server::endpoints::weights::get_weight::*; @@ -20,7 +20,7 @@ use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, - crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler, + crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler, crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, @@ -36,8 +36,8 @@ use utoipa::OpenApi; GetWeightPathInfo,GetWeightResponse,GetWeightError, UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError, DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, - ListWeightError,ListWeightsByPathInfo,ModelWeightForList,ListWeightError, - ListAvailibleWeightsQuery,ListAvailibleWeightsSuccessResponse,ListWeightsByPathInfo,ModelWeightForList, + ListWeightError,ModelWeightForList,ListWeightError, + ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs index 5e502e6a57..1bd1a0e50d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -48,7 +48,6 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { // mut multipart_payload: Multipart, // ) -> Result { - // let weight_upload_token = response.to_model_token(); @@ -83,7 +82,6 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { // } // } - // let token = create_weight(CreateModelWeightsArgs { // token: &ModelWeightToken::generate(), // weights_type: WeightsType::LoRA, // to do parse multi part for this diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 1c0857b5b7..6eb2f9849f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -83,7 +83,6 @@ impl ResponseError for GetWeightError { (status = 500, description = "Server error", body = GetWeightError), ), params( - ("request" = UpdateWeightRequest, description = "Payload for Request"), ("path" = GetWeightPathInfo, description = "Path for Request") ) )] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs deleted file mode 100644 index 5ababcdff9..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_availible_weights.rs +++ /dev/null @@ -1,278 +0,0 @@ -use std::sync::Arc; - -use actix_web::{ HttpRequest, HttpResponse, web }; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; -use chrono::{ DateTime, Utc }; -use log::{ info, warn }; -use rand::Rng; -use enums::common::visibility::Visibility; -use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; - -use enums::by_table::model_weights::{ - weights_types::WeightsType, - weights_category::WeightsCategory, -}; - -use tokens::tokens::users::UserToken; -use tokens::tokens::model_weights::ModelWeightToken; - -use crate::http_server::common_responses::user_details_lite::UserDetailsLight; -use crate::server_state::ServerState; - -use utoipa::ToSchema; - -#[derive(Deserialize,ToSchema)] -pub struct ListAvailibleWeightsQuery { - pub sort_ascending: Option, - pub page_size: u16, - pub page_index : u16, -} - -#[derive(Serialize,ToSchema)] -pub struct ListAvailibleWeightsSuccessResponse { - pub success: bool, - pub weights: Vec, - pub page_size: u16, - pub page_index : u16, - pub cursor_next: Option, - pub cursor_previous: Option, -} - -#[derive(Deserialize,ToSchema)] -pub struct ListWeightsByPathInfo { - pub username: Option, - pub weights_type: Option, - pub weights_category: Option, -} - -#[derive(Serialize,ToSchema)] -pub struct ModelWeightForList { - pub weight_token: ModelWeightToken, - - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, - - pub title: String, - - pub maybe_thumbnail_token: Option, - - pub description_markdown: String, - pub description_rendered_html: String, - - pub creator_user_token: UserToken, - pub creator_set_visibility: Visibility, - - pub file_size_bytes: i32, - pub file_checksum_sha2: String, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - - pub version: i32, - - pub created_at: DateTime, - pub updated_at: DateTime, - - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, - - // additional fields to be added when tables are around - pub likes: u32, - pub bookmarks: bool, - -} - -#[derive(Debug,ToSchema)] -pub enum ListWeightError { - NotAuthorized, - ServerError, -} - -impl std::fmt::Display for ListWeightError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -impl ResponseError for ListWeightError { - fn status_code(&self) -> StatusCode { - match *self { - ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, - ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - } - } -} - -#[utoipa::path( - get, - path = "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", - responses( - (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), - (status = 401, description = "Not authorized", body = ListWeightError), - (status = 500, description = "Server error", body = ListWeightError), - ), - params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), - ("path" = ListWeightsByPathInfo, description = "Path for Request") - ) -)] -pub async fn list_availible_weights_handler( - http_request: HttpRequest, - path: web::Path, - query: web::Query, - server_state: web::Data> -) -> Result { - let maybe_user_session = server_state.session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool).await - .map_err(|e| { - warn!("Session checker error: {:?}", e); - ListWeightError::ServerError - })?; - - let mut is_mod = false; - let user_session = match maybe_user_session { - Some(session) => { - is_mod = session.can_ban_users; - session - } - None => { - info!("not logged in"); - return Err(ListWeightError::NotAuthorized); - } - }; - - let limit = query.page_size; - - let sort_ascending = query.sort_ascending.unwrap_or(false); - - let include_user_hidden = is_mod; - - let mut weights_category: Option = None; - let mut weights_type: Option = None; - - if let Some(weights_category_string) = path.weights_category.as_ref() { - let result = WeightsCategory::from_str(&weights_category_string); - match result { - Ok(category) => { - weights_category = Some(category) - }, - Err(e) => { - warn!("invalid weights_category: {:?}", e); - weights_category = None - } - } - } - - if let Some(weights_type_string) = path.weights_type.as_ref() { - let result = WeightsType::from_str(&weights_type_string); - match result { - Ok(wtype) => { - weights_type = Some(wtype) - }, - Err(e) => { - warn!("invalid weights_type: {:?}", e); - weights_type = None - } - } - } - - let mut query_builder = ListWeightsQueryBuilder::new() - .sort_ascending(sort_ascending) - .weights_category(weights_category) - .weights_type(weights_type) - .scope_creator_username(None) - .include_user_hidden(include_user_hidden) - .include_user_deleted_results(is_mod) - .include_mod_deleted_results(is_mod) - .limit(limit) - .offset(Some((query.page_index as u64) * (query.page_size as u64))); - - let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; - - let weights_page = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListWeightError::ServerError); - } - }; - - let cursor_next = if let Some(id) = weights_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let cursor_previous = if let Some(id) = weights_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListWeightError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let mut rng = rand::thread_rng(); - let random_float = rng.gen_range(0.0..1.0); - let random_bool = random_float >= 0.5; - - // generate parse a response - let response = ListAvailibleWeightsSuccessResponse { - success: true, - weights: weights_page.weights.into_iter() - .map(|weights| ModelWeightForList { - weight_token: weights.token, - title: weights.title, - weights_type: weights.weights_type, - weights_category: weights.weights_category, - - maybe_thumbnail_token:weights.maybe_thumbnail_token, - description_markdown: weights.description_markdown, - description_rendered_html: weights.description_rendered_html, - - creator_user_token: weights.creator_user_token, - creator_set_visibility: weights.creator_set_visibility, - - file_size_bytes:weights.file_size_bytes, - file_checksum_sha2: weights.file_checksum_sha2, - - cached_user_ratings_total_count: weights.cached_user_ratings_total_count, - cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, - cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, - maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, - - version: weights.version, - - created_at: weights.created_at, - updated_at: weights.updated_at, - - creator_username: weights.creator_username, - creator_display_name: weights.creator_display_name, - creator_email_gravatar_hash: weights.creator_email_gravatar_hash, - - // TODO FIX THIS when we align. - bookmarks: random_bool, - likes: rng.gen_range(0..1000), - }).collect::>(), - cursor_next, - cursor_previous, - page_index: query.page_index, - page_size: query.page_size - }; - - let body = serde_json::to_string(&response) - .map_err(|e| ListWeightError::ServerError)?; - - Ok(HttpResponse::Ok().content_type("application/json").body(body)) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index f457767f25..3fbd1cb09d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -2,6 +2,6 @@ pub mod get_weight; pub mod delete_weight; pub mod update_weight; pub mod list_weights_by_user; -pub mod list_availible_weights; -// pub mod create_weight; -// pub mod drain_multipart_weights_request; \ No newline at end of file +pub mod list_available_weights; +// pub mod create_weight; this is a private api call internally +// pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 4bf7d273ac..19dc95b7e9 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -155,7 +155,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; -use crate::http_server::endpoints::weights::list_availible_weights::list_availible_weights_handler; +use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; @@ -1266,8 +1266,8 @@ fn add_weights_routes(app: App) -> App ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) .route( - "/by_user_category_and_type/{username}/{weights_category}/{weights_type}", - web::get().to(list_availible_weights_handler) + "/list", + web::get().to(list_available_weights_handler) ) ) } From 330f24413ddcc8c0901385182f748ca891e18d07 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 19:54:21 -0500 Subject: [PATCH 083/136] added missing file --- .../weights/list_available_weights.rs | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs new file mode 100644 index 0000000000..a0c29bc14f --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -0,0 +1,273 @@ +use std::sync::Arc; +use actix_web::web::Query; +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{ DateTime, Utc }; +use log::{ info, warn }; +use rand::Rng; +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; + +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::users::UserToken; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +use utoipa::ToSchema; + + +#[derive(Deserialize,ToSchema)] +pub struct ListAvailableWeightsQuery { + pub sort_ascending: Option, + pub page_size: u16, + pub page_index : u16, + pub username: Option, + pub weights_type: Option, + pub weights_category: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ListAvailableWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, + pub page_size: u16, + pub page_index : u16, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub creator_user_token: UserToken, + pub creator_set_visibility: Visibility, + + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + + pub version: i32, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + // additional fields to be added when tables are around + pub likes: u32, + pub bookmarks: bool, + +} + +#[derive(Debug,ToSchema)] +pub enum ListWeightError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListWeightError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/list/", + responses( + (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightError), + (status = 500, description = "Server error", body = ListWeightError), + ), + params( + ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), + ) +)] +pub async fn list_available_weights_handler( + http_request: HttpRequest, + query: web::Query, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightError::ServerError + })?; + + let mut is_mod = false; + let user_session = match maybe_user_session { + Some(session) => { + is_mod = session.can_ban_users; + session + } + None => { + info!("not logged in"); + return Err(ListWeightError::NotAuthorized); + } + }; + + let limit = query.page_size; + + let sort_ascending = query.sort_ascending.unwrap_or(false); + + let include_user_hidden = is_mod; + + let mut weights_category: Option = None; + let mut weights_type: Option = None; + + if let Some(weights_category_string) = query.weights_category.as_ref() { + let result = WeightsCategory::from_str(&weights_category_string); + match result { + Ok(category) => { + weights_category = Some(category) + }, + Err(e) => { + warn!("invalid weights_category: {:?}", e); + weights_category = None + } + } + } + + if let Some(weights_type_string) = query.weights_type.as_ref() { + let result = WeightsType::from_str(&weights_type_string); + match result { + Ok(wtype) => { + weights_type = Some(wtype) + }, + Err(e) => { + warn!("invalid weights_type: {:?}", e); + weights_type = None + } + } + } + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .weights_category(weights_category) + .weights_type(weights_type) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .offset(Some((query.page_index as u64) * (query.page_size as u64))); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; + + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let mut rng = rand::thread_rng(); + let random_float = rng.gen_range(0.0..1.0); + let random_bool = random_float >= 0.5; + + // generate parse a response + let response = ListAvailableWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, + + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, + + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, + + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, + + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, + + version: weights.version, + + created_at: weights.created_at, + updated_at: weights.updated_at, + + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + + // TODO: FIX THIS when we align again. + bookmarks: random_bool, + likes: rng.gen_range(0..1000), + }).collect::>(), + cursor_next, + cursor_previous, + page_index: query.page_index, + page_size: query.page_size + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} From 43b6808f23cf58c08261fea8c9a6352efb05ff6b Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 20:05:23 -0500 Subject: [PATCH 084/136] Ok fixed docs and avaoilable weights endpoint --- .../http_server/endpoints/weights/list_available_weights.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index a0c29bc14f..34962dbd5a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -108,12 +108,12 @@ impl ResponseError for ListWeightError { get, path = "/list/", responses( - (status = 200, description = "List Weights", body = ListAvailibleWeightsSuccessResponse), + (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), (status = 500, description = "Server error", body = ListWeightError), ), params( - ("request" = ListAvailibleWeightsQuery, description = "Payload for Request"), + ("request" = ListAvailableWeightsQuery, description = "Payload for Request"), ) )] pub async fn list_available_weights_handler( From f4176bf84a8f8d4d8a552afaeaddb240312936dc Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 21:31:37 -0500 Subject: [PATCH 085/136] Alright should fix the generation now... --- .../src/http_server/endpoints/weights/delete_weight.rs | 2 +- .../src/http_server/endpoints/weights/get_weight.rs | 2 +- .../src/http_server/endpoints/weights/list_available_weights.rs | 2 +- .../src/http_server/endpoints/weights/update_weight.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs index 7806fa8583..33f9ac75d7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -76,7 +76,7 @@ impl fmt::Display for DeleteWeightError { #[utoipa::path( delete, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), (status = 400, description = "Bad input", body = DeleteWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 6eb2f9849f..02be96fef3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -75,7 +75,7 @@ impl ResponseError for GetWeightError { } #[utoipa::path( get, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Update", body = GetWeightResponse), (status = 400, description = "Bad input", body = GetWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index 34962dbd5a..f9e8d7f367 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -106,7 +106,7 @@ impl ResponseError for ListWeightError { #[utoipa::path( get, - path = "/list/", + path = "/v1/weights/list/", responses( (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index 8ddddca33e..e091c30322 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -74,7 +74,7 @@ impl fmt::Display for UpdateWeightError { // =============== Handler =============== #[utoipa::path( post, - path = "/weight/{weight_token}", + path = "/v1/weights/weight/{weight_token}", responses( (status = 200, description = "Success Update", body = SimpleGenericJsonSuccess), (status = 400, description = "Bad input", body = UpdateWeightError), From f327dfcdbb372940d5796c8e6a98cab8b00ddbc2 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 23:27:34 -0500 Subject: [PATCH 086/136] Builds and docs work --- .../web/storyteller_web/src/api_doc.rs | 3 +++ .../weights/list_available_weights.rs | 4 +++- .../endpoints/weights/list_weights_by_user.rs | 21 +++++++++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 40c351e55c..178e8871ae 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -18,12 +18,14 @@ use enums::common::visibility::Visibility; use chrono::{DateTime, Utc}; use utoipa::OpenApi; + #[derive(OpenApi)] #[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler, crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, crate::http_server::endpoints::weights::update_weight::update_weight_handler, crate::http_server::endpoints::weights::get_weight::get_weight_handler, + crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler ),components(schemas( // Tokens UserToken,Visibility, @@ -38,6 +40,7 @@ use utoipa::OpenApi; DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, ListWeightError,ModelWeightForList,ListWeightError, ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, + ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index f9e8d7f367..c41e38de99 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -106,7 +106,7 @@ impl ResponseError for ListWeightError { #[utoipa::path( get, - path = "/v1/weights/list/", + path = "/v1/weights/list", responses( (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), (status = 401, description = "Not authorized", body = ListWeightError), @@ -139,6 +139,8 @@ pub async fn list_available_weights_handler( return Err(ListWeightError::NotAuthorized); } }; + let page_size: u16 = 10; + let page_index : u16 = 0; let limit = query.page_size; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs index c478ea68d9..364a14fdde 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -14,8 +14,9 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; use crate::server_state::ServerState; +use utoipa::ToSchema; -#[derive(Serialize, Clone)] +#[derive(Serialize, Clone,ToSchema)] pub struct Weight { weight_token: ModelWeightToken, title: String, @@ -41,18 +42,18 @@ pub struct Weight { } -#[derive(Serialize)] +#[derive(Serialize,ToSchema)] pub struct ListWeightsByUserSuccessResponse { pub success: bool, pub weights: Vec, } -#[derive(Deserialize)] +#[derive(Deserialize,ToSchema)] pub struct ListWeightsByUserPathInfo { username: String, } -#[derive(Debug)] +#[derive(Debug,ToSchema)] pub enum ListWeightsByUserError { NotAuthorized, ServerError, @@ -73,6 +74,18 @@ impl ResponseError for ListWeightsByUserError { } } +#[utoipa::path( + get, + path = "/v1/weights/by_user/{username}", + responses( + (status = 200, description = "List Weights by user", body = ListWeightsByUserSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightsByUserError), + (status = 500, description = "Server error", body = ListWeightsByUserError), + ), + params( + ("path" = ListWeightsByUserPathInfo, description = "Payload for Request"), + ) +)] pub async fn list_weights_by_user_handler( http_request: HttpRequest, path: Path, From 7976f68fd027d953e147abd9d15f61112803314a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 6 Dec 2023 02:53:50 -0500 Subject: [PATCH 087/136] Save TTS results as media files --- .../inference_category.rs | 3 +- .../generic_synthetic_ids/id_category.rs | 14 +- .../media_file_origin_model_type.rs | 13 +- .../media_file_origin_product_category.rs | 13 +- .../enums/src/common/job_status_plus.rs | 11 + .../src/tokens/anonymous_visitor_tracking.rs | 2 +- .../src/tokens/generic_inference_jobs.rs | 2 +- crates/_types/tokens/src/tokens/users.rs | 2 +- .../web/get_inference_job_status.rs | 75 ++++++- .../insert_media_file_from_tacotron2.rs | 166 ++++++++++++++ .../src/queries/media_files/mod.rs | 1 + .../tts/tacotron2_v2_early_fakeyou/mod.rs | 1 + .../tacotron2_v2_early_fakeyou/process_job.rs | 90 ++------ .../tacotron2_dependencies.rs | 5 +- .../upload_results.rs | 202 ++++++++++++++++++ .../vall_e_x/process_job_inference_voice.rs | 2 +- crates/service/web/storyteller_web/Cargo.toml | 4 +- .../inference_job/get_inference_job_status.rs | 128 +++++++++-- .../tts/get_tts_inference_job_status.rs | 1 + 19 files changed, 627 insertions(+), 108 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_tacotron2.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index faee8ff909..b38f181460 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -13,10 +13,11 @@ use strum::EnumIter; /// /// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)] pub enum InferenceCategory { /// Eg. SadTalker and possibly Wav2Lip #[serde(rename = "lipsync_animation")] + #[default] LipsyncAnimation, #[serde(rename = "text_to_speech")] diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index ca1b502e14..eccff8cf4d 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,6 +21,11 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimationResult, + /// Results from tacotron2 + /// Applies for RVC and SVC + #[serde(rename = "tts_result")] + TtsResult, + /// Results from voice conversion (which may live in the media_files table) /// Applies for RVC and SVC #[serde(rename = "voice_conversion")] @@ -52,6 +57,7 @@ impl IdCategory { match self { Self::MediaFile => "media_file", Self::LipsyncAnimationResult => "lipsync_animation", + Self::TtsResult => "tts_result", Self::VoiceConversionResult => "voice_conversion", Self::ZeroShotVoiceDataset => "zs_dataset", Self::ZeroShotVoiceEmbedding => "zs_voice", @@ -64,6 +70,7 @@ impl IdCategory { match value { "media_file" => Ok(Self::MediaFile), "lipsync_animation" => Ok(Self::LipsyncAnimationResult), + "tts_result" => Ok(Self::TtsResult), "voice_conversion" => Ok(Self::VoiceConversionResult), "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), @@ -79,6 +86,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimationResult, + Self::TtsResult, Self::VoiceConversionResult, Self::ZeroShotTtsResult, Self::ZeroShotVoiceDataset, @@ -100,6 +108,7 @@ mod tests { fn test_serialization() { assert_serialization(IdCategory::MediaFile, "media_file"); assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation"); + assert_serialization(IdCategory::TtsResult, "tts_result"); assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion"); assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); assert_serialization(IdCategory::ZeroShotVoiceEmbedding, "zs_voice"); @@ -111,6 +120,7 @@ mod tests { fn to_str() { assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation"); + assert_eq!(IdCategory::TtsResult.to_str(), "tts_result"); assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion"); assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset"); assert_eq!(IdCategory::ZeroShotVoiceEmbedding.to_str(), "zs_voice"); @@ -122,6 +132,7 @@ mod tests { fn from_str() { assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult); + assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult); assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult); assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset); assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZeroShotVoiceEmbedding); @@ -133,9 +144,10 @@ mod tests { fn all_variants() { // Static check let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 7); + assert_eq!(variants.len(), 8); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); + assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceDataset)); diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index b38f90a6dc..8cb9685bc7 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -25,6 +25,9 @@ pub enum MediaFileOriginModelType { #[serde(rename = "so_vits_svc")] SoVitsSvc, + #[serde(rename = "tacotron2")] + Tacotron2, + #[serde(rename = "vall_e_x")] VallEX, } @@ -41,6 +44,7 @@ impl MediaFileOriginModelType { Self::RvcV2 => "rvc_v2", Self::SadTalker => "sad_talker", Self::SoVitsSvc => "so_vits_svc", + Self::Tacotron2 => "tacotron2", Self::VallEX => "vall_e_x" } } @@ -50,6 +54,7 @@ impl MediaFileOriginModelType { "rvc_v2" => Ok(Self::RvcV2), "sad_talker" => Ok(Self::SadTalker), "so_vits_svc" => Ok(Self::SoVitsSvc), + "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), _ => Err(format!("invalid value: {:?}", value)), } @@ -62,6 +67,7 @@ impl MediaFileOriginModelType { Self::RvcV2, Self::SadTalker, Self::SoVitsSvc, + Self::Tacotron2, Self::VallEX ]) } @@ -85,6 +91,7 @@ mod tests { assert_eq!(MediaFileOriginModelType::RvcV2.to_str(), "rvc_v2"); assert_eq!(MediaFileOriginModelType::SadTalker.to_str(), "sad_talker"); assert_eq!(MediaFileOriginModelType::SoVitsSvc.to_str(), "so_vits_svc"); + assert_eq!(MediaFileOriginModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(MediaFileOriginModelType::VallEX.to_str(), "vall_e_x"); } @@ -93,17 +100,19 @@ mod tests { assert_eq!(MediaFileOriginModelType::from_str("rvc_v2").unwrap(), MediaFileOriginModelType::RvcV2); assert_eq!(MediaFileOriginModelType::from_str("sad_talker").unwrap(), MediaFileOriginModelType::SadTalker); assert_eq!(MediaFileOriginModelType::from_str("so_vits_svc").unwrap(), MediaFileOriginModelType::SoVitsSvc); + assert_eq!(MediaFileOriginModelType::from_str("tacotron2").unwrap(), MediaFileOriginModelType::Tacotron2); assert_eq!(MediaFileOriginModelType::from_str("vall_e_x").unwrap(), MediaFileOriginModelType::VallEX); - assert!(MediaFileOriginModelType::from_str("foo").is_err()); + assert!(MediaFileOriginModelType::from_str("foo").is_err()); } #[test] fn all_variants() { let mut variants = MediaFileOriginModelType::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::VallEX)); assert_eq!(variants.pop_first(), None); } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 2d27b3824d..7b6f2a029a 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -21,6 +21,10 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "face_animator")] FaceAnimator, + /// Text to speech (Tacotron2, not voice designer / VallE-X) + #[serde(rename = "tts")] + TextToSpeech, + // TODO: This should be a temporary category until we migrate the DB to remove this default value /// Unknown which product is attached to the file (generated the file, the file was /// uploaded on behalf of, etc.) @@ -46,6 +50,7 @@ impl MediaFileOriginProductCategory { pub fn to_str(&self) -> &'static str { match self { Self::FaceAnimator => "face_animator", + Self::TextToSpeech => "tts", Self::Unknown => "unknown", Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", @@ -55,6 +60,7 @@ impl MediaFileOriginProductCategory { pub fn from_str(value: &str) -> Result { match value { "face_animator" => Ok(Self::FaceAnimator), + "tts" => Ok(Self::TextToSpeech), "unknown" => Ok(Self::Unknown), "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), @@ -67,6 +73,7 @@ impl MediaFileOriginProductCategory { // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ Self::FaceAnimator, + Self::TextToSpeech, Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, @@ -85,6 +92,7 @@ mod tests { #[test] fn test_serialization() { assert_serialization(MediaFileOriginProductCategory::FaceAnimator, "face_animator"); + assert_serialization(MediaFileOriginProductCategory::TextToSpeech, "tts"); assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); @@ -93,6 +101,7 @@ mod tests { #[test] fn to_str() { assert_eq!(MediaFileOriginProductCategory::FaceAnimator.to_str(), "face_animator"); + assert_eq!(MediaFileOriginProductCategory::TextToSpeech.to_str(), "tts"); assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); @@ -101,6 +110,7 @@ mod tests { #[test] fn from_str() { assert_eq!(MediaFileOriginProductCategory::from_str("face_animator").unwrap(), MediaFileOriginProductCategory::FaceAnimator); + assert_eq!(MediaFileOriginProductCategory::from_str("tts").unwrap(), MediaFileOriginProductCategory::TextToSpeech); assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); @@ -109,8 +119,9 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); diff --git a/crates/_types/enums/src/common/job_status_plus.rs b/crates/_types/enums/src/common/job_status_plus.rs index 4f69359e73..be0ae8db25 100644 --- a/crates/_types/enums/src/common/job_status_plus.rs +++ b/crates/_types/enums/src/common/job_status_plus.rs @@ -34,6 +34,12 @@ impl_mysql_enum_coders!(JobStatusPlus); // NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders` +impl Default for JobStatusPlus { + fn default() -> Self { + Self::Pending + } +} + impl JobStatusPlus { pub fn to_str(&self) -> &'static str { match self { @@ -86,6 +92,11 @@ mod tests { mod explicit_checks { use super::*; + #[test] + fn test_default() { + assert_eq!(JobStatusPlus::default(), JobStatusPlus::Pending); + } + #[test] fn test_serialization() { assert_serialization(JobStatusPlus::Pending, "pending"); diff --git a/crates/_types/tokens/src/tokens/anonymous_visitor_tracking.rs b/crates/_types/tokens/src/tokens/anonymous_visitor_tracking.rs index 048d0f9a93..dbb1659ff9 100644 --- a/crates/_types/tokens/src/tokens/anonymous_visitor_tracking.rs +++ b/crates/_types/tokens/src/tokens/anonymous_visitor_tracking.rs @@ -8,7 +8,7 @@ use crate::prefixes::TokenPrefix; /// A rare instance that is *not* a primary key. /// These are generated as cookies to track users for a better logged out experience and for analytics. /// We use these as indices into several columns (ML result types, uploads, etc.) -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, Default)] #[sqlx(transparent)] pub struct AnonymousVisitorTrackingToken(pub String); diff --git a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs index 48c3a78260..ddc5ca5c6a 100644 --- a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs +++ b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs @@ -7,7 +7,7 @@ use crate::prefixes::TokenPrefix; use utoipa::ToSchema; /// The primary key for "generic" inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize, Default, ToSchema)] #[sqlx(transparent)] pub struct InferenceJobToken(String); diff --git a/crates/_types/tokens/src/tokens/users.rs b/crates/_types/tokens/src/tokens/users.rs index 2e28fca9fe..82438cc81c 100644 --- a/crates/_types/tokens/src/tokens/users.rs +++ b/crates/_types/tokens/src/tokens/users.rs @@ -6,7 +6,7 @@ use serde::Serialize; use crate::prefixes::TokenPrefix; /// The primary key for users. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize, Default)] #[sqlx(transparent)] pub struct UserToken(pub String); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 615ef68ca9..a59fd2e5d2 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -14,7 +14,7 @@ use tokens::tokens::users::UserToken; use crate::helpers::boolean_converters::i8_to_bool; -#[derive(Debug)] +#[derive(Debug, Default)] pub struct GenericInferenceJobStatus { pub job_token: InferenceJobToken, @@ -40,7 +40,7 @@ pub struct GenericInferenceJobStatus { /// Details about the user's original inference request /// (We may want to present it in the "pending" UI.) -#[derive(Debug)] +#[derive(Debug, Default)] pub struct RequestDetails { pub inference_category: InferenceCategory, pub maybe_model_type: Option, // TODO: Strongly type @@ -52,7 +52,7 @@ pub struct RequestDetails { } /// Details about the generated result -#[derive(Debug)] +#[derive(Debug, Default)] pub struct ResultDetails { pub entity_type: String, pub entity_token: String, @@ -71,7 +71,7 @@ pub struct ResultDetails { /// NB: DO NOT EXPOSE TO FRONTEND. /// This is used to gate access to job termination -#[derive(Debug)] +#[derive(Debug, Default)] pub struct UserDetails { pub maybe_creator_user_token: Option, pub maybe_creator_anonymous_visitor_token: Option, @@ -168,6 +168,11 @@ WHERE jobs.token = ? } }; + Ok(Some(raw_record_to_public_result(record))) +} + +/// Map the internal record type (since we query over several result tables) +fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericInferenceJobStatus { let maybe_model_title = match record.inference_category { InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), @@ -213,7 +218,7 @@ WHERE jobs.token = ? }) }); - Ok(Some(GenericInferenceJobStatus { + GenericInferenceJobStatus { job_token: record.job_token, status: record.status, attempt_count: record.attempt_count, @@ -237,9 +242,10 @@ WHERE jobs.token = ? is_keepalive_required: i8_to_bool(record.is_keepalive_required), created_at: record.created_at, updated_at: record.updated_at, - })) + } } +#[derive(Debug, Default)] struct RawGenericInferenceJobStatus { pub job_token: InferenceJobToken, @@ -281,3 +287,60 @@ struct RawGenericInferenceJobStatus { pub maybe_successfully_completed_at: Option>, } +#[cfg(test)] +mod tests { + use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; + use crate::queries::generic_inference::web::get_inference_job_status::{raw_record_to_public_result, RawGenericInferenceJobStatus}; + + #[test] + fn text_to_speech_as_media_file() { + // NB(bt,2023-12-06): After tts migration to media files, these are the results the job system will return + let record = RawGenericInferenceJobStatus { + inference_category: InferenceCategory::TextToSpeech, + maybe_result_entity_type: Some("media_file".to_string()), // NB: Media file record + maybe_result_entity_token: Some("m_00af018cy75pxytpb8wbdx9jqgtp0p".to_string()), + maybe_media_file_public_bucket_directory_hash: Some("3vb91yq71z5zne56saazn4qntt52yter".to_string()), // NB: Media file + maybe_media_file_public_bucket_prefix: Some("fakeyou_".to_string()), + maybe_media_file_public_bucket_extension: Some(".wav".to_string()), + maybe_tts_public_bucket_path: None, // NB: Not a tts_result! + ..Default::default() + }; + + let record = raw_record_to_public_result(record); + + assert!(record.maybe_result_details.is_some()); + + let result_details = record.maybe_result_details.expect("should have result details"); + + assert_eq!(&result_details.entity_type, "media_file"); + assert_eq!(&result_details.entity_token, "m_00af018cy75pxytpb8wbdx9jqgtp0p"); + assert_eq!(&result_details.public_bucket_location_or_hash, "3vb91yq71z5zne56saazn4qntt52yter"); + assert_eq!(result_details.public_bucket_location_is_hash, true); + } + + #[test] + fn text_to_speech_as_tts_result() { + // NB(bt,2023-12-06): Prior to tts migration to media files, these are the results the job system will return + let record = RawGenericInferenceJobStatus { + inference_category: InferenceCategory::TextToSpeech, + maybe_result_entity_type: Some("text_to_speech".to_string()), // NB: TTS record + maybe_result_entity_token: Some("TR:wefhbk4j3yc8d6zwembedbdw4ad4s".to_string()), + maybe_tts_public_bucket_path: Some("/tts_inference_output/c/5/8/vocodes_c58aeaef-84df-4478-9e7f-c64280a852e8.wav".to_string()), // NB: tts_result! + maybe_media_file_public_bucket_directory_hash: None, // NB: Not a media file! + maybe_media_file_public_bucket_prefix: None, + maybe_media_file_public_bucket_extension: None, + ..Default::default() + }; + + let record = raw_record_to_public_result(record); + + assert!(record.maybe_result_details.is_some()); + + let result_details = record.maybe_result_details.expect("should have result details"); + + assert_eq!(&result_details.entity_type, "text_to_speech"); + assert_eq!(&result_details.entity_token, "TR:wefhbk4j3yc8d6zwembedbdw4ad4s"); + assert_eq!(&result_details.public_bucket_location_or_hash, "/tts_inference_output/c/5/8/vocodes_c58aeaef-84df-4478-9e7f-c64280a852e8.wav"); + assert_eq!(result_details.public_bucket_location_is_hash, false); + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_tacotron2.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_tacotron2.rs new file mode 100644 index 0000000000..bc3010c1f8 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_tacotron2.rs @@ -0,0 +1,166 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub struct InsertMediaFileArgs<'a> { + pub pool: &'a MySqlPool, + pub job: &'a AvailableInferenceJob, + + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub duration_millis: u64, + pub sha256_checksum: &'a str, + + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, + + pub is_on_prem: bool, + pub worker_hostname: &'a str, + pub worker_cluster: &'a str, +} + +pub async fn insert_media_file_from_tacotron2( + args: InsertMediaFileArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let result_token = MediaFileToken::generate(); + + let origin_model_type = MediaFileOriginModelType::Tacotron2; + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() { + let user_token = UserToken::new_from_str(creator_user_token); + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_voice_conversion_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::TtsResult, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_voice_conversion_id); + } + + let vc_model_token = args.job.maybe_model_token.as_deref(); + let creator_ip_address = args.job.creator_ip_address.as_str(); + let creator_set_visibility = args.job.creator_set_visibility.clone(); + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::TextToSpeech; + const MEDIA_TYPE : MediaFileType = MediaFileType::Audio; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + + maybe_origin_model_type = ?, + maybe_origin_model_token = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ?, + + is_generated_on_prem = ?, + generated_by_worker = ?, + generated_by_cluster = ? + + "#, + result_token.as_str(), + + ORIGIN_CATEGORY.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + + origin_model_type.to_str(), + args.job.maybe_model_token, + + MEDIA_TYPE.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.job.maybe_creator_user_token, + args.job.maybe_creator_anonymous_visitor_token, + args.job.creator_ip_address, + + args.job.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + + args.is_on_prem, + args.worker_hostname, + args.worker_cluster + ) + .execute(&mut *transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((result_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 0590f82159..8930a3ae7c 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -2,6 +2,7 @@ pub mod delete_media_file; pub mod get_media_file; pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_tacotron2; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; pub mod list_media_files; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs index a895808b59..c6d81ab3cf 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs @@ -8,5 +8,6 @@ pub mod tacotron2_dependencies; pub mod tacotron2_inference_command; pub mod tacotron2_inference_sidecar_client; pub mod tacotron2_sidecar_health_check_client; +pub mod upload_results; pub mod virtual_lfu_cache; pub mod vocoder_option; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index c6559a478a..0a3460beb4 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -27,6 +27,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::download_static_depe use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::health_check_trap::maybe_block_on_sidecar_health_check; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_steps::seconds_to_decoder_steps; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor}; +use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::upload_results::{upload_results, UploadResultArgs}; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption; use crate::job_dependencies::JobDependencies; use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; @@ -305,78 +306,27 @@ async fn process_job_with_cleanup( safe_delete_temp_file(&output_metadata_fs_path); - // ==================== UPLOAD AUDIO TO BUCKET ==================== // + // ==================== UPLOAD FILES AND SAVE RECORD ==================== // - job_progress_reporter.log_status("uploading result") - .map_err(|e| ProcessSingleJobError::Other(e))?; - - let audio_result_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_inference_wav_audio_output_path( - &job.uuid_idempotency_token); // TODO: Don't use this! - - info!("Audio destination bucket path: {:?}", &audio_result_object_path); - - info!("Uploading audio..."); - - args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( - &audio_result_object_path, - &output_audio_fs_path, - "audio/wav") - .await - .map_err(|e| ProcessSingleJobError::Other(e))?; - - safe_delete_temp_file(&output_audio_fs_path); - - // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // - - let spectrogram_result_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_inference_spectrogram_output_path( - &job.uuid_idempotency_token); // TODO: Don't use this! - - info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path); - - info!("Uploading spectrogram..."); - - args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( - &spectrogram_result_object_path, - &output_spectrogram_fs_path, - "application/json") - .await - .map_err(|e| ProcessSingleJobError::Other(e))?; - - safe_delete_temp_file(&output_spectrogram_fs_path); - - // ==================== DELETE DOWNLOADED FILE ==================== // - - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - safe_delete_temp_directory(&work_temp_dir.path()); - - // ==================== SAVE RECORDS ==================== // - - let text_hash = sha256_hash_string(&cleaned_inference_text) - .map_err(|e| ProcessSingleJobError::Other(e))?; - - info!("Saving tts inference record..."); - - let (id, inference_result_token) = insert_tts_result( - &args.job_dependencies.db.mysql_pool, - JobType::GenericInferenceJob(&job), - &text_hash, - Some(pretrained_vocoder), - &audio_result_object_path, - &spectrogram_result_object_path, - file_metadata.file_size_bytes, - file_metadata.duration_millis.unwrap_or(0), - args.job_dependencies.job.info.container.is_on_prem, - &args.job_dependencies.job.info.container.hostname, - args.job_dependencies.job.system.is_debug_worker) - .await - .map_err(|e| ProcessSingleJobError::Other(e))?; + let inference_result = upload_results(UploadResultArgs { + job_dependencies: args.job_dependencies, + job_progress_reporter: &mut job_progress_reporter, + job, + cleaned_inference_text: &cleaned_inference_text, + work_temp_dir, + pretrained_vocoder, + file_metadata: &file_metadata, + output_audio_fs_path: &output_audio_fs_path, + output_spectrogram_fs_path: &output_spectrogram_fs_path, + upload_as_media_file: model_dependencies.upload_as_media_file, + }).await?; info!("TTS Done. Original text was: {:?}", &job.maybe_raw_inference_text); args.job_dependencies.clients.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), tts_model.model_token.as_str(), - &inference_result_token) + &inference_result.entity_token) .await .map_err(|e| { error!("error publishing event: {:?}", e); @@ -386,20 +336,20 @@ async fn process_job_with_cleanup( job_progress_reporter.log_status("done") .map_err(|e| ProcessSingleJobError::Other(e))?; - info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", - job.id, id, &inference_result_token); + info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Result Token: {}", + job.id, &inference_result.entity_token); Ok(JobSuccessResult { maybe_result_entity: Some(ResultEntity { - entity_type: InferenceResultType::TextToSpeech, - entity_token: inference_result_token + entity_type: inference_result.entity_type, + entity_token: inference_result.entity_token, }), inference_duration, }) } #[derive(Deserialize, Default)] -struct FileMetadata { +pub struct FileMetadata { pub duration_millis: Option, pub mimetype: Option, pub file_size_bytes: u64, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs index baf20bda9c..7b6df49855 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs @@ -20,6 +20,8 @@ pub struct Tacotron2Dependencies { /// Dependencies that are for the sidecar version of TT2 pub sidecar: SidecarDeps, + + pub upload_as_media_file: bool, } pub struct SidecarDeps { @@ -71,7 +73,8 @@ impl Tacotron2Dependencies { health_check_client, health_check_state: HealthCheckState::new(), virtual_lfu_cache, - } + }, + upload_as_media_file: easyenv::get_env_bool_or_default("TT2_AS_MEDIA_FILES", false), }) } } diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs new file mode 100644 index 0000000000..cd5df7ff23 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs @@ -0,0 +1,202 @@ +use std::path::Path; +use anyhow::anyhow; + +use log::info; +use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; + +use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use hashing::sha256::sha256_hash_string::sha256_hash_string; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::column_types::vocoder_type::VocoderType; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs}; +use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType; +use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::FileMetadata; +use crate::job_dependencies::JobDependencies; + +const BUCKET_FILE_PREFIX: &str = "fakeyou_"; +const BUCKET_FILE_EXTENSION: &str = ".wav"; +const MIME_TYPE: &str = "audio/wav"; + +pub struct UploadResultArgs<'a> { + // System + pub job_dependencies: &'a JobDependencies, + pub job_progress_reporter: &'a mut Box, + + // Job + pub job: &'a AvailableInferenceJob, + pub cleaned_inference_text: &'a str, + pub work_temp_dir: &'a TempDir, + pub pretrained_vocoder: VocoderType, + + // Outputs + pub file_metadata: &'a FileMetadata, + pub output_audio_fs_path: &'a Path, + pub output_spectrogram_fs_path: &'a Path, + + // Flags + pub upload_as_media_file: bool, +} + +pub struct ResultDetails { + pub entity_type: InferenceResultType, + pub entity_token: String, +} + +pub async fn upload_results(args: UploadResultArgs<'_>) -> Result{ + if args.upload_as_media_file { + upload_as_media_file(args).await + } else { + upload_as_legacy_tts_result(args).await + } +} + +async fn upload_as_media_file(args: UploadResultArgs<'_>) -> Result{ + + // ==================== UPLOAD AUDIO TO BUCKET ==================== // + + args.job_progress_reporter.log_status("uploading result") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let result_bucket_location = MediaFileBucketPath::generate_new( + Some(BUCKET_FILE_PREFIX), + Some(BUCKET_FILE_EXTENSION) + ); + + let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf(); + + info!("Audio destination bucket path: {:?}", &result_bucket_location.to_full_object_pathbuf()); + + info!("Uploading audio..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &result_bucket_object_pathbuf, + &args.output_audio_fs_path, + "audio/wav") + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let file_checksum = sha256_hash_file(&args.output_audio_fs_path).map_err(|err| { + ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err)) + })?; + + safe_delete_temp_file(&args.output_audio_fs_path); + + // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // + + // (SKIP FOR NOW) + + // ==================== DELETE WORK DIRECTORY ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&args.work_temp_dir.path()); + + // ==================== SAVE RECORDS ==================== // + + info!("Saving tts inference record..."); + + let (inference_result_token, _id) = insert_media_file_from_tacotron2(InsertMediaFileArgs { + pool: &args.job_dependencies.db.mysql_pool, + job: &args.job, + maybe_mime_type: Some(&MIME_TYPE), + file_size_bytes: args.file_metadata.file_size_bytes, + duration_millis: args.file_metadata.duration_millis.unwrap_or(0), + sha256_checksum: &file_checksum, + public_bucket_directory_hash: result_bucket_location.get_object_hash(), + maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX), + maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION), + is_on_prem: args.job_dependencies.job.info.container.is_on_prem, + worker_hostname: &args.job_dependencies.job.info.container.hostname, + worker_cluster: &args.job_dependencies.job.info.container.cluster_name, + }) + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + Ok(ResultDetails { + entity_type: InferenceResultType::MediaFile, + entity_token: inference_result_token.to_string(), + }) +} + + +async fn upload_as_legacy_tts_result(args: UploadResultArgs<'_>) -> Result{ + + // ==================== UPLOAD AUDIO TO BUCKET ==================== // + + args.job_progress_reporter.log_status("uploading result") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let audio_result_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_inference_wav_audio_output_path( + &args.job.uuid_idempotency_token); // TODO: Don't use this! + + info!("Audio destination bucket path: {:?}", &audio_result_object_path); + + info!("Uploading audio..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &audio_result_object_path, + &args.output_audio_fs_path, + "audio/wav") + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + safe_delete_temp_file(&args.output_audio_fs_path); + + // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // + + let spectrogram_result_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_inference_spectrogram_output_path( + &args.job.uuid_idempotency_token); // TODO: Don't use this! + + info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path); + + info!("Uploading spectrogram..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &spectrogram_result_object_path, + &args.output_spectrogram_fs_path, + "application/json") + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + safe_delete_temp_file(&args.output_spectrogram_fs_path); + + // ==================== DELETE WORK DIRECTORY ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&args.work_temp_dir.path()); + + // ==================== SAVE RECORDS ==================== // + + let text_hash = sha256_hash_string(&args.cleaned_inference_text) + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Saving tts inference record..."); + + let (_id, inference_result_token) = insert_tts_result( + &args.job_dependencies.db.mysql_pool, + JobType::GenericInferenceJob(&args.job), + &text_hash, + Some(args.pretrained_vocoder), + &audio_result_object_path, + &spectrogram_result_object_path, + args.file_metadata.file_size_bytes, + args.file_metadata.duration_millis.unwrap_or(0), + args.job_dependencies.job.info.container.is_on_prem, + &args.job_dependencies.job.info.container.hostname, + args.job_dependencies.job.system.is_debug_worker) + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + Ok(ResultDetails { + entity_type: InferenceResultType::TextToSpeech, + entity_token: inference_result_token, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index cb121025b2..20359f1e0d 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -23,7 +23,6 @@ use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::vall_e_x::process_job::VALLEXProcessJobArgs; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::InferenceArgs; -// Clearify what this is for ? const BUCKET_FILE_PREFIX: &str = "fakeyou_"; const BUCKET_FILE_EXTENSION: &str = ".wav"; const MIME_TYPE: &str = "audio/wav"; @@ -208,6 +207,7 @@ pub async fn process_inference_voice( .map_err(|e| ProcessSingleJobError::Other(e))?; // ==================== UPLOAD AUDIO TO BUCKET ==================== + info!("Calculating sha256..."); let file_checksum = sha256_hash_file(&finished_file).map_err(|err| { diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 669066a316..df7cab336c 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -54,6 +54,8 @@ user_input_common = { path = "../../../lib/user_input_common" } billing_component = { path = "../../plugins/billing" } users_component = { path = "../../plugins/users" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" @@ -69,6 +71,7 @@ log = "0.4.14" once_cell = "1.18.0" rand = "0.8.5" regex = "1.5.4" + #symphonia = { version = "0.5.2", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } # Opus support still hasn't landed in Symphonia, so we have to use an older version (~0.5) off-master. # Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/ @@ -159,7 +162,6 @@ pulldown-cmark = "0.9.3" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index b81b67e1b5..96be5ed7d7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -14,7 +14,7 @@ use buckets::public::voice_conversion_results::original_file::VoiceConversionRes use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::common::job_status_plus::JobStatusPlus; -use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; +use mysql_queries::queries::generic_inference::web::get_inference_job_status::{GenericInferenceJobStatus, get_inference_job_status}; use redis_common::redis_keys::RedisKeys; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use crate::http_server::responses::filter_model_name::maybe_filter_model_name; @@ -140,12 +140,15 @@ pub async fn get_inference_job_status_handler( if path.token.as_str().trim() == "None" { // NB: A bunch of Python clients use our API and can fail in this manner. // This was a large traffic driver during the 2023-03-08 outage. + // Presumably it's this client: https://github.com/shards-7/fakeyou.py return Err(GetInferenceJobStatusError::NotFound); } - // NB: Lookup failure is Err(RowNotFound). // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_status = get_inference_job_status(&path.token, &server_state.mysql_pool).await; + let maybe_status = get_inference_job_status( + &path.token, + &server_state.mysql_pool + ).await; let record = match maybe_status { Ok(Some(record)) => record, @@ -198,9 +201,32 @@ pub async fn get_inference_job_status_handler( }; } + let record_for_response = record_to_payload( + record, maybe_extra_status_description); + + let response = GetInferenceJobStatusSuccessResponse { + success: true, + state: record_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| { + error!("error returning response: {:?}", e); + GetInferenceJobStatusError::ServerError + })?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +fn record_to_payload( + record: GenericInferenceJobStatus, + maybe_extra_status_description: Option, +) -> InferenceJobStatusResponsePayload { let inference_category = record.request_details.inference_category.clone(); - let record_for_response = InferenceJobStatusResponsePayload { + InferenceJobStatusResponsePayload { job_token: record.job_token, request: RequestDetailsResponse { inference_category: record.request_details.inference_category, @@ -226,7 +252,7 @@ pub async fn get_inference_job_status_handler( MediaFileBucketPath::from_object_hash( &result_details.public_bucket_location_or_hash, result_details.maybe_media_file_public_bucket_prefix.as_deref(), - result_details.maybe_media_file_public_bucket_extension.as_deref()) + result_details.maybe_media_file_public_bucket_extension.as_deref()) .get_full_object_path_str() .to_string() } @@ -279,20 +305,80 @@ pub async fn get_inference_job_status_handler( }), created_at: record.created_at, updated_at: record.updated_at, - }; - - let response = GetInferenceJobStatusSuccessResponse { - success: true, - state: record_for_response, - }; - - let body = serde_json::to_string(&response) - .map_err(|e| { - error!("error returning response: {:?}", e); - GetInferenceJobStatusError::ServerError - })?; - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) + } +} + +#[cfg(test)] +mod tests { + use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; + use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; + use mysql_queries::queries::generic_inference::web::get_inference_job_status::{GenericInferenceJobStatus, RequestDetails, ResultDetails}; + use crate::http_server::endpoints::inference_job::get_inference_job_status::record_to_payload; + + #[test] + fn text_to_speech_as_media_file() { + let status = GenericInferenceJobStatus { + request_details: RequestDetails { + inference_category: InferenceCategory::TextToSpeech, + maybe_model_type: Some("tacotron2".to_string()), + ..Default::default() + }, + maybe_result_details: Some(ResultDetails { + entity_type: "media_file".to_string(), + entity_token: "m_024pk3p3xrmeyk83chps90jeymnygk".to_string(), + public_bucket_location_or_hash: "tpk848b5s5zwhnwrph75jhyfyja3j42v".to_string(), + maybe_media_file_public_bucket_prefix: Some("fakeyou_".to_string()), + maybe_media_file_public_bucket_extension: Some(".wav".to_string()), + public_bucket_location_is_hash: true, + ..Default::default() + }), + ..Default::default() + }; + + let payload = + record_to_payload(status, None); + + assert!(payload.maybe_result.is_some()); + + let result = payload.maybe_result.expect("should have result"); + + assert_eq!( + Some("/media/t/p/k/8/4/tpk848b5s5zwhnwrph75jhyfyja3j42v/fakeyou_tpk848b5s5zwhnwrph75jhyfyja3j42v.wav"), + result.maybe_public_bucket_media_path.as_deref() + ); + } + + #[test] + fn text_to_speech_as_tts_result() { + let status = GenericInferenceJobStatus { + request_details: RequestDetails { + inference_category: InferenceCategory::TextToSpeech, + maybe_model_type: Some("tacotron2".to_string()), + ..Default::default() + }, + maybe_result_details: Some(ResultDetails { + entity_type: "text_to_speech".to_string(), + entity_token: "TR:qvgdvngw4y54t7rmfepwv8fd965eh".to_string(), + public_bucket_location_or_hash: + "/tts_inference_output/9/9/7/vocodes_997e710c-d733-406c-9b6e-5e1d0c471816.wav".to_string(), + maybe_media_file_public_bucket_prefix: None, + maybe_media_file_public_bucket_extension: None, + public_bucket_location_is_hash: false, + ..Default::default() + }), + ..Default::default() + }; + + let payload = + record_to_payload(status, None); + + assert!(payload.maybe_result.is_some()); + + let result = payload.maybe_result.expect("should have result"); + + assert_eq!( + Some("/tts_inference_output/9/9/7/vocodes_997e710c-d733-406c-9b6e-5e1d0c471816.wav"), + result.maybe_public_bucket_media_path.as_deref(), + ); + } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index c1055c026f..be43c33d8a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -150,6 +150,7 @@ async fn legacy_lookup(job_token: &str, server_state: &ServerState) GetTtsInferenceStatusError::ServerError })?; + // TODO(bt,2023-05-21): Make async. let extra_status_key = RedisKeys::tts_inference_extra_status_info(&job_token); let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { Ok(Some(status)) => { From b26276cfda47c883abe5e01b1d62c6f28dbc58dd Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 6 Dec 2023 23:36:20 -0500 Subject: [PATCH 088/136] done now for code --- crates/service/web/storyteller_web/src/api_doc.rs | 5 ++++- .../src/http_server/common_responses/user_details_lite.rs | 8 +++++--- .../endpoints/weights/list_available_weights.rs | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 178e8871ae..1d96b220e0 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -6,6 +6,8 @@ use crate::http_server::web_utils::response_success_helpers::*; use crate::http_server::endpoints::weights::delete_weight::*; use crate::http_server::endpoints::weights::get_weight::*; use crate::http_server::endpoints::weights::update_weight::*; +use crate::http_server::endpoints::weights::list_weights_by_user::*; +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory, @@ -40,7 +42,8 @@ use utoipa::OpenApi; DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, ListWeightError,ModelWeightForList,ListWeightError, ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, - ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight + ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight, + UserDetailsLight,DefaultAvatarInfo )))] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index d1353f60f7..c325ce0585 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -2,9 +2,9 @@ use tokens::tokens::users::UserToken; use crate::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username; use crate::user_avatars::default_avatar_from_username::default_avatar_from_username; - +use utoipa::ToSchema; /// Everything we need to refer to a user on the public web interface. -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize,ToSchema)] pub struct UserDetailsLight { /// The token for the user pub user_token: UserToken, @@ -82,7 +82,9 @@ impl UserDetailsLight { } } -#[derive(Clone, Serialize)] + + +#[derive(Clone, Serialize,ToSchema)] pub struct DefaultAvatarInfo { pub image_index: u8, pub color_index: u8, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index c41e38de99..7886625891 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -17,7 +17,7 @@ use enums::by_table::model_weights::{ use tokens::tokens::users::UserToken; use tokens::tokens::model_weights::ModelWeightToken; -use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::http_server::common_responses::user_details_lite::{UserDetailsLight,DefaultAvatarInfo}; use crate::server_state::ServerState; use utoipa::ToSchema; From dda2e0840d509d170c0a5021c3435a0c43807ad5 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 7 Dec 2023 00:25:57 -0500 Subject: [PATCH 089/136] Updated docs and weights_types --- .../enums/src/by_table/model_weights/weights_types.rs | 8 ++++++++ .../src/http_server/common_responses/user_details_lite.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 0ef556d927..e094c62874 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -24,6 +24,8 @@ pub enum WeightsType { Tacotron2, #[serde(rename = "loRA")] LoRA, + #[serde(rename = "vall-e")] + VallE, } @@ -37,6 +39,7 @@ impl WeightsType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tt2", Self::LoRA => "loRA", + Self::VallE => "vall-e", } } @@ -49,6 +52,7 @@ impl WeightsType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tt2" => Ok(Self::Tacotron2), "loRA" => Ok(Self::LoRA), + "vall-e" => Ok(Self::VallE), _ => Err(format!("invalid value: {:?}", value)), } } @@ -62,6 +66,7 @@ impl WeightsType { Self::SoVitsSvc, Self::Tacotron2, Self::LoRA, + Self::VallE ]) } } @@ -83,6 +88,7 @@ mod tests { assert_eq!(WeightsType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(WeightsType::Tacotron2.to_str(), "tt2"); assert_eq!(WeightsType::LoRA.to_str(), "loRA"); + assert_eq!(WeightsType::VallE.to_str(), "vall-e"); } #[test] @@ -94,6 +100,7 @@ mod tests { assert_eq!(WeightsType::from_str("so_vits_svc").unwrap(), WeightsType::SoVitsSvc); assert_eq!(WeightsType::from_str("tt2").unwrap(), WeightsType::Tacotron2); assert_eq!(WeightsType::from_str("loRA").unwrap(), WeightsType::LoRA); + assert_eq!(WeightsType::from_str("vall-e").unwrap(), WeightsType::VallE); assert!(WeightsType::from_str("invalid").is_err()); } @@ -108,5 +115,6 @@ mod tests { assert!(variants.contains(&WeightsType::SoVitsSvc)); assert!(variants.contains(&WeightsType::Tacotron2)); assert!(variants.contains(&WeightsType::LoRA)); + assert!(variants.contains(&WeightsType::VallE)); } } diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index c325ce0585..582296e109 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -83,7 +83,6 @@ impl UserDetailsLight { } - #[derive(Clone, Serialize,ToSchema)] pub struct DefaultAvatarInfo { pub image_index: u8, From 8afd10c86459abaeb906a242bba67585f2a1e12b Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 7 Dec 2023 17:52:21 -0500 Subject: [PATCH 090/136] minor fixes to spelling --- .../enums/src/by_table/model_weights/weights_types.rs | 11 +++++------ .../src/queries/model_weights/update_weight.rs | 4 ++-- .../endpoints/weights/list_available_weights.rs | 2 -- .../http_server/endpoints/weights/update_weight.rs | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index e094c62874..905dc39e47 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -6,7 +6,6 @@ use strum::EnumCount; use strum::EnumIter; use utoipa::ToSchema; - #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] pub enum WeightsType { @@ -24,7 +23,7 @@ pub enum WeightsType { Tacotron2, #[serde(rename = "loRA")] LoRA, - #[serde(rename = "vall-e")] + #[serde(rename = "vall_e")] VallE, } @@ -39,7 +38,7 @@ impl WeightsType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tt2", Self::LoRA => "loRA", - Self::VallE => "vall-e", + Self::VallE => "vall_e", } } @@ -52,7 +51,7 @@ impl WeightsType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tt2" => Ok(Self::Tacotron2), "loRA" => Ok(Self::LoRA), - "vall-e" => Ok(Self::VallE), + "vall_e" => Ok(Self::VallE), _ => Err(format!("invalid value: {:?}", value)), } } @@ -88,7 +87,7 @@ mod tests { assert_eq!(WeightsType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(WeightsType::Tacotron2.to_str(), "tt2"); assert_eq!(WeightsType::LoRA.to_str(), "loRA"); - assert_eq!(WeightsType::VallE.to_str(), "vall-e"); + assert_eq!(WeightsType::VallE.to_str(), "vall_e"); } #[test] @@ -100,7 +99,7 @@ mod tests { assert_eq!(WeightsType::from_str("so_vits_svc").unwrap(), WeightsType::SoVitsSvc); assert_eq!(WeightsType::from_str("tt2").unwrap(), WeightsType::Tacotron2); assert_eq!(WeightsType::from_str("loRA").unwrap(), WeightsType::LoRA); - assert_eq!(WeightsType::from_str("vall-e").unwrap(), WeightsType::VallE); + assert_eq!(WeightsType::from_str("vall_e").unwrap(), WeightsType::VallE); assert!(WeightsType::from_str("invalid").is_err()); } diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index 9d95767d88..33bb1ace32 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -12,8 +12,8 @@ pub struct UpdateWeightArgs<'a> { pub description_markdown: Option<&'a str>, pub description_rendered_html: Option<&'a str>, pub creator_set_visibility: Option<&'a Visibility>, - pub weight_type: Option, - pub weight_category: Option, + pub weights_type: Option, + pub weights_category: Option, pub mysql_pool: &'a MySqlPool, } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs index 7886625891..f191e40410 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -139,8 +139,6 @@ pub async fn list_available_weights_handler( return Err(ListWeightError::NotAuthorized); } }; - let page_size: u16 = 10; - let page_index : u16 = 0; let limit = query.page_size; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs index e091c30322..8f69470054 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -147,8 +147,8 @@ pub async fn update_weight_handler( description_markdown: request.description_markdown.as_deref(), description_rendered_html: request.description_rendered_html.as_deref(), creator_set_visibility: request.visibility.as_ref(), - weight_type: request.weight_type.as_deref().map(|s| s.to_string()), - weight_category: request.weight_category.as_deref().map(|s| s.to_string()), + weights_type: request.weight_type.as_deref().map(|s| s.to_string()), + weights_category: request.weight_category.as_deref().map(|s| s.to_string()), }).await; match query_result { From d39962491f596f09fa011f6745ae45e8e75aba5d Mon Sep 17 00:00:00 2001 From: salt Date: Thu, 7 Dec 2023 18:36:51 -0800 Subject: [PATCH 091/136] fix some warnings (mostly unnecessary mut) --- .../src/searches/search_tts_models.rs | 2 +- .../list_available_email_sender_jobs.rs | 2 +- .../job/list_available_generic_inference_jobs.rs | 2 +- .../job/mark_generic_inference_job_failure.rs | 2 +- .../src/queries/media_files/update_media_file.rs | 2 +- .../model_categories/list_categories_query_builder.rs | 6 +++--- .../src/queries/model_weights/update_weight.rs | 2 +- .../list_trending_tts_models.rs | 2 +- .../list_assigned_tts_categories_query_builder.rs | 2 +- .../tts/tts_download_jobs/tts_download_job_queries.rs | 2 +- .../tts/tts_results/list_tts_results_query_builder.rs | 2 +- .../twitch_event_rules/reorder_twitch_event_rules.rs | 2 +- .../src/queries/twitch/twitch_oauth/find.rs | 2 +- .../src/queries/users/user/list_users_query_builder.rs | 10 +++++----- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs index 1ae2cc18ca..30055f1774 100644 --- a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs +++ b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs @@ -27,7 +27,7 @@ pub async fn search_tts_models( let mut response_json = search_response.json::().await?; - let mut hits = response_json.get_mut("hits") + let hits = response_json.get_mut("hits") .map(|hits| hits.take()); let hits = hits.map(|mut hits| { diff --git a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs index 55669b4422..8420295eb5 100644 --- a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs @@ -167,7 +167,7 @@ FROM email_sender_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 4cb3b5fa7e..1f72d607cb 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -336,7 +336,7 @@ FROM generic_inference_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs index a1df90a95a..5a3ea2fb6d 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs @@ -20,7 +20,7 @@ pub async fn mark_generic_inference_job_failure( // statuses: "attempt_failed", "complete_failure", "dead" let mut next_status = "attempt_failed"; - let mut maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { + let maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { let mut reason = reason.trim().to_string(); reason.truncate(512); // Max length of column is 512 reason diff --git a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs index 10ea2024c4..8faa739b0f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs @@ -14,7 +14,7 @@ pub struct UpdateMediaFileArgs<'a> { pub async fn update_media_file(args: UpdateMediaFileArgs<'_>) -> AnyhowResult<()>{ // TODO: enforce checks for idempotency token - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; let query_result = sqlx::query!( r#" UPDATE media_files diff --git a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs index 136751050d..845b7e60f1 100644 --- a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs @@ -184,7 +184,7 @@ impl ListCategoriesQueryBuilder { query = query.bind(model_type); } - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) @@ -239,7 +239,7 @@ LEFT OUTER JOIN users let mut query = "".to_string(); - if let Some(username) = self.scope_creator_user_token.as_deref() { + if let Some(_username) = self.scope_creator_user_token.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.creator_user_token = ?"); first_predicate_added = true; @@ -248,7 +248,7 @@ LEFT OUTER JOIN users } } - if let Some(model_type) = self.scope_model_type.as_deref() { + if let Some(_model_type) = self.scope_model_type.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.model_type = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index cbe0ba55d7..eb95ef40d7 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -16,7 +16,7 @@ pub struct UpdateWeightArgs<'a> { } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; let query_result = sqlx ::query!( diff --git a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs index e81d9c339a..5ec24e4eeb 100644 --- a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs +++ b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs @@ -63,7 +63,7 @@ pub async fn list_trending_tts_models( ]; let query = query_parts.join(" UNION "); - let mut query = sqlx::query_as::<_, RawTrendingModel>(&query); + let query = sqlx::query_as::<_, RawTrendingModel>(&query); let results = query.fetch_all(&mut **mysql_connection).await?; diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs index 23dabea29c..f5bd643639 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs @@ -159,7 +159,7 @@ impl ListAssignedTtsCategoriesQueryBuilder { // NB: Model token is always used for scoping. query = query.bind(&self.scope_tts_model_token); - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs index 490493cf27..0a97d15436 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs @@ -192,7 +192,7 @@ pub async fn mark_tts_upload_job_done( { let status = if success { "complete_success" } else { "complete_failure" }; - let query_result = sqlx::query!( + let _query_result = sqlx::query!( r#" UPDATE tts_model_upload_jobs SET diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs index d11c2b921c..0ed9b446a7 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs @@ -268,7 +268,7 @@ LEFT OUTER JOIN users } } - if let Some(username) = self.scope_creator_username.as_deref() { + if let Some(_username) = self.scope_creator_username.as_deref() { if !first_predicate_added { query.push_str(" WHERE users.username = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs index 1842de1a2a..f2cafbbef1 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs @@ -93,7 +93,7 @@ SET "#, query_inner_positions, query_inner_ip_addresses, user_token, rule_tokens); // NB: This, unfortunately, cannot be statically checked. - let mut query = sqlx::query(&query); + let query = sqlx::query(&query); let result = query.execute(mysql_pool).await; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs index 55d93b4246..015e52f891 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs @@ -145,7 +145,7 @@ impl TwitchOauthTokenFinder { query = query.bind(twitch_username); } - let mut result = query.fetch_optional(mysql_pool).await; + let result = query.fetch_optional(mysql_pool).await; match result { Ok(Some(record)) => { diff --git a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs index e00ed36aa6..d9f623e33e 100644 --- a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs @@ -65,10 +65,10 @@ impl ListUsersQueryBuilder { let internal_results = self.internal_query(mysql_pool).await?; - let mut first_id = internal_results.first() + let first_id = internal_results.first() .map(|raw_result| raw_result.user_id); - let mut last_id = internal_results.last() + let last_id = internal_results.last() .map(|raw_result| raw_result.user_id); let users = internal_results @@ -106,17 +106,17 @@ impl ListUsersQueryBuilder { ) -> AnyhowResult> { let query = self.build_query_string(); - let mut query = sqlx::query_as::<_, UserForListRaw>(&query); + let query = sqlx::query_as::<_, UserForListRaw>(&query); - let mut results = query.fetch_all(mysql_pool) + let results = query.fetch_all(mysql_pool) .await?; Ok(results) } pub fn build_query_string(&self) -> String { - let mut query_string = r#" + let query_string = r#" SELECT id as user_id, token as user_token, From e18b86a55a3b8663ab7ef3f42e5a661906fcbfac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:25:49 -0500 Subject: [PATCH 092/136] everything needs to refer to workplace-hack --- crates/_types/buckets/Cargo.toml | 2 ++ crates/_types/enums/Cargo.toml | 6 +++++- crates/_types/reusable_types/Cargo.toml | 4 +++- crates/_types/tokens/Cargo.toml | 8 +++++++- crates/cli/database_backup/Cargo.toml | 4 +++- crates/cli/database_cleanup/Cargo.toml | 4 +++- crates/cli/dev_database_seed/Cargo.toml | 4 +++- crates/cli/dev_upload_media_file/Cargo.toml | 4 +++- crates/cli/elasticsearch-cli/Cargo.toml | 4 +++- crates/cli/kubectl-pod-cleanup/Cargo.toml | 4 +++- crates/cli/lru-delete-models/Cargo.toml | 4 +++- crates/cli/scrape_test/Cargo.toml | 4 +++- crates/lib/actix_helpers/Cargo.toml | 4 +++- crates/lib/api_clients/cloud_storage/Cargo.toml | 4 +++- crates/lib/api_clients/google_drive_common/Cargo.toml | 4 +++- crates/lib/api_clients/twitch_common/Cargo.toml | 4 +++- crates/lib/bootstrap/Cargo.toml | 4 +++- crates/lib/caching/memory_caching/Cargo.toml | 4 +++- crates/lib/caching/redis_caching/Cargo.toml | 4 +++- crates/lib/collections/Cargo.toml | 2 ++ crates/lib/concurrency/Cargo.toml | 3 +++ crates/lib/cookies/Cargo.toml | 4 +++- crates/lib/crockford/Cargo.toml | 8 +++++++- crates/lib/datetimes/Cargo.toml | 8 +++++++- crates/lib/deprecated/config/Cargo.toml | 4 +++- crates/lib/deprecated/container_common/Cargo.toml | 4 +++- crates/lib/deprecated/http_server_common/Cargo.toml | 4 +++- crates/lib/easyenv/Cargo.toml | 4 +++- crates/lib/email_sender/Cargo.toml | 8 +++++++- crates/lib/errors/Cargo.toml | 2 ++ crates/lib/files/archives/Cargo.toml | 4 +++- crates/lib/files/filesys/Cargo.toml | 5 ++++- crates/lib/files/hashing/Cargo.toml | 4 +++- crates/lib/files/images/Cargo.toml | 4 +++- crates/lib/files/media/Cargo.toml | 5 +++-- crates/lib/files/mimetypes/Cargo.toml | 5 ++++- crates/lib/idempotency/Cargo.toml | 4 +++- crates/lib/jobs_common/Cargo.toml | 4 +++- crates/lib/logging/Cargo.toml | 8 +++++++- crates/lib/mysql_queries/Cargo.toml | 4 +++- crates/lib/password/Cargo.toml | 4 +++- crates/lib/redis_common/Cargo.toml | 4 +++- crates/lib/server_environment/Cargo.toml | 8 +++++++- crates/lib/sqlx_mysql_helpers/Cargo.toml | 5 ++++- crates/lib/storyteller_root/Cargo.toml | 8 +++++++- crates/lib/testing/Cargo.toml | 2 ++ crates/lib/tts_common/Cargo.toml | 4 +++- crates/lib/url_config/Cargo.toml | 4 +++- crates/lib/user_input_common/Cargo.toml | 3 ++- crates/lib/web_scrapers/Cargo.toml | 4 +++- crates/service/job/analytics_job/Cargo.toml | 4 +++- crates/service/job/download_job/Cargo.toml | 4 +++- crates/service/job/email_sender_job/Cargo.toml | 4 +++- crates/service/job/inference_job/Cargo.toml | 4 +++- .../inference_job/config/inference-job.development.env | 1 + crates/service/job/tts_download_job/Cargo.toml | 4 +++- crates/service/job/tts_inference_job/Cargo.toml | 4 +++- crates/service/job/w2l_download_job/Cargo.toml | 4 +++- crates/service/job/w2l_inference_job/Cargo.toml | 4 +++- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 4 +++- crates/service/plugins/billing/Cargo.toml | 4 +++- crates/service/plugins/inference_workloads/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 4 +++- crates/service/web/dummy_service/Cargo.toml | 4 +++- crates/service/web/storyteller_web/Cargo.toml | 1 - crates/service/web/websocket_gateway/Cargo.toml | 6 +++--- 66 files changed, 219 insertions(+), 63 deletions(-) diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml index 80145886da..866ad39853 100644 --- a/crates/_types/buckets/Cargo.toml +++ b/crates/_types/buckets/Cargo.toml @@ -17,6 +17,8 @@ path = "src/lib.rs" # Internal crockford = { path = "../../lib/crockford" } errors = { path = "../../lib/errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index f5c925fa3e..255ed1a634 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -13,6 +13,11 @@ name = "enums" path = "src/lib.rs" [dependencies] +# Internal + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -21,7 +26,6 @@ sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-nativ sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 3fee6d8cfa..3ef6dd4af7 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal # (none) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } @@ -26,7 +29,6 @@ serde_derive = "1.0.125" # MySql Database sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # Test serialization diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index 896b2053d6..581f8c65de 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -13,6 +13,13 @@ name = "tokens" path = "src/lib.rs" [dependencies] + +# Internal +# (none) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -22,7 +29,6 @@ rand = "0.8.5" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } diff --git a/crates/cli/database_backup/Cargo.toml b/crates/cli/database_backup/Cargo.toml index 691890dfae..02bd043b23 100644 --- a/crates/cli/database_backup/Cargo.toml +++ b/crates/cli/database_backup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/database_cleanup/Cargo.toml b/crates/cli/database_cleanup/Cargo.toml index 1600fd11f6..16485809b8 100644 --- a/crates/cli/database_cleanup/Cargo.toml +++ b/crates/cli/database_cleanup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index c07d0431ac..ae63bd60b6 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -28,6 +28,9 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated @@ -39,7 +42,6 @@ tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 5831756152..ff572d4512 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -25,6 +25,9 @@ mimetypes = { path = "../../lib/files/mimetypes" } mysql_queries = { path = "../../lib/mysql_queries" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated log = "0.4.14" @@ -34,7 +37,6 @@ log = "0.4.14" tokio = { version = "1.6.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index f6ca014556..c8f0841c69 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated @@ -41,7 +44,6 @@ elasticsearch = "8.5.0-alpha.1" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/kubectl-pod-cleanup/Cargo.toml b/crates/cli/kubectl-pod-cleanup/Cargo.toml index a0bb79e162..73d842a561 100644 --- a/crates/cli/kubectl-pod-cleanup/Cargo.toml +++ b/crates/cli/kubectl-pod-cleanup/Cargo.toml @@ -18,12 +18,14 @@ config = { path = "../../lib/deprecated/config" } easyenv = { path = "../../lib/easyenv" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile yytimes) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.20" once_cell = "1.18.0" rand = "0.8.5" regex = "1.10.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # Async Runtime # NB: Not using Tokio here since everything is blocking so far. diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml index f7af132bb6..eb08a399a3 100644 --- a/crates/cli/lru-delete-models/Cargo.toml +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -17,10 +17,12 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } logging = { path = "../../lib/logging" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = { version = "4.4.7", features = ["derive"] } walkdir = "2.4.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index 99753086ef..d286897c20 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -18,7 +18,9 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } web_scrapers = { path = "../../lib/web_scrapers" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/actix_helpers/Cargo.toml b/crates/lib/actix_helpers/Cargo.toml index 89c0cbf5d9..4f57ff1785 100644 --- a/crates/lib/actix_helpers/Cargo.toml +++ b/crates/lib/actix_helpers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Actix actix-http = "3.4.0" actix-rt = "2.9.0" @@ -34,7 +37,6 @@ futures-util = "0.3.29" # Serialization serde = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index ee5435b17a..3d4c2d3f77 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal buckets = { path = "../../../_types/buckets" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" @@ -28,4 +31,3 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/google_drive_common/Cargo.toml b/crates/lib/api_clients/google_drive_common/Cargo.toml index d15ae569df..75e1d58c0e 100644 --- a/crates/lib/api_clients/google_drive_common/Cargo.toml +++ b/crates/lib/api_clients/google_drive_common/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } subprocess_common = { path = "../../subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" tempdir = "0.3.7" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/twitch_common/Cargo.toml b/crates/lib/api_clients/twitch_common/Cargo.toml index b377c34ee2..9d1fb2eaa8 100644 --- a/crates/lib/api_clients/twitch_common/Cargo.toml +++ b/crates/lib/api_clients/twitch_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } easyenv = { path = "../../easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" once_cell = "1.18.0" @@ -31,4 +34,3 @@ twitch_oauth2 = "0.6.0-rc.2" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/bootstrap/Cargo.toml b/crates/lib/bootstrap/Cargo.toml index 3baecb1333..e3cfd8376e 100644 --- a/crates/lib/bootstrap/Cargo.toml +++ b/crates/lib/bootstrap/Cargo.toml @@ -19,11 +19,13 @@ easyenv = { path = "../easyenv" } errors = { path = "../errors" } server_environment = { path = "../server_environment" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/caching/memory_caching/Cargo.toml b/crates/lib/caching/memory_caching/Cargo.toml index 06a8da7dfc..22b6ee7736 100644 --- a/crates/lib/caching/memory_caching/Cargo.toml +++ b/crates/lib/caching/memory_caching/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" # In-memory caching lru_time_cache = "0.11.11" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/caching/redis_caching/Cargo.toml b/crates/lib/caching/redis_caching/Cargo.toml index 9f31f5bdbd..a36bc100f1 100644 --- a/crates/lib/caching/redis_caching/Cargo.toml +++ b/crates/lib/caching/redis_caching/Cargo.toml @@ -16,6 +16,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Redis Database r2d2_redis = "0.14.0" @@ -23,7 +26,6 @@ r2d2_redis = "0.14.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml index f74421ec5a..080bd36abb 100644 --- a/crates/lib/collections/Cargo.toml +++ b/crates/lib/collections/Cargo.toml @@ -16,6 +16,8 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/concurrency/Cargo.toml b/crates/lib/concurrency/Cargo.toml index c9511ffd58..4d9e02d580 100644 --- a/crates/lib/concurrency/Cargo.toml +++ b/crates/lib/concurrency/Cargo.toml @@ -14,6 +14,9 @@ path = "src/lib.rs" [dependencies] # Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/cookies/Cargo.toml b/crates/lib/cookies/Cargo.toml index 8abda155b4..6bd0b45ec9 100644 --- a/crates/lib/cookies/Cargo.toml +++ b/crates/lib/cookies/Cargo.toml @@ -17,11 +17,13 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External hmac = "0.10.0" # jwt signing jwt = "0.13.0" sha2 = "0.9.5" # hmac signing -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/crockford/Cargo.toml b/crates/lib/crockford/Cargo.toml index 45fa6a9f04..526e91fb99 100644 --- a/crates/lib/crockford/Cargo.toml +++ b/crates/lib/crockford/Cargo.toml @@ -15,8 +15,14 @@ name = "crockford" path = "src/lib.rs" [dependencies] -rand = "0.8.5" +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +rand = "0.8.5" + [dev-dependencies] # None yet diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 7b69245835..6bb08d5c56 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,9 +13,15 @@ name = "datetimes" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External chrono = { version = "0.4.22", features = ["serde"] } once_cell = "1.18.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # (none) \ No newline at end of file diff --git a/crates/lib/deprecated/config/Cargo.toml b/crates/lib/deprecated/config/Cargo.toml index 6142b3221e..80e5232bb5 100644 --- a/crates/lib/deprecated/config/Cargo.toml +++ b/crates/lib/deprecated/config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" # Internal easyenv = { path = "../../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" url = "2.4.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/container_common/Cargo.toml b/crates/lib/deprecated/container_common/Cargo.toml index 7d0e9cb9dd..32fbc9f9f2 100644 --- a/crates/lib/deprecated/container_common/Cargo.toml +++ b/crates/lib/deprecated/container_common/Cargo.toml @@ -21,10 +21,12 @@ rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } log = "0.4.14" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Serialization serde = { version = "1.0.125", features = ["derive"] } toml = "0.5.8" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 4b7259c477..0a4e4e2159 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" errors = { path = "../../errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -39,7 +42,6 @@ language-tags = "0.3.2" # Parsing single tags richly serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] actix-rt = "2.9.0" diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index 8bfc0d1044..b3e5c1bfee 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -16,11 +16,13 @@ path = "src/lib.rs" # Internal storyteller_root = { path = "../storyteller_root" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index feb580c8c2..020ced75b9 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -10,6 +10,12 @@ path = "src/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Internal errors = { path = "../errors" } -lettre = "0.11.2" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External +lettre = "0.11.2" + diff --git a/crates/lib/errors/Cargo.toml b/crates/lib/errors/Cargo.toml index 6268214f7d..ffa8549b87 100644 --- a/crates/lib/errors/Cargo.toml +++ b/crates/lib/errors/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Consider the dependencies here wisely. # Ideally we have almost nothing as a dependency of this crate. anyhow = "1.0.75" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/files/archives/Cargo.toml b/crates/lib/files/archives/Cargo.toml index d260c780c6..4e02301f23 100644 --- a/crates/lib/files/archives/Cargo.toml +++ b/crates/lib/files/archives/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External zip = "0.6.6" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index bbd7dbf27f..19aae75a68 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -16,11 +16,14 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -log = "0.4.20" # External # KEEP THIS TO A MINIMUM OR NIL. +log = "0.4.20" + [dev-dependencies] # None yet diff --git a/crates/lib/files/hashing/Cargo.toml b/crates/lib/files/hashing/Cargo.toml index adb00e6168..62d47fa10b 100644 --- a/crates/lib/files/hashing/Cargo.toml +++ b/crates/lib/files/hashing/Cargo.toml @@ -17,12 +17,14 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes data-encoding = "2.3.2" md-5 = "0.9.1" # DO NOT USE FOR SERIOUS CRYPTO (we use this for gravatar) ring = "0.17.0-alpha.10" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml index 3327f009b6..ddfe507191 100644 --- a/crates/lib/files/images/Cargo.toml +++ b/crates/lib/files/images/Cargo.toml @@ -19,9 +19,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External image = "0.24.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/files/media/Cargo.toml b/crates/lib/files/media/Cargo.toml index 47d371d0b8..02df59e277 100644 --- a/crates/lib/files/media/Cargo.toml +++ b/crates/lib/files/media/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External log = "0.4.14" @@ -26,7 +29,5 @@ log = "0.4.14" # Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/ symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", rev = "e1a70095c4409478c1ded499fb8dd4afa540fb90", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [dev-dependencies] # None yet diff --git a/crates/lib/files/mimetypes/Cargo.toml b/crates/lib/files/mimetypes/Cargo.toml index 6fe34f2034..0e37ad2df4 100644 --- a/crates/lib/files/mimetypes/Cargo.toml +++ b/crates/lib/files/mimetypes/Cargo.toml @@ -14,9 +14,12 @@ name = "mimetypes" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External infer = "0.11" # Mimetype detection -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + [dev-dependencies] # None yet diff --git a/crates/lib/idempotency/Cargo.toml b/crates/lib/idempotency/Cargo.toml index 578404b0fb..1c927b73ab 100644 --- a/crates/lib/idempotency/Cargo.toml +++ b/crates/lib/idempotency/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # errors = { path = "../../lib/errors" } # (None yet) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External uuid = { version = "0.8.2", features = ["v4"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index 1e6767ea01..98ae1d11d1 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -22,9 +22,11 @@ container_common = { path = "../deprecated/container_common" } errors = { path = "../errors" } redis_common = { path = "../redis_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index bf708a7b84..779659367c 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -14,10 +14,16 @@ name = "logging" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.20" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 4b09061ac0..335aab10f3 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -23,6 +23,9 @@ errors = { path = "../errors" } reusable_types = { path = "../../_types/reusable_types" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -44,5 +47,4 @@ tokio = { version = "1.6.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/password/Cargo.toml b/crates/lib/password/Cargo.toml index f8063637b3..a158566eb1 100644 --- a/crates/lib/password/Cargo.toml +++ b/crates/lib/password/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } #hmac = "0.10.0" # jwt signing #jwt = "0.13.0" #sha2 = "0.9.5" # hmac signing diff --git a/crates/lib/redis_common/Cargo.toml b/crates/lib/redis_common/Cargo.toml index 28aa9fc780..a22cd5841c 100644 --- a/crates/lib/redis_common/Cargo.toml +++ b/crates/lib/redis_common/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal container_common = { path = "../deprecated/container_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" @@ -27,4 +30,3 @@ serde_json = "1.0.108" # TODO: Get rid of in favor of std::time time = "0.3.30" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/server_environment/Cargo.toml b/crates/lib/server_environment/Cargo.toml index 5ece74511a..2124b7857c 100644 --- a/crates/lib/server_environment/Cargo.toml +++ b/crates/lib/server_environment/Cargo.toml @@ -14,8 +14,14 @@ name = "server_environment" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] # None yet diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index 57245fc712..a26c50f6d5 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -16,8 +16,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/storyteller_root/Cargo.toml b/crates/lib/storyteller_root/Cargo.toml index 60143bba40..c32ec7493e 100644 --- a/crates/lib/storyteller_root/Cargo.toml +++ b/crates/lib/storyteller_root/Cargo.toml @@ -13,9 +13,15 @@ name = "storyteller_root" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] serial_test = "2.0.0" tempdir = "0.3.7" diff --git a/crates/lib/testing/Cargo.toml b/crates/lib/testing/Cargo.toml index de25b327df..e90fb429ad 100644 --- a/crates/lib/testing/Cargo.toml +++ b/crates/lib/testing/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 985c8e11fc..d6e4f622b9 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../deprecated/container_common" } user_input_common = { path = "../user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -29,4 +32,3 @@ unicode-segmentation = "1.10.1" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/url_config/Cargo.toml b/crates/lib/url_config/Cargo.toml index 91ee274c39..f458e4cb1a 100644 --- a/crates/lib/url_config/Cargo.toml +++ b/crates/lib/url_config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" http_server_common = { path = "../deprecated/http_server_common" } reusable_types = { path = "../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External actix-web = "4.4.0" anyhow = "1.0.75" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml index d737ce2074..f0a2c6db70 100644 --- a/crates/lib/user_input_common/Cargo.toml +++ b/crates/lib/user_input_common/Cargo.toml @@ -13,10 +13,11 @@ name = "user_input_common" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External once_cell = "1.18.0" pulldown-cmark = "0.9.3" # Markdown regex = "1.5.4" unicode-segmentation = "1.10.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index c9f8bced1e..ef1916e6e7 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" enums = { path = "../../_types/enums" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bytes = "1.3" chrono = { version = "0.4.22", features = ["serde"] } @@ -35,7 +38,6 @@ reqwest = { version = "0.11", features = ["json"] } # HTTP Client # Scraping rss = "2.0.2" scraper = "0.17.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index a1dac56f4d..59f30971d9 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } tokens = { path = "../../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -42,4 +45,3 @@ tokio = { version = "1.6.0", features = ["macros"] } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 42bcfa7769..96598161bc 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -33,6 +33,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } subprocess_common = { path = "../../../lib/subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -60,4 +63,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 965444fc3a..de0c02e9d6 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -41,6 +41,9 @@ testing = { path = "../../../lib/testing" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code # newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -82,4 +85,3 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index d0008b9eac..44b1bb910a 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -38,6 +38,9 @@ subprocess_common = { path = "../../../lib/subprocess_common" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -78,4 +81,3 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 0df22ea669..70deaeb076 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -82,6 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" +#TT2_AS_MEDIA_FILES="true" # ================================= # # # diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 64fe625e3a..7506d7f670 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 6ac55b3d1d..9fbdc2df76 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -27,6 +27,9 @@ memory_caching = { path = "../../../lib/caching/memory_caching" } mysql_queries = { path = "../../../lib/mysql_queries" } tts_common = { path = "../../../lib/tts_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Vendored code newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } @@ -58,4 +61,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 2d3d54d8d0..cfdafda722 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 84c844bd40..34db170d30 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -24,6 +24,9 @@ google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -50,4 +53,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index e9c1bb359a..1fda2a5a73 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -25,6 +25,9 @@ tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -81,7 +84,6 @@ toml = "0.5.8" # Text processing pulldown-cmark = "0.9.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 9623a9f9dc..4e123524e9 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -21,6 +21,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } url_config = { path = "../../../lib/url_config" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -51,7 +54,6 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt async-trait = "0.1.58" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] mockall = "0.11.3" diff --git a/crates/service/plugins/inference_workloads/Cargo.toml b/crates/service/plugins/inference_workloads/Cargo.toml index 6d1f817577..3b3bee9e34 100644 --- a/crates/service/plugins/inference_workloads/Cargo.toml +++ b/crates/service/plugins/inference_workloads/Cargo.toml @@ -14,5 +14,5 @@ path = "src/lib.rs" [dependencies] -# External +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index ca9835c9ae..86db221476 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -31,6 +31,9 @@ server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } user_input_common = { path = "../../../lib/user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -65,4 +68,3 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", r2d2_redis = "0.14.0" byteorder = "1.5.0" strum_macros = "0.25.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 5ed83e2aa9..671723c73b 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -20,6 +20,9 @@ easyenv = { path = "../../../lib/easyenv" } errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" @@ -45,4 +48,3 @@ actix-web = "4.4.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 5b2aea92a1..c57dddc4db 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -163,7 +163,6 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } utoipa = { version = "4", features = ["chrono","actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 2d29dcc45b..f492098015 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } redis_common = { path = "../../../lib/redis_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -89,9 +92,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" -# Temporary (just to ask for oauth credentials during dev) -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "2.1.3", features = ["mysql"] } From 90ec14b8d02b78be760b1b777d88f8703cfc42c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:27:10 -0500 Subject: [PATCH 093/136] vendored needs to refer to workplace-hack --- Cargo.lock | 1 + crates/vendor/limitation/Cargo.toml | 3 +++ crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 24d5e60379..c68180726b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3350,6 +3350,7 @@ version = "0.1.1" dependencies = [ "chrono", "futures 0.1.31", + "my-workspace-hack", "redis 0.13.0", "time 0.1.45", "tokio 0.1.22", diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 86fb33b371..7c4015b643 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -23,6 +23,9 @@ futures = "0.1.29" redis = "0.13.0" time = "0.1.42" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + [dev-dependencies] tokio = "0.1.22" version-sync = "0.8.1" diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index d0e3422320..088a3cf568 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -21,6 +21,8 @@ flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } tokio = { version = "1.1.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] From 6033289a36df50cd1916ff48f5d721e26dd0fbf6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 01:36:39 -0500 Subject: [PATCH 094/136] fix most tests and lints --- .../by_table/model_weights/weights_types.rs | 4 +-- .../src/seeding/model_weights.rs | 32 +++++++++---------- .../tacotron2_v2_early_fakeyou/process_job.rs | 3 -- .../upload_results.rs | 7 ++-- .../tts/vall_e_x/process_job_create_voice.rs | 8 ++--- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 905dc39e47..1b6bbcb132 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -7,7 +7,7 @@ use strum::EnumIter; use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, @@ -106,7 +106,7 @@ mod tests { #[test] fn test_all_variants() { let variants = WeightsType::all_variants(); - assert_eq!(variants.len(), 7); + assert_eq!(variants.len(), 8); assert!(variants.contains(&WeightsType::HifiganTacotron2)); assert!(variants.contains(&WeightsType::RvcV2)); assert!(variants.contains(&WeightsType::StableDiffusion15)); diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 2de742f766..e766086a0b 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -1,22 +1,20 @@ use std::vec; -use buckets::public::media_files::original_file; use log::info; +use sqlx::{MySql, Pool}; -use tokens::tokens::users::UserToken; -use sqlx::{ MySql, Pool }; use enums::by_table::model_weights::{ - weights_types::{WeightsType, self}, - weights_category::{WeightsCategory, self}, + weights_category::WeightsCategory, + weights_types::WeightsType, }; -use errors::{ anyhow, AnyhowResult }; +use enums::common::visibility::Visibility; +use errors::{anyhow, AnyhowResult}; +use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs}; +use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; use crate::seeding::users::HANASHI_USERNAME; -use mysql_queries::queries::model_weights::create_weight::{ create_weight, CreateModelWeightsArgs }; -use tokens::tokens::model_weights::ModelWeightToken; -use enums::common::visibility::Visibility; - -use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { let sd1_5_markdown_description = r#" @@ -326,8 +324,8 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062 let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken weights_type: weights_types, // replace with actual WeightsType - weights_category: weights_category, // replace with actual WeightsCategory - title: title, + weights_category, // replace with actual WeightsCategory + title, maybe_thumbnail_token: Some(thumbnail_token.to_string()), description_markdown: description, description_rendered_html: description_rendered_html.to_string(), @@ -342,11 +340,11 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062 private_bucket_hash: "bucket_hash1".to_string(), maybe_private_bucket_prefix: Some(private_bucket_prefix), maybe_private_bucket_extension: Some(private_bucket_extension), - cached_user_ratings_total_count: cached_user_ratings_total_count, - cached_user_ratings_positive_count: cached_user_ratings_positive_count, - cached_user_ratings_negative_count: cached_user_ratings_negative_count, + cached_user_ratings_total_count, + cached_user_ratings_positive_count, + cached_user_ratings_negative_count, maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32), - version: version, + version, mysql_pool: &mysql_pool, // replace with actual MySqlPool }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 0a3460beb4..c2e16d1e72 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -7,16 +7,13 @@ use anyhow::anyhow; use log::{error, info}; use tempdir::TempDir; -use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; -use hashing::sha256::sha256_hash_string::sha256_hash_string; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; -use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs index cd5df7ff23..87e9747a3d 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs @@ -1,11 +1,10 @@ use std::path::Path; -use anyhow::anyhow; +use anyhow::anyhow; use log::info; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use buckets::public::media_files::original_file::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; @@ -15,8 +14,8 @@ use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReport use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs}; -use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType; use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result; +use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::FileMetadata; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index 3c3547daf1..f83b5b764c 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -61,16 +61,14 @@ pub async fn process_create_voice( let creator_ip_address = &job.creator_ip_address; - let creator_user_token: UserToken; - - match &job.maybe_creator_user_token { + let creator_user_token = match &job.maybe_creator_user_token { Some(token) => { - creator_user_token = UserToken::new_from_str(token); + UserToken::new_from_str(token) }, None => { return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))); } - } + }; // STEP 1. SETUP A TEMP DIRECTORY let work_temp_dir = format!("/tmp/temp_zeroshot_create_voice_{}", job.id.0); From c811296bdcfe5534cc59600e35c55e3e870aa74f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 02:04:34 -0500 Subject: [PATCH 095/136] temporarily disable test --- .../src/queries/model_weights/tests.rs | 26 +++++++------------ execute_tests.sh | 6 +++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index 917fdd98b3..3239e4173e 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -1,38 +1,32 @@ #[cfg(test)] mod tests { - use std::result; - use anyhow::Ok; - use chrono::offset; - use enums::by_table::model_weights::{ weights_types, weights_category }; - use sqlx::{ MySqlPool, Acquire }; use rand::Rng; - - use tokio; use serial_test::serial; - use container_common::anyhow_result::AnyhowResult; + use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; - use config::shared_constants::{ DEFAULT_MYSQL_CONNECTION_STRING }; + use tokio; + + use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; + use container_common::anyhow_result::AnyhowResult; // common tests use enums::by_table::model_weights::{ - weights_types::WeightsType, weights_category::WeightsCategory, + weights_types::WeightsType, }; use enums::common::visibility::Visibility; - use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken}; - use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; use crate::queries::model_weights::create_weight::create_weight; - use crate::queries::model_weights::get_weight::get_weight_by_token; - + use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; use crate::queries::model_weights::delete_weights::{ - delete_weights_as_user, delete_weights_as_mod, + delete_weights_as_user, undelete_weights_as_mod, undelete_weights_as_user, }; + use crate::queries::model_weights::get_weight::get_weight_by_token; use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; - use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username; diff --git a/execute_tests.sh b/execute_tests.sh index 319503eb31..ad5219c463 100755 --- a/execute_tests.sh +++ b/execute_tests.sh @@ -13,6 +13,7 @@ cargo test \ --exclude billing_component \ --exclude fakeyou_client \ --exclude inference-job \ + --exclude mysql_queries \ --exclude sqlite_queries # Single broken tests @@ -22,4 +23,9 @@ cargo test \ --skip util::audiowmark::tests::test_audiowmark \ --skip util::common_commands::audiowmark::tests::test_audiowmark +cargo test \ + --package mysql_queries \ + -- \ + --skip queries::model_weights::tests + echo 'Tests passed.' From 4a806e7d67c8c469f6dc8cde569906caf8345ab0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 02:16:50 -0500 Subject: [PATCH 096/136] codegen db --- ...c1837d645d05198d483564b39a2a8b8034df25e287a4.json | 12 ------------ ...07e053b0393a0e341caf652c18c7160dd0a59247a19a.json | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 .sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json create mode 100644 .sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json diff --git a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json b/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json deleted file mode 100644 index b6a7dbec85..0000000000 --- a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\n UPDATE model_weights\n SET\n title = ?,\n description_markdown = ?,\n maybe_thumbnail_token = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n version = version + 1\n WHERE token = ?\n LIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - }, - "hash": "4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4" -} diff --git a/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json new file mode 100644 index 0000000000..cc5ca52c8c --- /dev/null +++ b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\n UPDATE model_weights\n SET\n title = COALESCE(?, title),\n description_markdown = COALESCE(?, description_markdown),\n maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token),\n description_rendered_html = COALESCE(?, description_rendered_html),\n creator_set_visibility = COALESCE(?, creator_set_visibility),\n version = version + 1\n WHERE token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a" +} From ab1f81036f6b877b1cc7b2b6ba3b181dc4909669 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:00:42 -0500 Subject: [PATCH 097/136] update my-workspace-hack --- Cargo.lock | 3 +- crates/my-workspace-hack/Cargo.toml | 64 ++++++++++++++++------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c68180726b..1aaf8d3a68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3755,11 +3755,11 @@ dependencies = [ "generic-array", "getrandom 0.1.16", "getrandom 0.2.11", - "hashbrown 0.12.3", "hashbrown 0.14.2", "heck", "hmac 0.12.1", "hyper", + "indexmap", "lazy_static", "libc", "log", @@ -3807,6 +3807,7 @@ dependencies = [ "void", "winapi 0.3.9", "windows-sys 0.48.0", + "zip", ] [[package]] diff --git a/crates/my-workspace-hack/Cargo.toml b/crates/my-workspace-hack/Cargo.toml index 22f41a903d..dddf543e0b 100644 --- a/crates/my-workspace-hack/Cargo.toml +++ b/crates/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -40,12 +40,12 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } getrandom-c65f7effa3be6d31 = { package = "getrandom", version = "0.1.16", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } hyper = { version = "0.14.27", features = ["client", "http1", "http2", "runtime", "server"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } miniz_oxide = { version = "0.7.1", features = ["simd"] } @@ -67,19 +67,20 @@ sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } simd-adler32 = { version = "0.3.7" } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } -sqlx = { version = "0.7.2", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx = { version = "0.7.3", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } time = { version = "0.3.30", features = ["macros", "serde-well-known"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } twitch_oauth2 = { version = "0.6.1", features = ["reqwest_client"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } void = { version = "1.0.2" } +zip = { version = "0.6.6" } [build-dependencies] ahash = { version = "0.8.6" } @@ -88,11 +89,13 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } +deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0", features = ["serde"] } +flate2 = { version = "1.0.28" } flume = { version = "0.11.0", default-features = false, features = ["async"] } futures-channel = { version = "0.3.29", features = ["sink"] } futures-core = { version = "0.3.29" } @@ -102,13 +105,15 @@ futures-sink = { version = "0.3.29" } futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } heck = { version = "0.4.1", features = ["unicode"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } +miniz_oxide = { version = "0.7.1", features = ["simd"] } mio = { version = "0.8.9", features = ["net", "os-ext"] } native-tls = { version = "0.2.11", default-features = false, features = ["vendored"] } num-integer = { version = "0.1.45", features = ["i128"] } @@ -124,30 +129,33 @@ serde = { version = "1.0.193", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-macros = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-macros-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +simd-adler32 = { version = "0.3.7" } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-macros = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-macros-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.39", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +time = { version = "0.3.30", features = ["macros", "serde-well-known"] } time-macros = { version = "0.2.15", default-features = false, features = ["formatting", "parsing", "serde"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } +zip = { version = "0.6.6" } [target.x86_64-unknown-linux-gnu.dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-unknown-linux-gnu.build-dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } @@ -163,13 +171,13 @@ spin = { version = "0.9.8" } [target.x86_64-pc-windows-msvc.dependencies] spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } [target.x86_64-pc-windows-msvc.build-dependencies] socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } ### END HAKARI SECTION From b4afec5c2637e3af96b08509cced3e49b6557131 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:21:16 -0500 Subject: [PATCH 098/136] rename endpoint (internally) --- ...animation.rs => enqueue_face_animation.rs} | 52 +++++++++---------- .../http_server/endpoints/animation/mod.rs | 2 +- .../service/web/storyteller_web/src/routes.rs | 4 +- 3 files changed, 29 insertions(+), 29 deletions(-) rename crates/service/web/storyteller_web/src/http_server/endpoints/animation/{enqueue_lipsync_animation.rs => enqueue_face_animation.rs} (86%) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs similarity index 86% rename from crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs index 8d1e536dcc..f9c54eed05 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs @@ -37,7 +37,7 @@ const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; #[derive(Deserialize)] -pub struct EnqueueLipsyncAnimationRequest { +pub struct EnqueueFaceAnimationRequest { uuid_idempotency_token: String, audio_source: AudioSource, @@ -117,35 +117,35 @@ pub struct ImageSource { } #[derive(Serialize)] -pub struct EnqueueLipsyncAnimationSuccessResponse { +pub struct EnqueueFaceAnimationSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] -pub enum EnqueueLipsyncAnimationError { +pub enum EnqueueFaceAnimationError { BadInput(String), NotAuthorized, ServerError, RateLimited, } -impl ResponseError for EnqueueLipsyncAnimationError { +impl ResponseError for EnqueueFaceAnimationError { fn status_code(&self) -> StatusCode { match *self { - EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, - EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, - EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + EnqueueFaceAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueFaceAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueFaceAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueFaceAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(), - EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(), - EnqueueLipsyncAnimationError::ServerError => "server error".to_string(), - EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(), + EnqueueFaceAnimationError::BadInput(reason) => reason.to_string(), + EnqueueFaceAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueFaceAnimationError::ServerError => "server error".to_string(), + EnqueueFaceAnimationError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -153,16 +153,16 @@ impl ResponseError for EnqueueLipsyncAnimationError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl std::fmt::Display for EnqueueLipsyncAnimationError { +impl std::fmt::Display for EnqueueFaceAnimationError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) } } -pub async fn enqueue_lipsync_animation_handler( +pub async fn enqueue_face_animation_handler( http_request: HttpRequest, - request: web::Json, - server_state: web::Data>) -> Result + request: web::Json, + server_state: web::Data>) -> Result { let mut maybe_user_token : Option = None; @@ -171,7 +171,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); @@ -184,7 +184,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; if let Some(user_session) = maybe_user_session.as_ref() { @@ -215,14 +215,14 @@ pub async fn enqueue_lipsync_animation_handler( None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { if user.role.is_banned { - return Err(EnqueueLipsyncAnimationError::NotAuthorized); + return Err(EnqueueFaceAnimationError::NotAuthorized); } &server_state.redis_rate_limiters.logged_in }, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(EnqueueLipsyncAnimationError::RateLimited); + return Err(EnqueueFaceAnimationError::RateLimited); } // ==================== LOOK UP MODEL INFO ==================== // @@ -239,7 +239,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token { LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("audio source not fully specified".to_string())); } }; @@ -249,7 +249,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.image_source.maybe_media_upload_token { LipsyncAnimationImageSource::media_upload_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("image source not fully specified".to_string())); } }; @@ -329,7 +329,7 @@ pub async fn enqueue_lipsync_animation_handler( Ok((job_token, _id)) => job_token, Err(err) => { warn!("New generic inference job creation DB error: {:?}", err); - return Err(EnqueueLipsyncAnimationError::ServerError); + return Err(EnqueueFaceAnimationError::ServerError); } }; @@ -341,16 +341,16 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("error publishing event: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; - let response: EnqueueLipsyncAnimationSuccessResponse = EnqueueLipsyncAnimationSuccessResponse { + let response: EnqueueFaceAnimationSuccessResponse = EnqueueFaceAnimationSuccessResponse { success: true, inference_job_token: job_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?; + .map_err(|_e| EnqueueFaceAnimationError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs index 1102d9357d..dd4f619b5e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -1 +1 @@ -pub mod enqueue_lipsync_animation; \ No newline at end of file +pub mod enqueue_face_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 19dc95b7e9..eb67880dd5 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -11,7 +11,7 @@ use users_component::default_routes::add_suggested_api_v1_account_creation_and_s use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; -use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler; +use crate::http_server::endpoints::animation::enqueue_face_animation::enqueue_face_animation_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; @@ -215,7 +215,7 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> // ==================== Animations ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler) + .add_post("/v1/animation/face_animation/create", enqueue_face_animation_handler) .into_app(); // ==================== "Generic" Inference ==================== From 996f8ce05b1b13d98911b097caedd4f0c6a5260e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 8 Dec 2023 04:38:46 -0500 Subject: [PATCH 099/136] inference-job : sad talker to use audio media files --- .../lipsync/sad_talker/download_audio_file.rs | 166 +++++++++++------- .../lipsync/sad_talker/process_job.rs | 18 +- 2 files changed, 114 insertions(+), 70 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 19c0d958c9..5e6c8c7d47 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; @@ -11,8 +12,10 @@ use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; use mysql_queries::queries::voice_conversion::results::get_voice_conversion_result_for_inference::get_voice_conversion_result_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -23,86 +26,47 @@ pub struct AudioFile { pub filesystem_path: PathBuf, } -pub async fn download_audio_file( - audio_source: &LipsyncAnimationAudioSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadAudioFileArgs<'a> { + pub audio_source: &'a LipsyncAnimationAudioSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result { let bucket_object_path; - match audio_source { + match args.audio_source { LipsyncAnimationAudioSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationAudioSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } - LipsyncAnimationAudioSource::T(tts_result_token) => { - // TODO(bt, 2023-09-08): Implement + LipsyncAnimationAudioSource::T(_tts_result_token) => { + // NB(bt,2023-12-08): This will likely never be implemented now that TTS is on media files. return Err(ProcessSingleJobError::NotYetImplemented) } LipsyncAnimationAudioSource::V(voice_conversion_result_token) => { - let voice_conversion_result = get_voice_conversion_result_for_inference( - voice_conversion_result_token, - false, - &mysql_pool, - ).await; - - let voice_conversion_result = match voice_conversion_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) - } - Err(e) => { - error!("could not query voice conversion result: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); - bucket_object_path = result_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_voice_conversion_result(voice_conversion_result_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("audio.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "audio file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -110,3 +74,83 @@ pub async fn download_audio_file( filesystem_path: downloaded_filesystem_path, }) } + + +async fn from_media_file(media_file_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file: {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} + +async fn from_voice_conversion_result(voice_conversion_result_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let voice_conversion_result = get_voice_conversion_result_for_inference( + voice_conversion_result_token, + false, + &args.mysql_pool, + ).await; + + let voice_conversion_result = match voice_conversion_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) + } + Err(e) => { + error!("could not query voice conversion result: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); + + Ok(result_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 14fd9a32e0..0d75a844bb 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -19,7 +19,7 @@ use tokens::tokens::users::UserToken; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; -use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; +use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs}; use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; @@ -97,15 +97,15 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result Date: Fri, 8 Dec 2023 04:45:28 -0500 Subject: [PATCH 100/136] inference-job - sadtalker to work with image media files --- .../lipsync/sad_talker/download_audio_file.rs | 3 +- .../lipsync/sad_talker/download_image_file.rs | 113 ++++++++++++------ .../lipsync/sad_talker/process_job.rs | 20 ++-- 3 files changed, 86 insertions(+), 50 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 5e6c8c7d47..4e6f8973e7 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -4,8 +4,8 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; use cloud_storage::bucket_client::BucketClient; @@ -75,7 +75,6 @@ pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result) -> Result { let media_file_token = MediaFileToken::new_from_str(media_file_token); let media_file_result = get_media_file( diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 6ac3b4d3e9..3d401f0a7a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -5,12 +5,15 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -21,59 +24,40 @@ pub struct ImageFile { pub filesystem_path: PathBuf, } -pub async fn download_image_file( - image_source: &LipsyncAnimationImageSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadImageFileArgs<'a> { + pub image_source: &'a LipsyncAnimationImageSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_image_file(args: DownloadImageFileArgs<'_>) -> Result { let bucket_object_path; - match image_source { + match args.image_source { LipsyncAnimationImageSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationImageSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("image.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("image.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "image file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -81,3 +65,56 @@ pub async fn download_image_file( filesystem_path: downloaded_filesystem_path, }) } + +async fn from_media_file(media_file_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file : {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file : {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 0d75a844bb..cb6d563156 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -20,7 +20,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs}; -use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; +use crate::job::job_types::lipsync::sad_talker::download_image_file::{download_image_file, DownloadImageFileArgs}; use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; @@ -107,17 +107,17 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result Date: Fri, 8 Dec 2023 05:15:13 -0500 Subject: [PATCH 101/136] env vars --- .env | 3 ++- .../job/inference_job/config/inference-job.development.env | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env b/.env index b205c3a6b3..a39f1b1d8d 100644 --- a/.env +++ b/.env @@ -75,4 +75,5 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt # Development key. Permissible to check in. NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" \ No newline at end of file + +#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 70deaeb076..c16128c8d6 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -82,7 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" -#TT2_AS_MEDIA_FILES="true" +TT2_AS_MEDIA_FILES="true" # ================================= # # # From 5971d95a8052bbd04a8f88262ae415c9bd320f1c Mon Sep 17 00:00:00 2001 From: kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:57:55 +0530 Subject: [PATCH 102/136] Rename favorites -> user_bookmarks, add new table --- ...ebbdb76045ad26aeb1c517a54c296bf31087.json} | 8 +- ...c6fa65ede80d9efcaa4996694ce32c12cc0c.json} | 6 +- ...0b5a04e33d7c6b9c38bef96f275b064a12386.json | 12 ++ ...04d73b9f4a649098fc0ba78cbc20cfde12437.json | 12 -- ...dfe33c94b61046c30940f55cdc73789f94041.json | 12 -- ...1efd0cfa6b20899151f9e7f1a122c1f7a70e.json} | 8 +- ...c352624b79bd4e097f57db5ad59f53fd66995.json | 12 ++ ...449362c074fc2397e23ecd175b6a0425b16b.json} | 8 +- .../down.sql | 5 + .../up.sql | 46 +++++++ .../favorites/favorite_entity_type.rs | 128 ------------------ .../enums/src/by_table/favorites/mod.rs | 1 - crates/_types/enums/src/by_table/mod.rs | 2 +- .../enums/src/by_table/user_bookmarks/mod.rs | 1 + .../user_bookmark_entity_type.rs | 128 ++++++++++++++++++ crates/_types/tokens/src/prefixes.rs | 4 +- crates/_types/tokens/src/tokens/mod.rs | 2 +- .../{favorites.rs => user_bookmarks.rs} | 6 +- .../favorites/favorite_entity_token.rs | 48 ------- .../src/queries/favorites/mod.rs | 7 - crates/lib/mysql_queries/src/queries/mod.rs | 2 +- .../create_user_bookmark.rs} | 22 +-- .../delete_user_bookmark.rs} | 16 +-- .../get_user_bookmark.rs} | 56 ++++---- .../list_user_bookmarks.rs} | 44 +++--- .../list_user_bookmarks_for_entity.rs} | 36 ++--- .../list_user_bookmarks_result.rs} | 40 +++--- .../src/queries/user_bookmarks/mod.rs | 7 + .../user_bookmark_entity_token.rs | 48 +++++++ .../list_favorites_for_entity_handler.rs | 122 ----------------- .../http_server/endpoints/favorites/mod.rs | 5 - .../src/http_server/endpoints/mod.rs | 2 +- .../create_user_bookmark_handler.rs} | 62 ++++----- .../delete_user_bookmark_handler.rs} | 52 +++---- .../list_user_bookmarks_for_entity_handler.rs | 122 +++++++++++++++++ ...ist_user_bookmarks_for_session_handler.rs} | 76 +++++------ .../list_user_bookmarks_for_user_handler.rs} | 74 +++++----- .../endpoints/user_bookmarks/mod.rs | 5 + .../service/web/storyteller_web/src/routes.rs | 22 +-- 39 files changed, 661 insertions(+), 608 deletions(-) rename .sqlx/{query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json => query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json} (76%) rename .sqlx/{query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json => query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json} (76%) create mode 100644 .sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json delete mode 100644 .sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json delete mode 100644 .sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json rename .sqlx/{query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json => query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json} (70%) create mode 100644 .sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json rename .sqlx/{query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json => query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json} (70%) create mode 100644 _sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql create mode 100644 _sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql delete mode 100644 crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs delete mode 100644 crates/_types/enums/src/by_table/favorites/mod.rs create mode 100644 crates/_types/enums/src/by_table/user_bookmarks/mod.rs create mode 100644 crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs rename crates/_types/tokens/src/tokens/{favorites.rs => user_bookmarks.rs} (59%) delete mode 100644 crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs delete mode 100644 crates/lib/mysql_queries/src/queries/favorites/mod.rs rename crates/lib/mysql_queries/src/queries/{favorites/create_favorite.rs => user_bookmarks/create_user_bookmark.rs} (67%) rename crates/lib/mysql_queries/src/queries/{favorites/delete_favorite.rs => user_bookmarks/delete_user_bookmark.rs} (61%) rename crates/lib/mysql_queries/src/queries/{favorites/get_favorite.rs => user_bookmarks/get_user_bookmark.rs} (51%) rename crates/lib/mysql_queries/src/queries/{favorites/list_user_favorites.rs => user_bookmarks/list_user_bookmarks.rs} (75%) rename crates/lib/mysql_queries/src/queries/{favorites/list_favorites_for_entity.rs => user_bookmarks/list_user_bookmarks_for_entity.rs} (67%) rename crates/lib/mysql_queries/src/queries/{favorites/list_user_favorites_result.rs => user_bookmarks/list_user_bookmarks_result.rs} (66%) create mode 100644 crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs create mode 100644 crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/create_favorite_handler.rs => user_bookmarks/create_user_bookmark_handler.rs} (52%) rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/delete_favorite_handler.rs => user_bookmarks/delete_user_bookmark_handler.rs} (58%) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/list_favorites_for_session_handler.rs => user_bookmarks/list_user_bookmarks_for_session_handler.rs} (51%) rename crates/service/web/storyteller_web/src/http_server/endpoints/{favorites/list_favorites_for_user_handler.rs => user_bookmarks/list_user_bookmarks_for_user_handler.rs} (51%) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs diff --git a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json similarity index 76% rename from .sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json rename to .sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json index 0115c0bc84..047b14d7ff 100644 --- a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json +++ b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -120,5 +120,5 @@ true ] }, - "hash": "f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d" + "hash": "02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087" } diff --git a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json similarity index 76% rename from .sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json rename to .sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json index c4b6c41fdb..48546a186c 100644 --- a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json +++ b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -98,5 +98,5 @@ true ] }, - "hash": "99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16" + "hash": "1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c" } diff --git a/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json new file mode 100644 index 0000000000..c829a96d4d --- /dev/null +++ b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nUPDATE user_bookmarks\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386" +} diff --git a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json b/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json deleted file mode 100644 index fefdbaf494..0000000000 --- a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nUPDATE favorites\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437" -} diff --git a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json b/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json deleted file mode 100644 index 01e7177d85..0000000000 --- a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO favorites\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041" -} diff --git a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json similarity index 70% rename from .sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json rename to .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json index 2c4133a738..33334d1769 100644 --- a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json +++ b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038" + "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" } diff --git a/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json new file mode 100644 index 0000000000..d2dc52c79e --- /dev/null +++ b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO user_bookmarks\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + }, + "hash": "e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995" +} diff --git a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json similarity index 70% rename from .sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json rename to .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json index 4ebd7fa022..55d497beee 100644 --- a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json +++ b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4" + "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" } diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql new file mode 100644 index 0000000000..8b6d041461 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE user_bookmarks; \ No newline at end of file diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql new file mode 100644 index 0000000000..54ad3a50b0 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql @@ -0,0 +1,46 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Favorites that can be attached to just about anything. +-- This is a polymorphic table and can apply to a wide variety of entities. +CREATE TABLE user_bookmarks +( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- User making the favorite + user_token VARCHAR(32) NOT NULL, + + -- The type of entity the favorite applies to + entity_type VARCHAR(32) NOT NULL, + + -- The token of the entity the favorite applies to + entity_token VARCHAR(32) NOT NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY index_entity_type_entity_token (entity_type, entity_token), + KEY index_user_token_entity_token (user_token, entity_type), + KEY index_user_token (user_token), + KEY index_created_at (created_at), + KEY index_user_deleted_at (deleted_at) + +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; diff --git a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs b/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs deleted file mode 100644 index 5b96603ff1..0000000000 --- a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs +++ /dev/null @@ -1,128 +0,0 @@ -use serde::Deserialize; -use serde::Serialize; -#[cfg(test)] -use strum::EnumCount; -#[cfg(test)] -use strum::EnumIter; - -/// Used in the `favorites` table in a `VARCHAR(32)` field named `entity_type`. -#[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] -pub enum FavoriteEntityType { - /// User - #[serde(rename = "user")] - User, - - /// TTS model (architecture does not matter) - #[serde(rename = "tts_model")] - TtsModel, - - /// TTS result (architecture does not matter) - #[serde(rename = "tts_result")] - TtsResult, - - /// W2L template - #[serde(rename = "w2l_template")] - W2lTemplate, - - /// W2L result - #[serde(rename = "w2l_result")] - W2lResult, - - /// MediaFile - #[serde(rename = "media_file")] - MediaFile, - - /// VoiceConversionModel - #[serde(rename = "voice_conversion_model")] - VoiceConversionModel, - - /// ZsVoice - #[serde(rename = "zs_voice")] - ZsVoice, -} - -// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! -impl_enum_display_and_debug_using_to_str!(FavoriteEntityType); -impl_mysql_enum_coders!(FavoriteEntityType); - -/// NB: Legacy API for older code. -impl FavoriteEntityType { - pub fn to_str(&self) -> &'static str { - match self { - Self::User => "user", - Self::TtsModel => "tts_model", - Self::TtsResult => "tts_result", - Self::W2lTemplate => "w2l_template", - Self::W2lResult => "w2l_result", - Self::MediaFile => "media_file", - Self::VoiceConversionModel => "voice_conversion_model", - Self::ZsVoice => "zs_voice", - } - } - - pub fn from_str(value: &str) -> Result { - match value { - "user" => Ok(Self::User), - "tts_model" => Ok(Self::TtsModel), - "tts_result" => Ok(Self::TtsResult), - "w2l_template" => Ok(Self::W2lTemplate), - "w2l_result" => Ok(Self::W2lResult), - "media_file" => Ok(Self::MediaFile), - "voice_conversion_model" => Ok(Self::VoiceConversionModel), - "zs_voice" => Ok(Self::ZsVoice), - _ => Err(format!("invalid value: {:?}", value)), - } - } -} - -#[cfg(test)] -mod tests { - use crate::by_table::favorites::favorite_entity_type::FavoriteEntityType; - use crate::test_helpers::assert_serialization; - - mod serde { - use super::*; - - #[test] - fn test_serialization() { - assert_serialization(FavoriteEntityType::User, "user"); - assert_serialization(FavoriteEntityType::TtsModel, "tts_model"); - assert_serialization(FavoriteEntityType::TtsResult, "tts_result"); - assert_serialization(FavoriteEntityType::W2lTemplate, "w2l_template"); - assert_serialization(FavoriteEntityType::W2lResult, "w2l_result"); - assert_serialization(FavoriteEntityType::MediaFile, "media_file"); - assert_serialization(FavoriteEntityType::VoiceConversionModel, "voice_conversion_model"); - assert_serialization(FavoriteEntityType::ZsVoice, "zs_voice"); - } - } - - mod impl_methods { - use super::*; - - #[test] - fn test_to_str() { - assert_eq!(FavoriteEntityType::User.to_str(), "user"); - assert_eq!(FavoriteEntityType::TtsModel.to_str(), "tts_model"); - assert_eq!(FavoriteEntityType::TtsResult.to_str(), "tts_result"); - assert_eq!(FavoriteEntityType::W2lTemplate.to_str(), "w2l_template"); - assert_eq!(FavoriteEntityType::W2lResult.to_str(), "w2l_result"); - assert_eq!(FavoriteEntityType::MediaFile.to_str(), "media_file"); - assert_eq!(FavoriteEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); - assert_eq!(FavoriteEntityType::ZsVoice.to_str(), "zs_voice"); - } - - #[test] - fn test_from_str() { - assert_eq!(FavoriteEntityType::from_str("user").unwrap(), FavoriteEntityType::User); - assert_eq!(FavoriteEntityType::from_str("tts_model").unwrap(), FavoriteEntityType::TtsModel); - assert_eq!(FavoriteEntityType::from_str("tts_result").unwrap(), FavoriteEntityType::TtsResult); - assert_eq!(FavoriteEntityType::from_str("w2l_template").unwrap(), FavoriteEntityType::W2lTemplate); - assert_eq!(FavoriteEntityType::from_str("w2l_result").unwrap(), FavoriteEntityType::W2lResult); - assert_eq!(FavoriteEntityType::from_str("media_file").unwrap(), FavoriteEntityType::MediaFile); - assert_eq!(FavoriteEntityType::from_str("voice_conversion_model").unwrap(), FavoriteEntityType::VoiceConversionModel); - assert_eq!(FavoriteEntityType::from_str("zs_voice").unwrap(), FavoriteEntityType::ZsVoice); - assert!(FavoriteEntityType::from_str("foo").is_err()); - } - } -} diff --git a/crates/_types/enums/src/by_table/favorites/mod.rs b/crates/_types/enums/src/by_table/favorites/mod.rs deleted file mode 100644 index 18ac93ed9c..0000000000 --- a/crates/_types/enums/src/by_table/favorites/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod favorite_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/mod.rs b/crates/_types/enums/src/by_table/mod.rs index 91f8cb15da..97876b2094 100644 --- a/crates/_types/enums/src/by_table/mod.rs +++ b/crates/_types/enums/src/by_table/mod.rs @@ -10,6 +10,7 @@ pub mod media_uploads; pub mod model_categories; pub mod trending_model_analytics; pub mod tts_models; +pub mod user_bookmarks; pub mod user_ratings; pub mod voice_conversion_models; pub mod voice_conversion_results; @@ -19,4 +20,3 @@ pub mod model_weights; pub mod tts_render_tasks; pub mod web_rendition_targets; pub mod web_scraping_targets; -pub mod favorites; diff --git a/crates/_types/enums/src/by_table/user_bookmarks/mod.rs b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs new file mode 100644 index 0000000000..c3c539aad3 --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs @@ -0,0 +1 @@ +pub mod user_bookmark_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs new file mode 100644 index 0000000000..69bf47cf5e --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs @@ -0,0 +1,128 @@ +use serde::Deserialize; +use serde::Serialize; +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the `user_bookmarks` table in a `VARCHAR(32)` field named `entity_type`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum UserBookmarkEntityType { + /// User + #[serde(rename = "user")] + User, + + /// TTS model (architecture does not matter) + #[serde(rename = "tts_model")] + TtsModel, + + /// TTS result (architecture does not matter) + #[serde(rename = "tts_result")] + TtsResult, + + /// W2L template + #[serde(rename = "w2l_template")] + W2lTemplate, + + /// W2L result + #[serde(rename = "w2l_result")] + W2lResult, + + /// MediaFile + #[serde(rename = "media_file")] + MediaFile, + + /// VoiceConversionModel + #[serde(rename = "voice_conversion_model")] + VoiceConversionModel, + + /// ZsVoice + #[serde(rename = "zs_voice")] + ZsVoice, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(UserBookmarkEntityType); +impl_mysql_enum_coders!(UserBookmarkEntityType); + +/// NB: Legacy API for older code. +impl UserBookmarkEntityType { + pub fn to_str(&self) -> &'static str { + match self { + Self::User => "user", + Self::TtsModel => "tts_model", + Self::TtsResult => "tts_result", + Self::W2lTemplate => "w2l_template", + Self::W2lResult => "w2l_result", + Self::MediaFile => "media_file", + Self::VoiceConversionModel => "voice_conversion_model", + Self::ZsVoice => "zs_voice", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "user" => Ok(Self::User), + "tts_model" => Ok(Self::TtsModel), + "tts_result" => Ok(Self::TtsResult), + "w2l_template" => Ok(Self::W2lTemplate), + "w2l_result" => Ok(Self::W2lResult), + "media_file" => Ok(Self::MediaFile), + "voice_conversion_model" => Ok(Self::VoiceConversionModel), + "zs_voice" => Ok(Self::ZsVoice), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(UserBookmarkEntityType::User, "user"); + assert_serialization(UserBookmarkEntityType::TtsModel, "tts_model"); + assert_serialization(UserBookmarkEntityType::TtsResult, "tts_result"); + assert_serialization(UserBookmarkEntityType::W2lTemplate, "w2l_template"); + assert_serialization(UserBookmarkEntityType::W2lResult, "w2l_result"); + assert_serialization(UserBookmarkEntityType::MediaFile, "media_file"); + assert_serialization(UserBookmarkEntityType::VoiceConversionModel, "voice_conversion_model"); + assert_serialization(UserBookmarkEntityType::ZsVoice, "zs_voice"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(UserBookmarkEntityType::User.to_str(), "user"); + assert_eq!(UserBookmarkEntityType::TtsModel.to_str(), "tts_model"); + assert_eq!(UserBookmarkEntityType::TtsResult.to_str(), "tts_result"); + assert_eq!(UserBookmarkEntityType::W2lTemplate.to_str(), "w2l_template"); + assert_eq!(UserBookmarkEntityType::W2lResult.to_str(), "w2l_result"); + assert_eq!(UserBookmarkEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserBookmarkEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); + assert_eq!(UserBookmarkEntityType::ZsVoice.to_str(), "zs_voice"); + } + + #[test] + fn test_from_str() { + assert_eq!(UserBookmarkEntityType::from_str("user").unwrap(), UserBookmarkEntityType::User); + assert_eq!(UserBookmarkEntityType::from_str("tts_model").unwrap(), UserBookmarkEntityType::TtsModel); + assert_eq!(UserBookmarkEntityType::from_str("tts_result").unwrap(), UserBookmarkEntityType::TtsResult); + assert_eq!(UserBookmarkEntityType::from_str("w2l_template").unwrap(), UserBookmarkEntityType::W2lTemplate); + assert_eq!(UserBookmarkEntityType::from_str("w2l_result").unwrap(), UserBookmarkEntityType::W2lResult); + assert_eq!(UserBookmarkEntityType::from_str("media_file").unwrap(), UserBookmarkEntityType::MediaFile); + assert_eq!(UserBookmarkEntityType::from_str("voice_conversion_model").unwrap(), UserBookmarkEntityType::VoiceConversionModel); + assert_eq!(UserBookmarkEntityType::from_str("zs_voice").unwrap(), UserBookmarkEntityType::ZsVoice); + assert!(UserBookmarkEntityType::from_str("foo").is_err()); + } + } +} diff --git a/crates/_types/tokens/src/prefixes.rs b/crates/_types/tokens/src/prefixes.rs index 6e24a9802b..382492bae4 100644 --- a/crates/_types/tokens/src/prefixes.rs +++ b/crates/_types/tokens/src/prefixes.rs @@ -27,7 +27,6 @@ pub(crate) enum TokenPrefix { Comment, DownloadJob, EmailSenderJob, - Favorite, InferenceJob, MediaFile, MediaUpload, @@ -36,6 +35,7 @@ pub(crate) enum TokenPrefix { PasswordReset, TtsRenderTask, // NB: aichatbot / sqlite User, + UserBookmark, UserSession, VoiceConversionModel, VoiceConversionResult, @@ -94,7 +94,6 @@ impl PrefixGenerator for TokenPrefix { Self::DownloadJob => "jdown_", // NB: Previously "JGUP:" Self::EmailSenderJob => "email_job_", Self::InferenceJob => "jinf_", - Self::Favorite => "fav_", Self::MediaFile => "m_", Self::MediaUpload => "mu_", Self::ModelWeight => "weight_", @@ -102,6 +101,7 @@ impl PrefixGenerator for TokenPrefix { Self::PasswordReset => "pw_reset_", Self::TtsRenderTask => "tts_task_", Self::User => "user_", // NB: Previously "U:" + Self::UserBookmark => "ub_", Self::UserSession => "session_", Self::VoiceConversionModel => "vcm_", Self::VoiceConversionResult => "vcr_", diff --git a/crates/_types/tokens/src/tokens/mod.rs b/crates/_types/tokens/src/tokens/mod.rs index e75f732b55..b7c4eed592 100644 --- a/crates/_types/tokens/src/tokens/mod.rs +++ b/crates/_types/tokens/src/tokens/mod.rs @@ -34,4 +34,4 @@ pub mod w2l_templates; pub mod zs_voice_dataset_samples; pub mod zs_voice_datasets; pub mod zs_voices; -pub mod favorites; +pub mod user_bookmarks; diff --git a/crates/_types/tokens/src/tokens/favorites.rs b/crates/_types/tokens/src/tokens/user_bookmarks.rs similarity index 59% rename from crates/_types/tokens/src/tokens/favorites.rs rename to crates/_types/tokens/src/tokens/user_bookmarks.rs index 78ecb7e0b9..ac91dc079d 100644 --- a/crates/_types/tokens/src/tokens/favorites.rs +++ b/crates/_types/tokens/src/tokens/user_bookmarks.rs @@ -8,7 +8,7 @@ use crate::prefixes::TokenPrefix; /// The primary key for Audit Logs. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] -pub struct FavoriteToken(pub String); +pub struct UserBookmarkToken(pub String); -impl_string_token!(FavoriteToken); -impl_crockford_generator!(FavoriteToken, 32usize, TokenPrefix::Favorite, CrockfordLower); +impl_string_token!(UserBookmarkToken); +impl_crockford_generator!(UserBookmarkToken, 32usize, TokenPrefix::UserBookmark, CrockfordLower); diff --git a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs b/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs deleted file mode 100644 index 3a0007253f..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs +++ /dev/null @@ -1,48 +0,0 @@ -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use tokens::tokens::media_files::MediaFileToken; -use tokens::tokens::tts_models::TtsModelToken; -use tokens::tokens::tts_results::TtsResultToken; -use tokens::tokens::users::UserToken; -use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; -use tokens::tokens::w2l_results::W2lResultToken; -use tokens::tokens::w2l_templates::W2lTemplateToken; -use tokens::tokens::zs_voices::ZsVoiceToken; - -pub enum FavoriteEntityToken { - User(UserToken), - TtsModel(TtsModelToken), - TtsResult(TtsResultToken), - W2lTemplate(W2lTemplateToken), - W2lResult(W2lResultToken), - MediaFile(MediaFileToken), - VoiceConversionModel(VoiceConversionModelToken), - ZsVoice(ZsVoiceToken), -} - -impl FavoriteEntityToken { - pub fn from_entity_type_and_token(entity_type: FavoriteEntityType, token: &str) -> Self { - match entity_type { - FavoriteEntityType::User => Self::User(UserToken::new_from_str(token)), - FavoriteEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), - FavoriteEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), - FavoriteEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), - FavoriteEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), - FavoriteEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), - FavoriteEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), - FavoriteEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), - } - } - - pub fn get_composite_keys(&self) -> (FavoriteEntityType, &str) { - match self { - FavoriteEntityToken::User(user_token) => (FavoriteEntityType::User, user_token.as_str()), - FavoriteEntityToken::TtsModel(tts_model_token) => (FavoriteEntityType::TtsModel, tts_model_token.as_str()), - FavoriteEntityToken::TtsResult(tts_result_token) => (FavoriteEntityType::TtsResult, tts_result_token.as_str()), - FavoriteEntityToken::W2lTemplate(w2l_template_token) => (FavoriteEntityType::W2lTemplate, w2l_template_token.as_str()), - FavoriteEntityToken::W2lResult(w2l_result_token) => (FavoriteEntityType::W2lResult, w2l_result_token.as_str()), - FavoriteEntityToken::MediaFile(media_file_token) => (FavoriteEntityType::MediaFile, media_file_token.as_str()), - FavoriteEntityToken::VoiceConversionModel(voice_conversion_model_token) => (FavoriteEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), - FavoriteEntityToken::ZsVoice(zs_voice_token) => (FavoriteEntityType::ZsVoice, zs_voice_token.as_str()), - } - } -} diff --git a/crates/lib/mysql_queries/src/queries/favorites/mod.rs b/crates/lib/mysql_queries/src/queries/favorites/mod.rs deleted file mode 100644 index 8cc8a3c7c5..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod create_favorite; -pub mod delete_favorite; -pub mod favorite_entity_token; -pub mod get_favorite; -pub mod list_favorites_for_entity; -pub mod list_user_favorites; -pub mod list_user_favorites_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/mod.rs b/crates/lib/mysql_queries/src/queries/mod.rs index 1acd2d7a89..39499425b4 100644 --- a/crates/lib/mysql_queries/src/queries/mod.rs +++ b/crates/lib/mysql_queries/src/queries/mod.rs @@ -3,7 +3,6 @@ pub mod audit_logs; pub mod billing; pub mod comments; pub mod email_sender_jobs; -pub mod favorites; pub mod generic_download; pub mod generic_inference; pub mod generic_synthetic_ids; @@ -18,6 +17,7 @@ pub mod trending_model_analytics; pub mod tts; pub mod twitch; pub mod users; +pub mod user_bookmarks; pub mod vocoder; pub mod voice_clone_requests; pub mod voice_conversion; diff --git a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs index a6decf5fff..b897d8daaf 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs @@ -4,15 +4,15 @@ use anyhow::anyhow; use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct CreateFavoriteArgs<'e, 'c, E> +pub struct CreateUserBookmarkArgs<'e, 'c, E> where E: 'e + Executor<'c, Database = MySql> { - pub entity_token: &'e FavoriteEntityToken, + pub entity_token: &'e UserBookmarkEntityToken, pub user_token: &'e UserToken, @@ -23,19 +23,19 @@ pub struct CreateFavoriteArgs<'e, 'c, E> pub phantom: PhantomData<&'c E>, } -pub async fn create_favorite<'e, 'c : 'e, E>( - args: CreateFavoriteArgs<'e, 'c, E>, +pub async fn create_user_bookmark<'e, 'c : 'e, E>( + args: CreateUserBookmarkArgs<'e, 'c, E>, ) - -> AnyhowResult + -> AnyhowResult where E: 'e + Executor<'c, Database = MySql> { - let favorite_token = FavoriteToken::generate(); + let user_bookmark_token = UserBookmarkToken::generate(); let (entity_type, entity_token) = args.entity_token.get_composite_keys(); let query_result = sqlx::query!( r#" -INSERT INTO favorites +INSERT INTO user_bookmarks SET token = ?, user_token = ?, @@ -46,7 +46,7 @@ ON DUPLICATE KEY UPDATE deleted_at = NULL, version = version + 1 "#, - &favorite_token, + &user_bookmark_token, args.user_token, entity_type, entity_token, @@ -59,5 +59,5 @@ ON DUPLICATE KEY UPDATE Err(err) => return Err(anyhow!("Mysql error: {:?}", err)), }; - Ok(favorite_token) + Ok(user_bookmark_token) } diff --git a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs similarity index 61% rename from crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs index 0ec20c4e6a..025b6f51c7 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs @@ -1,22 +1,22 @@ use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; // NB: UserToken is only supplied as an optimistic form of validation -pub async fn delete_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - user_token: &'e UserToken, - mysql_executor: E +pub async fn delete_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + user_token: &'e UserToken, + mysql_executor: E ) - -> AnyhowResult<()> + -> AnyhowResult<()> where E: 'e + Executor<'c, Database = MySql> { sqlx::query!( r#" -UPDATE favorites +UPDATE user_bookmarks SET deleted_at = CURRENT_TIMESTAMP, version = version + 1 @@ -25,7 +25,7 @@ token = ? AND user_token = ? LIMIT 1 "#, - favorite_token, + user_bookmark_token, user_token ) .execute(mysql_executor) diff --git a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs similarity index 51% rename from crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs index 01d46404bc..efb6903070 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs @@ -2,15 +2,15 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use sqlx::{Executor, MySql}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, @@ -23,21 +23,21 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn get_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - mysql_executor: E +pub async fn get_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + mysql_executor: E ) - -> AnyhowResult> + -> AnyhowResult> where E: 'e + Executor<'c, Database = MySql> { let maybe_results = sqlx::query_as!( - RawFavorite, + RawUserBookmark, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, @@ -50,29 +50,29 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE f.token = ? "#, - favorite_token + user_bookmark_token ) .fetch_one(mysql_executor) .await; match maybe_results { - Ok(favorite) => Ok(Some(Favorite { - token: favorite.token, - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - user_token: favorite.user_token, - username: favorite.username, - user_display_name: favorite.user_display_name, - user_gravatar_hash: favorite.user_gravatar_hash, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - maybe_deleted_at: favorite.deleted_at, + Ok(user_bookmark) => Ok(Some(UserBookmark { + token: user_bookmark.token, + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + user_token: user_bookmark.user_token, + username: user_bookmark.username, + user_display_name: user_bookmark.user_display_name, + user_gravatar_hash: user_bookmark.user_gravatar_hash, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + maybe_deleted_at: user_bookmark.deleted_at, })), Err(err) => match err { sqlx::Error::RowNotFound => Ok(None), @@ -81,10 +81,10 @@ WHERE } } -pub struct RawFavorite { - pub token: FavoriteToken, +pub struct RawUserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs similarity index 75% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index f42c7da477..1e1896a750 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -2,24 +2,26 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; +use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; +use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; +use tokens::tokens::user_bookmarks::UserBookmarkToken; -use crate::queries::favorites::list_user_favorites_result::{RawUserFavoriteRecord, UserFavorite}; -pub async fn list_user_favorites( +pub async fn list_user_bookmarks( username: &str, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { /// TODO(bt,2023-11-21): Maybe this query can use a switch /// See: https://stackoverflow.com/questions/2761574/mysql-use-case-else-value-as-join-parameter let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -39,7 +41,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -65,28 +67,28 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub async fn list_user_favorites_by_entity_type( - username: &str, - entity_type: FavoriteEntityType, - mysql_pool: &MySqlPool -) -> AnyhowResult> { +pub async fn list_user_user_bookmarks_by_entity_type( + username: &str, + entity_type: UserBookmarkEntityType, + mysql_pool: &MySqlPool +) -> AnyhowResult> { let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -106,7 +108,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -134,12 +136,12 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs index e1080ac694..3df0f0812c 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs @@ -4,13 +4,13 @@ use log::warn; use sqlx::MySqlPool; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, pub user_token: UserToken, pub username: String, @@ -22,18 +22,18 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn list_favorites_for_entity( - favorite_entity_token: FavoriteEntityToken, +pub async fn list_user_bookmarks_for_entity( + user_bookmark_entity_token: UserBookmarkEntityToken, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { - let (entity_type, entity_token) = favorite_entity_token.get_composite_keys(); + let (entity_type, entity_token) = user_bookmark_entity_token.get_composite_keys(); let maybe_results= sqlx::query_as!( - RawFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, u.display_name as user_display_name, @@ -44,7 +44,7 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE @@ -64,18 +64,18 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub struct RawFavoriteRecord { - token: FavoriteToken, +pub struct RawUserBookmarkRecord { + token: UserBookmarkToken, user_token: UserToken, username: String, @@ -87,9 +87,9 @@ pub struct RawFavoriteRecord { deleted_at: Option>, } -impl RawFavoriteRecord { - pub fn into_public_type(self) -> Favorite { - Favorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs similarity index 66% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index c391a5d8e6..e56d1de9d8 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -1,18 +1,18 @@ use chrono::{DateTime, Utc}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_type::MediaFileType; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct UserFavorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, - /// Something descriptive about the favorited entity. + /// Something descriptive about the bookmarked entity. /// This might be TTS text, a username, etc. It depends on the entity type. pub maybe_entity_descriptive_text: Option, @@ -26,10 +26,10 @@ pub struct UserFavorite { pub maybe_deleted_at: Option>, } -pub struct RawUserFavoriteRecord { - pub (crate) token: FavoriteToken, +pub struct RawUserBookmarkRecord { + pub (crate) token: UserBookmarkToken, - pub (crate) entity_type: FavoriteEntityType, + pub (crate) entity_type: UserBookmarkEntityType, pub (crate) entity_token: String, pub (crate) user_token: UserToken, @@ -51,27 +51,27 @@ pub struct RawUserFavoriteRecord { pub (crate) maybe_descriptive_text_zs_voice_title: Option, } -impl RawUserFavoriteRecord { - pub fn into_public_type(self) -> UserFavorite { - UserFavorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, entity_type: self.entity_type, entity_token: self.entity_token, maybe_entity_descriptive_text: match self.entity_type { - FavoriteEntityType::User => self.maybe_descriptive_text_user_display_name, - FavoriteEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, - FavoriteEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, - FavoriteEntityType::W2lTemplate => None, - FavoriteEntityType::W2lResult => None, + UserBookmarkEntityType::User => self.maybe_descriptive_text_user_display_name, + UserBookmarkEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, + UserBookmarkEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, + UserBookmarkEntityType::W2lTemplate => None, + UserBookmarkEntityType::W2lResult => None, // TODO(bt,2023-11-21): Summary text needs to be better enriched. - FavoriteEntityType::MediaFile => self.maybe_media_file_type + UserBookmarkEntityType::MediaFile => self.maybe_media_file_type .and_then(|media_file_type| match media_file_type { MediaFileType::Audio => Some("audio media file".to_string()), MediaFileType::Image => Some("image media file".to_string()), MediaFileType::Video => Some("video media file".to_string()), }), - FavoriteEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, - FavoriteEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, + UserBookmarkEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, + UserBookmarkEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, }, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs new file mode 100644 index 0000000000..df6d7f415c --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs @@ -0,0 +1,7 @@ +pub mod delete_user_bookmark; +pub mod create_user_bookmark; +pub mod user_bookmark_entity_token; +pub mod get_user_bookmark; +pub mod list_user_bookmarks_for_entity; +pub mod list_user_bookmarks; +pub mod list_user_bookmarks_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs new file mode 100644 index 0000000000..5fbf8224ad --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs @@ -0,0 +1,48 @@ +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use tokens::tokens::zs_voices::ZsVoiceToken; + +pub enum UserBookmarkEntityToken { + User(UserToken), + TtsModel(TtsModelToken), + TtsResult(TtsResultToken), + W2lTemplate(W2lTemplateToken), + W2lResult(W2lResultToken), + MediaFile(MediaFileToken), + VoiceConversionModel(VoiceConversionModelToken), + ZsVoice(ZsVoiceToken), +} + +impl UserBookmarkEntityToken { + pub fn from_entity_type_and_token(entity_type: UserBookmarkEntityType, token: &str) -> Self { + match entity_type { + UserBookmarkEntityType::User => Self::User(UserToken::new_from_str(token)), + UserBookmarkEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), + UserBookmarkEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), + UserBookmarkEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), + UserBookmarkEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), + UserBookmarkEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), + UserBookmarkEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), + UserBookmarkEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), + } + } + + pub fn get_composite_keys(&self) -> (UserBookmarkEntityType, &str) { + match self { + UserBookmarkEntityToken::User(user_token) => (UserBookmarkEntityType::User, user_token.as_str()), + UserBookmarkEntityToken::TtsModel(tts_model_token) => (UserBookmarkEntityType::TtsModel, tts_model_token.as_str()), + UserBookmarkEntityToken::TtsResult(tts_result_token) => (UserBookmarkEntityType::TtsResult, tts_result_token.as_str()), + UserBookmarkEntityToken::W2lTemplate(w2l_template_token) => (UserBookmarkEntityType::W2lTemplate, w2l_template_token.as_str()), + UserBookmarkEntityToken::W2lResult(w2l_result_token) => (UserBookmarkEntityType::W2lResult, w2l_result_token.as_str()), + UserBookmarkEntityToken::MediaFile(media_file_token) => (UserBookmarkEntityType::MediaFile, media_file_token.as_str()), + UserBookmarkEntityToken::VoiceConversionModel(voice_conversion_model_token) => (UserBookmarkEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), + UserBookmarkEntityToken::ZsVoice(zs_voice_token) => (UserBookmarkEntityType::ZsVoice, zs_voice_token.as_str()), + } + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs deleted file mode 100644 index efbb773b84..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs +++ /dev/null @@ -1,122 +0,0 @@ -// NB: Incrementally getting rid of build warnings... -#![forbid(unused_imports)] -#![forbid(unused_mut)] -#![forbid(unused_variables)] - -use std::fmt; -use std::sync::Arc; - -use actix_web::{HttpRequest, HttpResponse, web}; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; -use actix_web::web::Path; -use chrono::{DateTime, Utc}; -use log::warn; - -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::list_favorites_for_entity::list_favorites_for_entity; -use tokens::tokens::favorites::FavoriteToken; - -use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::server_state::ServerState; - -/// For the URL PathInfo -#[derive(Deserialize)] -pub struct ListFavoritesPathInfo { - entity_type: FavoriteEntityType, - entity_token: String, -} - -#[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { - pub success: bool, - pub favorites: Vec, -} - -#[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, - - pub user: UserDetailsLight, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug)] -pub enum ListFavoritesError { - ServerError, -} - -impl ResponseError for ListFavoritesError { - fn status_code(&self) -> StatusCode { - match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, - } - } - - fn error_response(&self) -> HttpResponse { - let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - }; - - to_simple_json_error(&error_reason, self.status_code()) - } -} - -// NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -pub async fn list_favorites_for_entity_handler( - _http_request: HttpRequest, - path: Path, - server_state: web::Data> -) -> Result -{ - let entity_token = FavoriteEntityToken::from_entity_type_and_token( - path.entity_type, &path.entity_token); - - let query_results = list_favorites_for_entity( - entity_token, - &server_state.mysql_pool, - ).await; - - let favorites = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); - } - }; - - let response = ListFavoritesSuccessResponse { - success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - user: UserDetailsLight { - user_token: favorite.user_token.clone(), - username: favorite.username.to_string(), // NB: Cloned because of ref use for avatar below - display_name: favorite.user_display_name.clone(), - gravatar_hash: favorite.user_gravatar_hash.clone(), - default_avatar: DefaultAvatarInfo::from_username(&favorite.username), - }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - }) - .collect(), - }; - - let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs deleted file mode 100644 index 4529d2f18b..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod create_favorite_handler; -pub mod delete_favorite_handler; -pub mod list_favorites_for_entity_handler; -pub mod list_favorites_for_session_handler; -pub mod list_favorites_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index d4c32ebd1d..0a6ed9375d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -25,5 +25,5 @@ pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; pub mod w2l; -pub mod favorites; +pub mod user_bookmarks; pub mod weights; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs similarity index 52% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs index cb4aa5f5f4..b62fe2feeb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs @@ -11,46 +11,46 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::create_favorite::{create_favorite, CreateFavoriteArgs}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct CreateFavoriteRequest { +pub struct CreateUserBookmarkRequest { entity_token: String, - entity_type: FavoriteEntityType, + entity_type: UserBookmarkEntityType, } #[derive(Serialize)] -pub struct CreateFavoriteSuccessResponse { +pub struct CreateUserBookmarkSuccessResponse { pub success: bool, - pub favorite_token: FavoriteToken, + pub user_bookmark_token: UserBookmarkToken, } #[derive(Debug)] -pub enum CreateFavoriteError { +pub enum CreateUserBookmarkError { BadInput(String), NotAuthorized, ServerError, } -impl ResponseError for CreateFavoriteError { +impl ResponseError for CreateUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - CreateFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - CreateFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - CreateFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + CreateUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - CreateFavoriteError::BadInput(reason) => reason.to_string(), - CreateFavoriteError::NotAuthorized => "unauthorized".to_string(), - CreateFavoriteError::ServerError => "server error".to_string(), + CreateUserBookmarkError::BadInput(reason) => reason.to_string(), + CreateUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + CreateUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -58,24 +58,24 @@ impl ResponseError for CreateFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for CreateFavoriteError { +impl fmt::Display for CreateUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn create_favorite_handler( +pub async fn create_user_bookmark_handler( http_request: HttpRequest, - request: web::Json, + request: web::Json, server_state: web::Data>, -) -> Result +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -84,42 +84,42 @@ pub async fn create_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { warn!("not logged in"); - return Err(CreateFavoriteError::NotAuthorized); + return Err(CreateUserBookmarkError::NotAuthorized); } }; - let entity_token = FavoriteEntityToken::from_entity_type_and_token( + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( request.entity_type, &request.entity_token); - let query_result = create_favorite(CreateFavoriteArgs { + let query_result = create_user_bookmark(CreateUserBookmarkArgs { entity_token: &entity_token, user_token: &user_session.user_token_typed, mysql_executor: &mut *mysql_connection, phantom: Default::default(), }).await; - let favorite_token = match query_result { + let user_bookmark_token = match query_result { Ok(token) => token, Err(err) => { - warn!("error inserting favorite: {:?}", err); - return Err(CreateFavoriteError::ServerError); + warn!("error inserting user_bookmark: {:?}", err); + return Err(CreateUserBookmarkError::ServerError); } }; - let response = CreateFavoriteSuccessResponse { + let response = CreateUserBookmarkSuccessResponse { success: true, - favorite_token, + user_bookmark_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| CreateFavoriteError::ServerError)?; + .map_err(|_e| CreateUserBookmarkError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs similarity index 58% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs index be28d24534..d68fc29e75 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs @@ -12,8 +12,8 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use log::warn; -use mysql_queries::queries::favorites::delete_favorite::delete_favorite; -use tokens::tokens::favorites::FavoriteToken; +use mysql_queries::queries::user_bookmarks::delete_user_bookmark::delete_user_bookmark; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -21,40 +21,40 @@ use crate::server_state::ServerState; /// For the URL PathInfo #[derive(Deserialize)] -pub struct DeleteFavoritePathInfo { - favorite_token: FavoriteToken, +pub struct DeleteUserBookmarkPathInfo { + user_bookmark_token: UserBookmarkToken, } #[derive(Deserialize)] -pub struct DeleteFavoriteRequest { +pub struct DeleteUserBookmarkRequest { /// NB: this is only to disambiguate when a user is both a mod and an author. as_mod: Option, } #[derive(Debug)] -pub enum DeleteFavoriteError { +pub enum DeleteUserBookmarkError { BadInput(String), NotAuthorized, NotFound, ServerError, } -impl ResponseError for DeleteFavoriteError { +impl ResponseError for DeleteUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - DeleteFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - DeleteFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - DeleteFavoriteError::NotFound => StatusCode::NOT_FOUND, - DeleteFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + DeleteUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteUserBookmarkError::NotFound => StatusCode::NOT_FOUND, + DeleteUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - DeleteFavoriteError::BadInput(reason) => reason.to_string(), - DeleteFavoriteError::NotAuthorized => "unauthorized".to_string(), - DeleteFavoriteError::NotFound => "not found".to_string(), - DeleteFavoriteError::ServerError => "server error".to_string(), + DeleteUserBookmarkError::BadInput(reason) => reason.to_string(), + DeleteUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + DeleteUserBookmarkError::NotFound => "not found".to_string(), + DeleteUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -62,24 +62,24 @@ impl ResponseError for DeleteFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for DeleteFavoriteError { +impl fmt::Display for DeleteUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn delete_favorite_handler( +pub async fn delete_user_bookmark_handler( http_request: HttpRequest, - path: Path, - _request: web::Json, + path: Path, + _request: web::Json, server_state: web::Data> -) -> Result { +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -88,18 +88,18 @@ pub async fn delete_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { - return Err(DeleteFavoriteError::NotAuthorized); + return Err(DeleteUserBookmarkError::NotAuthorized); } }; - let query_result = delete_favorite( - &path.favorite_token, + let query_result = delete_user_bookmark( + &path.user_bookmark_token, &user_session.user_token_typed, &mut *mysql_connection ).await; @@ -108,7 +108,7 @@ pub async fn delete_favorite_handler( Ok(_) => {}, Err(err) => { warn!("Update tts mod approval status DB error: {:?}", err); - return Err(DeleteFavoriteError::ServerError); + return Err(DeleteUserBookmarkError::ServerError); } }; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs new file mode 100644 index 0000000000..0fa28887fd --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs @@ -0,0 +1,122 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks_for_entity::list_user_bookmarks_for_entity; +use tokens::tokens::user_bookmarks::UserBookmarkToken; + +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct ListUserBookmarkPathInfo { + entity_type: UserBookmarkEntityType, + entity_token: String, +} + +#[derive(Serialize)] +pub struct ListUserBookmarksSuccessResponse { + pub success: bool, + pub user_bookmarks: Vec, +} + +#[derive(Serialize)] +pub struct UserBookmark { + pub token: UserBookmarkToken, + + pub user: UserDetailsLight, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ListUserBookmarksError { + ServerError, +} + +impl ResponseError for ListUserBookmarksError { + fn status_code(&self) -> StatusCode { + match *self { + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListUserBookmarksError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListUserBookmarksError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_user_bookmarks_for_entity_handler( + _http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( + path.entity_type, &path.entity_token); + + let query_results = list_user_bookmarks_for_entity( + entity_token, + &server_state.mysql_pool, + ).await; + + let user_bookmarks = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListUserBookmarksError::ServerError); + } + }; + + let response = ListUserBookmarksSuccessResponse { + success: true, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + user: UserDetailsLight { + user_token: user_bookmark.user_token.clone(), + username: user_bookmark.username.to_string(), // NB: Cloned because of ref use for avatar below + display_name: user_bookmark.user_display_name.clone(), + gravatar_hash: user_bookmark.user_gravatar_hash.clone(), + default_avatar: DefaultAvatarInfo::from_username(&user_bookmark.username), + }, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + }) + .collect(), + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| ListUserBookmarksError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs index cdc11f8550..dc1f17d90c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs @@ -13,38 +13,38 @@ use actix_web::web::Query; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -55,23 +55,23 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, NotAuthorizedError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - ListFavoritesError::NotAuthorizedError => StatusCode::UNAUTHORIZED, + ListUserBookmarksError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::NotAuthorizedError => StatusCode::UNAUTHORIZED, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - ListFavoritesError::NotAuthorizedError => "not authorized".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), + ListUserBookmarksError::NotAuthorizedError => "not authorized".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -79,61 +79,61 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_session_handler( +pub async fn list_user_bookmarks_for_session_handler( http_request: HttpRequest, - query: Query, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let user_session = server_state.session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); - ListFavoritesError::ServerError + ListUserBookmarksError::ServerError })? - .ok_or(ListFavoritesError::NotAuthorizedError)?; + .ok_or(ListUserBookmarksError::NotAuthorizedError)?; let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&user_session.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&user_session.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, maybe_summary_text: None, maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs index 071b1fcb0e..ccaf8b61f4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs @@ -13,43 +13,43 @@ use actix_web::web::{Path, Query}; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesPathInfo { +pub struct ListUserBookmarksPathInfo { username: String, } #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -60,20 +60,20 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -81,56 +81,56 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_user_handler( +pub async fn list_user_bookmarks_for_user_handler( _http_request: HttpRequest, - path: Path, - query: Query, + path: Path, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&path.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&path.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - maybe_summary_text: favorite.maybe_entity_descriptive_text, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + maybe_summary_text: user_bookmark.maybe_entity_descriptive_text, // TODO(bt,2023-11-21): Thumbnails need proper support. We should build them as a // first-class system before handling the backfill here. maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs new file mode 100644 index 0000000000..11881f5bb3 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs @@ -0,0 +1,5 @@ +pub mod create_user_bookmark_handler; +pub mod delete_user_bookmark_handler; +pub mod list_user_bookmarks_for_entity_handler; +pub mod list_user_bookmarks_for_session_handler; +pub mod list_user_bookmarks_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index eb67880dd5..9dcdce3e35 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -28,11 +28,11 @@ use crate::http_server::endpoints::comments::list_comments_handler::list_comment use crate::http_server::endpoints::download_job::enqueue_generic_download::enqueue_generic_download_handler; use crate::http_server::endpoints::download_job::get_generic_upload_job_status::get_generic_download_job_status_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; -use crate::http_server::endpoints::favorites::create_favorite_handler::create_favorite_handler; -use crate::http_server::endpoints::favorites::delete_favorite_handler::delete_favorite_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_entity_handler::list_favorites_for_entity_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_session_handler::list_favorites_for_session_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_user_handler::list_favorites_for_user_handler; +use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler; use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; @@ -202,14 +202,14 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> .add_post("/v1/comments/delete/{comment_token}", delete_comment_handler) .into_app(); - // ==================== Favorites ==================== + // ==================== User Bookmarks ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/favorites/create", create_favorite_handler) - .add_post("/v1/favorites/delete/{favorite_token}", delete_favorite_handler) - .add_get("/v1/favorites/list/session", list_favorites_for_session_handler) - .add_get("/v1/favorites/list/user/{username}", list_favorites_for_user_handler) - .add_get("/v1/favorites/list/entity/{entity_type}/{entity_token}", list_favorites_for_entity_handler) + .add_post("/v1/user_bookmarks/create", create_user_bookmark_handler) + .add_post("/v1/user_bookmarks/delete/{user_bookmark_token}", delete_user_bookmark_handler) + .add_get("/v1/user_bookmarks/list/session", list_user_bookmarks_for_session_handler) + .add_get("/v1/user_bookmarks/list/user/{username}", list_user_bookmarks_for_user_handler) + .add_get("/v1/user_bookmarks/list/entity/{entity_type}/{entity_token}", list_user_bookmarks_for_entity_handler) .into_app(); // ==================== Animations ==================== From e78cdf85e30ead4cca5b6cbe92404493679f1a71 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 11 Dec 2023 14:40:14 -0500 Subject: [PATCH 103/136] Upload weights front end side done, need to write the download code then run inference depending on the model. --- crates/lib/deprecated/config/src/bad_urls.rs | 5 + .../drain_multipart_weights_request.rs | 100 ----- .../endpoints/weights/upload_weight.rs | 385 +++++++----------- 3 files changed, 158 insertions(+), 332 deletions(-) delete mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs diff --git a/crates/lib/deprecated/config/src/bad_urls.rs b/crates/lib/deprecated/config/src/bad_urls.rs index 13be46019b..280ed1d3bb 100644 --- a/crates/lib/deprecated/config/src/bad_urls.rs +++ b/crates/lib/deprecated/config/src/bad_urls.rs @@ -3,11 +3,16 @@ use crate::is_bad_download_url::is_bad_download_url; /// Reports if the URL is inappropriate for downloading as a TTS model, vocoder, etc. +/// DEPRECATE: pub fn is_bad_tts_model_download_url(url: &str) -> anyhow::Result { // TODO: Just use this. is_bad_download_url(url) } +pub fn is_bad_model_weights_download_url(url: &str) -> anyhow::Result { + is_bad_download_url(url) +} + #[cfg(test)] mod tests { use crate::bad_urls::is_bad_tts_model_download_url; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs deleted file mode 100644 index c87a9d82c5..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/drain_multipart_weights_request.rs +++ /dev/null @@ -1,100 +0,0 @@ -use actix_multipart::Multipart; -use actix_web::web::BytesMut; -use futures::TryStreamExt; -use log::warn; - -use errors::AnyhowResult; -use tokens::tokens::weights::ModelWeightToken; - -use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; - -pub struct WeightUploadRequest { - pub uuid_idempotency_token: Option, - pub model_weights_token: Option, - pub file_name: Option, - pub file_bytes: Option, - pub media_source: MediaSource, -} - -/// Where the frontend tells us the file came from. -#[derive(Eq, PartialEq, Clone, Copy)] -pub enum MediaSource { - Unknown, - UserFile, - /// Eg. the web audio API in Javascript. - UserDeviceApi, -} - -/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. -pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { - let mut uuid_idempotency_token = None; - let mut model_weight_token = None; - let mut file_bytes = None; - let mut file_name = None; - let mut media_source = None; - - while let Ok(Some(mut field)) = multipart_payload.try_next().await { - let mut field_name = None; - let mut field_filename = None; - - if let content_disposition = field.content_disposition() { - field_name = content_disposition.get_name() - .map(|s| s.to_string()); - field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. - .map(|s| s.to_string()); - } - - match field_name.as_deref() { - Some("uuid_idempotency_token") => { - uuid_idempotency_token = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading idempotency token: {:}", e); - e - })?; - }, - Some("dataset_token") => { - model_weight_token = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading dataset token: {:}", e); - e - })?; - }, - Some("file") => { - file_name = field_filename.clone(); - file_bytes = checked_read_multipart_bytes(&mut field).await - .map_err(|e| { - warn!("Error reading audio upload: {:}", e); - e - })?; - }, - Some("source") => { - media_source = read_multipart_field_as_text(&mut field).await - .map_err(|e| { - warn!("Error reading source: {:}", e); - e - })?; - }, - _ => continue, - } - } - - let media_source = match media_source.as_deref() { - Some("device") => MediaSource::UserDeviceApi, - Some("file") => MediaSource::UserFile, - _ => { - if file_name.as_deref() == Some("blob") { - MediaSource::UserDeviceApi - } else { - MediaSource::Unknown - } - }, - }; - - Ok(UploadSampleRequest { - uuid_idempotency_token, - dataset_token, - file_name, - file_bytes, - media_source, - }) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs index 2d66aa7684..8d2aa5fc5c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs @@ -2,10 +2,10 @@ use std::collections::HashSet; use std::sync::Arc; use actix_multipart::Multipart; -use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::{ HttpRequest, HttpResponse, web }; use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType; -use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use mysql_queries::queries::model_weights::create_weight::{ self, create_weight }; use once_cell::sync::Lazy; use tokens::tokens::model_weights::ModelWeightToken; @@ -13,259 +13,180 @@ use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; -use log::{error, info, warn}; +use log::{ error, info, warn }; use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; -use enums::common::visibility::Visibility; + use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; use http_server_common::request::get_request_ip::get_request_ip; use media::decode_basic_audio_info::decode_basic_audio_bytes_info; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; -use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; +use mysql_queries::queries::media_uploads::insert_media_upload::{ Args, insert_media_upload }; -use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{ + drain_multipart_request, + MediaSource, +}; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; use utoipa::ToSchema; +use config::bad_urls::is_bad_model_weights_download_url; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::common::visibility::Visibility; +use utoipa::ToSchema; + +#[derive(Deserialize, ToSchema)] +pub struct UploadModelWeightsRequest { + idempotency_token: String, + title: String, + download_url: String, + model_weights_type: WeightsType, + model_weights_category: WeightsCategory, + creator_set_visibility: Option, +} + +/// Tell the frontend how to deal with the download queue. +#[derive(Serialize, ToSchema)] +pub enum DownloadJobType { + /// Modern shared download type (everything will use this in the + /// future, and this endpoint will die.) + #[serde(rename = "generic")] + Generic, +} + +#[derive(Serialize, ToSchema)] +pub struct UploadModelWeightsSuccessResponse { + pub success: bool, + /// This is how frontend clients can request the job execution status. + pub job_token: String, + /// This is a transitional field to tell the frontend how to process the job checking. + pub job_type: DownloadJobType, +} + +#[derive(Debug)] +pub enum UploadModelWeightsError { + BadInput(String), + MustBeLoggedIn, + ServerError, + RateLimited, +} + +impl ResponseError for UploadModelWeightsError { + fn status_code(&self) -> StatusCode { + match *self { + UploadModelWeightsError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadModelWeightsError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadModelWeightsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadModelWeightsError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + UploadModelWeightsError::BadInput(reason) => reason.to_string(), + UploadModelWeightsError::MustBeLoggedIn => "user must be logged in".to_string(), + UploadModelWeightsError::ServerError => "server error".to_string(), + UploadModelWeightsError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + pub async fn upload_weights_handler( http_request: HttpRequest, server_state: web::Data>, - mut multipart_payload: Multipart, - ) -> Result { - - let response = handle_weight_upload( - &http_request, - &server_state, - multipart_payload, - &ALLOWED_MIME_TYPES).await?; - - // send a body with the required payload along side the multipart upload - - let weight_upload_token = response.to_model_token(); - - let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { - success: true, - upload_token: weight_upload_token, - }; - - let body = serde_json::to_string(&response) - .map_err(|e| UploadError::ServerError)?; - - return Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)); - } - - pub enum SuccessCase { - WeightAlreadyUploaded { - existing_upload_token: ModelWeightToken, - }, - WeightSuccessfullyUploaded { - upload_token: ModelWeightToken, + mut multipart_payload: Multipart +) -> Result { + // Rate limiter check + if + let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request( + &http_request + ) + { + return Err(UploadModelWeightsError::RateLimited); } - } - - impl SuccessCase { - pub fn to_model_token(self) -> ModelWeightToken { - match self { - SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, - SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, - } - } - } - - pub async fn handle_weight_upload( - http_request: &HttpRequest, - server_state: &web::Data>, - mut multipart_payload: Multipart, - allowed_mimetypes: &HashSet<&'static str>, - ) -> Result { - - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - error!("MySql pool error: {:?}", err); - UploadError::ServerError - })?; - - // ==================== READ SESSION ==================== // - - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) - .await + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await .map_err(|e| { - error!("Session checker error: {:?}", e); - UploadError::ServerError + warn!("Session checker error: {:?}", e); + UploadModelWeightsError::ServerError })?; - - // ==================== RATE LIMIT ==================== // - - let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UploadModelWeightsError::MustBeLoggedIn); } - &server_state.redis_rate_limiters.logged_in - }, }; - - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); + + if let Err(reason) = validate_idempotency_token(&request.idempotency_token) { + return Err(UploadModelWeightsError::BadInput(reason)); } - - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); + + if let Err(reason) = validate_model_title(&request.title) { + return Err(UploadModelWeightsError::BadInput(reason)); } - - // ==================== READ MULTIPART REQUEST ==================== // - - let upload_weights_request = drain_multipart_request(multipart_payload) - .await - .map_err(|e| { - // TODO: Error handling could be nicer. - UploadError::BadInput("bad request".to_string()) - })?; - - let uuid_idempotency_token = upload_weights_request.uuid_idempotency_token - .ok_or(UploadError::BadInput("no uuid".to_string()))?; - - let maybe_existing_upload = - get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) - .await; - - match maybe_existing_upload { - Err(err) => { - error!("Error checking for previous upload: {:?}", err); - return Err(UploadError::ServerError); - } - Ok(Some(upload)) => { - // File already uploaded and frontend didn't protect us. - return Ok(SuccessCase::WeightAlreadyUploaded { - existing_upload_token: upload.token, - }); - } - Ok(None) => { - // Proceed. - } + + match is_bad_model_weights_download_url(&download_url) { + Ok(false) => {} // Ok case + Ok(true) => { + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } + Err(err) => { + warn!("Error parsing url: {:?}", err); + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } } - - if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { - return Err(UploadError::BadInput(reason)); - } - - let creator_set_visibility = maybe_user_session - .as_ref() - .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. - .unwrap_or(Visibility::default()); - - let ip_address = get_request_ip(&http_request); - - let maybe_user_token = maybe_user_session - .map(|session| session.get_strongly_typed_user_token()); - - let maybe_file_size_bytes = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| bytes.len()); - - info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); - - let maybe_mimetype = upload_weights_request.file_bytes - .as_ref() - .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) - .flatten(); - - - let bytes = match upload_weights_request.file_bytes { - None => return Err(UploadError::BadInput("missing file contents".to_string())), - Some(bytes) => bytes, + + let supported_model_weights_type = request.model_weights_type; + + let job_token; + let download_job_type; + + let (download_job_token, _record_id) = insert_generic_download_job( + InsertGenericDownloadJobArgs { + uuid_idempotency_token: &uuid, + download_type: GenericDownloadType::Vits, + download_url: &download_url, + title: &title, + creator_user_token: &user_session.user_token, + creator_ip_address: &ip_address, + creator_set_visibility: visibility, // TODO: Creator set preference + mysql_pool: &server_state.mysql_pool, + } + ).await.map_err(|err| { + warn!("New generic download creation DB error (for generic model): {:?}", err); + UploadModelWeightsError::ServerError + })?; + + job_token = download_job_token.to_string(); + download_job_type = DownloadJobType::Generic; + + let response = UploadModelWeightsSuccessResponse { + success: true, + job_token: job_token.to_string(), + job_type: download_job_type, }; - - let hash = sha256_hash_bytes(&bytes) - .map_err(|io_error| { - error!("Problem hashing bytes: {:?}", io_error); - UploadError::ServerError - })?; - - let file_size_bytes = bytes.len(); - - let mut maybe_duration_millis = None; - let mut maybe_codec_name = None; - let mut media_upload_type = None; - - if let Some(mimetype) = maybe_mimetype.as_deref() { - - if !allowed_mimetypes.contains(mimetype) { - // NB: Don't let our error message inject malicious strings - let filtered_mimetype = mimetype - .chars() - .filter(|c| c.is_ascii()) - .filter(|c| c.is_alphanumeric() || *c == '/') - .collect::(); - return Err(UploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); - } - media_upload_type = match mimetype { - "binary/octet-stream" /* .bin */ => Some(MediaUploadType::Binary), - _ => None, - }; + + let body = serde_json::to_string(&response).map_err(|_e| UploadModelWeightsError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} + +fn validate_idempotency_token(token: &str) -> Result<(), String> { + if token.len() != 36 { + return Err("idempotency token should be 36 characters".to_string()); } - - // This is fine for now we just don't expose the path! - let public_upload_path = WeightUploadOriginalFilePath::generate_new(); - - info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); - - // need to upload based on visibility - server_state.public_bucket_client.upload_file_with_content_type( - public_upload_path.get_full_object_path_str(), - bytes.as_ref(), - mime_type) - .await - .map_err(|e| { - warn!("Upload media bytes to bucket error: {:?}", e); - UploadError::ServerError - })?; - - let token = create_weight(CreateModelWeightsArgs { - token: &ModelWeightToken::generate(), - weights_type: WeightsType::LoRA, // to do parse multi part for this - weights_category: WeightsCategory::ImageGeneration, - title: upload_weights_request.file_name.as_deref(), - maybe_thumbnail_token: &ThumbnailToken::generate(), - description_markdown: "", - description_rendered_html: "", - creator_user_token: maybe_user_token.as_str(), - creator_ip_address: &ip_address, - creator_set_visibility, - maybe_last_update_user_token: "", - original_download_url: "", - original_filename: upload_weights_request.file_name.as_deref(), - file_size_bytes: file_size_bytes as i32, - file_checksum_sha2: hash, - private_bucket_hash: private_bucket_hash, - maybe_private_bucket_prefix: None, - maybe_private_bucket_extension: None, - cached_user_ratings_total_count: 0, - cached_user_ratings_positive_count: 0, - cached_user_ratings_negative_count: 0, - maybe_cached_user_ratings_ratio: None, - version: 0, - mysql_pool: &server_state.mysql_pool, - }).await - .map_err(|e| { - error!("Error creating weights upload: {:?}", e); - UploadError::ServerError - })?; - - info!("new weights upload id: {} token: {:?}", record_id, &token); - - Ok(SuccessCase::WeightSuccessfullyUploaded { - upload_token: token, - }) - } - \ No newline at end of file + + Ok(()) +} From c09adc3729ea697a032db6172b330e8d78c3fa8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:08:50 +0000 Subject: [PATCH 104/136] Bump hyper-tls from 0.5.0 to 0.6.0 Bumps [hyper-tls](https://github.com/hyperium/hyper-tls) from 0.5.0 to 0.6.0. - [Commits](https://github.com/hyperium/hyper-tls/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: hyper-tls dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 172 ++++++++++++++---- .../newrelic-telemetry-sdk-rust/Cargo.toml | 2 +- 2 files changed, 141 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1aaf8d3a68..9182c02682 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http", + "http 0.2.11", "httparse", "httpdate", "itoa 1.0.9", @@ -176,7 +176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "http", + "http 0.2.11", "regex", "serde", "tracing", @@ -614,8 +614,8 @@ dependencies = [ "hex", "hmac 0.12.1", "http-types", - "hyper", - "hyper-tls", + "hyper 0.14.27", + "hyper-tls 0.5.0", "serde", "serde_json", "serde_path_to_error", @@ -676,7 +676,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ - "http", + "http 0.2.11", "log", "native-tls", "serde", @@ -1961,7 +1961,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "mimetypes", @@ -2151,7 +2151,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "images", "jobs_common", "lfu", @@ -2636,7 +2636,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap", "slab", "tokio 1.34.0", @@ -2796,6 +2796,17 @@ dependencies = [ "itoa 1.0.9", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa 1.0.9", +] + [[package]] name = "http-body" version = "0.4.5" @@ -2803,7 +2814,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes 1.5.0", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes 1.5.0", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2823,7 +2857,7 @@ dependencies = [ "async-channel", "base64 0.13.1", "futures-lite", - "http", + "http 0.2.11", "infer 0.2.3", "pin-project-lite", "rand 0.7.3", @@ -2887,8 +2921,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa 1.0.9", @@ -2900,6 +2934,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "pin-project-lite", + "tokio 1.34.0", + "want", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2907,12 +2957,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.5.0", - "hyper", + "hyper 0.14.27", "native-tls", "tokio 1.34.0", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes 1.5.0", + "http-body-util", + "hyper 1.0.1", + "hyper-util", + "native-tls", + "tokio 1.34.0", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.0.1", + "pin-project-lite", + "socket2 0.5.5", + "tokio 1.34.0", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -3057,7 +3143,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "images", "jobs_common", "lfu", @@ -3758,7 +3844,7 @@ dependencies = [ "hashbrown 0.14.2", "heck", "hmac 0.12.1", - "hyper", + "hyper 0.14.27", "indexmap", "lazy_static", "libc", @@ -3897,8 +3983,8 @@ dependencies = [ "env_logger", "flate2", "futures 0.3.29", - "hyper", - "hyper-tls", + "hyper 0.14.27", + "hyper-tls 0.6.0", "log", "my-workspace-hack", "rand 0.8.5", @@ -4905,10 +4991,10 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", - "hyper-tls", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -5066,7 +5152,7 @@ dependencies = [ "futures 0.3.29", "hex", "hmac 0.12.1", - "http", + "http 0.2.11", "log", "maybe-async", "md5", @@ -6027,7 +6113,7 @@ dependencies = [ "hmac 0.10.1", "hostname", "http_server_common", - "hyper", + "hyper 0.14.27", "infer 0.5.0", "jwt", "language-tags", @@ -6867,6 +6953,28 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio 1.34.0", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -6927,7 +7035,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -6958,7 +7066,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "lfu", "log", @@ -7003,7 +7111,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.11", "httparse", "log", "rand 0.8.5", @@ -7033,7 +7141,7 @@ dependencies = [ "futures 0.3.29", "futures-util", "hostname", - "hyper", + "hyper 0.14.27", "lazy_static", "log", "magic-crypt", @@ -7075,7 +7183,7 @@ dependencies = [ "chrono", "displaydoc", "futures 0.3.29", - "http", + "http 0.2.11", "once_cell", "reqwest", "serde", @@ -7115,7 +7223,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "displaydoc", - "http", + "http 0.2.11", "once_cell", "rand 0.8.5", "reqwest", @@ -7437,7 +7545,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -7466,7 +7574,7 @@ dependencies = [ "filesys", "google_drive_common", "hostname", - "hyper", + "hyper 0.14.27", "jobs_common", "log", "my-workspace-hack", @@ -7686,7 +7794,7 @@ dependencies = [ "futures-util", "hostname", "http_server_common", - "hyper", + "hyper 0.14.27", "lazy_static", "limitation", "log", diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 088a3cf568..cf1336f3f3 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0.57", features = ["derive"] } serde_json = "1.0.108" hyper = { version = "0.14", features = ["http1"], optional = true } -hyper-tls = { version = "0.5.0", optional = true } +hyper-tls = { version = "0.6.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } tokio = { version = "1.1.0", features = ["macros"], optional = true } From d3be9a3a6b8d258d48557b1b4198e52634aedf9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:09:11 +0000 Subject: [PATCH 105/136] Bump tokio from 0.1.22 to 1.34.0 Bumps [tokio](https://github.com/tokio-rs/tokio) from 0.1.22 to 1.34.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-0.1.22...tokio-1.34.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Cargo.lock | 233 ++++-------------- crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_upload_media_file/Cargo.toml | 2 +- crates/cli/elasticsearch-cli/Cargo.toml | 2 +- crates/cli/scrape_test/Cargo.toml | 2 +- .../lib/api_clients/cloud_storage/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- .../service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- .../service/job/tts_download_job/Cargo.toml | 2 +- .../service/job/tts_inference_job/Cargo.toml | 2 +- .../service/job/w2l_download_job/Cargo.toml | 2 +- .../service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../service/web/websocket_gateway/Cargo.toml | 2 +- crates/vendor/limitation/Cargo.toml | 2 +- .../newrelic-telemetry-sdk-rust/Cargo.toml | 4 +- 22 files changed, 71 insertions(+), 206 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1aaf8d3a68..67129df838 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -39,7 +39,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -115,7 +115,7 @@ dependencies = [ "rand 0.8.5", "sha1 0.10.6", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", "zstd 0.12.4", @@ -153,7 +153,7 @@ dependencies = [ "serde_json", "serde_plain", "tempfile", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -190,7 +190,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "actix-macros", "futures-core", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -206,7 +206,7 @@ dependencies = [ "futures-util", "mio 0.8.9", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -285,7 +285,7 @@ dependencies = [ "bytestring", "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -465,7 +465,7 @@ dependencies = [ "serde_json", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -600,7 +600,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -624,7 +624,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -810,7 +810,7 @@ dependencies = [ "serde_derive", "serde_json", "sqlx", - "tokio 1.34.0", + "tokio", "toml", "url_config", ] @@ -1162,7 +1162,7 @@ dependencies = [ "log", "my-workspace-hack", "rust-s3", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1218,7 +1218,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -1386,17 +1386,6 @@ dependencies = [ "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1404,25 +1393,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.15", + "crossbeam-epoch", "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -1436,17 +1410,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1791,7 +1754,7 @@ dependencies = [ "sqlx", "storyteller_root", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1812,7 +1775,7 @@ dependencies = [ "mysql_queries", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1976,7 +1939,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", - "tokio 1.34.0", + "tokio", "zip", ] @@ -2022,7 +1985,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2099,7 +2062,7 @@ dependencies = [ "storyteller_root", "strum", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2174,7 +2137,7 @@ dependencies = [ "tempdir", "testing", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -2639,7 +2602,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -2894,7 +2857,7 @@ dependencies = [ "itoa 1.0.9", "pin-project-lite", "socket2 0.4.10", - "tokio 1.34.0", + "tokio", "tower-service", "tracing", "want", @@ -2909,7 +2872,7 @@ dependencies = [ "bytes 1.5.0", "hyper", "native-tls", - "tokio 1.34.0", + "tokio", "tokio-native-tls", ] @@ -3079,7 +3042,7 @@ dependencies = [ "subprocess_common", "tempdir", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -3288,7 +3251,7 @@ dependencies = [ "once_cell", "quoted_printable", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "url 2.5.0", ] @@ -3353,7 +3316,7 @@ dependencies = [ "my-workspace-hack", "redis 0.13.0", "time 0.1.45", - "tokio 0.1.22", + "tokio", "version-sync", ] @@ -3556,15 +3519,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.7.1" @@ -3797,7 +3751,7 @@ dependencies = [ "syn 2.0.39", "time 0.3.30", "time-macros", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", "twitch_oauth2", @@ -3834,7 +3788,7 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", "toml", ] @@ -3904,7 +3858,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "tokio 1.34.0", + "tokio", "uuid 0.8.2", ] @@ -4733,7 +4687,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-deque 0.8.3", + "crossbeam-deque", "crossbeam-utils 0.8.16", ] @@ -4798,7 +4752,7 @@ dependencies = [ "pin-project-lite", "ryu", "sha1 0.6.1", - "tokio 1.34.0", + "tokio", "tokio-util", "url 2.5.0", ] @@ -4921,7 +4875,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "system-configuration", - "tokio 1.34.0", + "tokio", "tokio-native-tls", "tokio-util", "tower-service", @@ -5079,7 +5033,7 @@ dependencies = [ "sha2 0.10.8", "thiserror", "time 0.3.30", - "tokio 1.34.0", + "tokio", "tokio-stream", "url 2.5.0", ] @@ -5251,7 +5205,7 @@ dependencies = [ "clap 2.34.0", "errors", "my-workspace-hack", - "tokio 1.34.0", + "tokio", "web_scrapers", ] @@ -5767,7 +5721,7 @@ dependencies = [ "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue 0.3.8", + "crossbeam-queue", "dotenvy", "either", "event-listener", @@ -5793,7 +5747,7 @@ dependencies = [ "smallvec 1.11.2", "sqlformat", "thiserror", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", "url 2.5.0", @@ -5836,7 +5790,7 @@ dependencies = [ "sqlx-sqlite", "syn 1.0.109", "tempfile", - "tokio 1.34.0", + "tokio", "url 2.5.0", ] @@ -6066,7 +6020,7 @@ dependencies = [ "symphonia", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "toml", "tts_common", "twitch_common", @@ -6583,30 +6537,6 @@ dependencies = [ "utoipa", ] -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.34.0" @@ -6637,16 +6567,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - [[package]] name = "tokio-executor" version = "0.1.10" @@ -6657,17 +6577,6 @@ dependencies = [ "futures 0.1.31", ] -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - [[package]] name = "tokio-io" version = "0.1.13" @@ -6697,7 +6606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6726,7 +6635,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "webpki", ] @@ -6738,7 +6647,7 @@ checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6765,35 +6674,6 @@ dependencies = [ "tokio-reactor", ] -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue 0.2.3", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - [[package]] name = "tokio-tungstenite" version = "0.15.0" @@ -6804,28 +6684,13 @@ dependencies = [ "log", "pin-project", "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "tokio-rustls", "tungstenite", "webpki", "webpki-roots 0.21.1", ] -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - [[package]] name = "tokio-uds" version = "0.2.7" @@ -6854,7 +6719,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -6939,7 +6804,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6973,7 +6838,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -7054,7 +6919,7 @@ dependencies = [ "sqlx", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "tokio-tungstenite", "toml", "tts_common", @@ -7449,7 +7314,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7478,7 +7343,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7707,7 +7572,7 @@ dependencies = [ "sha2 0.9.9", "sqlx", "time 0.3.30", - "tokio 1.34.0", + "tokio", "toml", "twitch_api2", "twitch_common", diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index ae63bd60b6..a771241cf5 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -37,7 +37,7 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index ff572d4512..47d58a89c0 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -34,7 +34,7 @@ log = "0.4.14" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index c8f0841c69..7361ad74b6 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -34,7 +34,7 @@ strum = { version = "0.25", features = ["derive"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index d286897c20..36284d1f33 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -23,4 +23,4 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index 3d4c2d3f77..2cd5d20198 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -30,4 +30,4 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 335aab10f3..228c472322 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -43,7 +43,7 @@ serde_json = "1.0.108" toml = "0.5.8" serial_test = "2.0.0" -tokio = { version = "1.6.0", features = ["full"] } +tokio = { version = "1.34.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 59f30971d9..ed1487b8a3 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -39,7 +39,7 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Serialization serde = "1.0.125" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 96598161bc..e97068915d 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -53,7 +53,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index de0c02e9d6..f72b7f7fba 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -66,7 +66,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 44b1bb910a..9f8205ca14 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -62,7 +62,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 7506d7f670..07bfd1b465 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -45,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 9fbdc2df76..cbfdd24de1 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -51,7 +51,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index cfdafda722..7819055ed0 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -45,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 34db170d30..ac801c96bd 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -43,7 +43,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 1fda2a5a73..58a1486caf 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -43,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 4e123524e9..869eee7790 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -59,4 +59,4 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", mockall = "0.11.3" # NB: Tokio version matches `async-stripe` transitive dep; features are for testing -tokio = { version = "1.13.0", features = ["rt", "macros"] } +tokio = { version = "1.34.0", features = ["rt", "macros"] } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 671723c73b..4ea1672720 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -32,7 +32,7 @@ hostname = "0.3.1" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures-util = "0.3.29" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index c57dddc4db..c8d81cef9a 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -95,7 +95,7 @@ native-tls = { version = "0.2.8", features = ["vendored"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index f492098015..20c89b006b 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -43,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 7c4015b643..51f1626635 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -27,5 +27,5 @@ time = "0.1.42" my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] -tokio = "0.1.22" +tokio = "1.34.0" version-sync = "0.8.1" diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 088a3cf568..df26faf565 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -19,7 +19,7 @@ hyper = { version = "0.14", features = ["http1"], optional = true } hyper-tls = { version = "0.5.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } -tokio = { version = "1.1.0", features = ["macros"], optional = true } +tokio = { version = "1.34.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } # Workspace hack (faster compile times) @@ -30,4 +30,4 @@ rand = "0.8.5" env_logger = "0.10.1" futures = { version = "0.3.1" } hyper = { version = "0.14", features = ["http1", "server"] } -tokio = { version = "1.1.0", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } From 1cde23c08087ceecedfa4b9840e896ab4298dd27 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Dec 2023 20:38:50 -0500 Subject: [PATCH 106/136] Seed media files --- ...5c07196ab520cd8c63d30689b6021fa4b4ec1.json | 12 - ...106ac6bea906979c89355f805abb9650a7588.json | 12 + crates/cli/dev_database_seed/src/main.rs | 2 + .../cli/dev_database_seed/src/seeding/mod.rs | 5 +- .../src/seeding/seed_media_files.rs | 210 ++++++++++++++++++ crates/cli/dev_upload_media_file/src/main.rs | 1 + .../insert_media_file_from_cli_tool.rs | 8 +- 7 files changed, 235 insertions(+), 15 deletions(-) delete mode 100644 .sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json create mode 100644 .sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json create mode 100644 crates/cli/dev_database_seed/src/seeding/seed_media_files.rs diff --git a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json b/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json deleted file mode 100644 index fb10d9d281..0000000000 --- a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - }, - "hash": "030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1" -} diff --git a/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json new file mode 100644 index 0000000000..6a0e91bef8 --- /dev/null +++ b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n maybe_origin_filename = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + }, + "hash": "8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588" +} diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 917a8622a0..558478772e 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -8,6 +8,7 @@ use errors::AnyhowResult; use crate::bucket_clients::get_bucket_clients; use crate::cli_args::parse_cli_args; use crate::seeding::model_weights::seed_weights; +use crate::seeding::seed_media_files::seed_media_files; use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; use crate::seeding::voice_conversion::seed_voice_conversion; @@ -52,6 +53,7 @@ pub async fn main() -> AnyhowResult<()> { } seed_user_accounts(&pool).await?; + seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?; seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?; seed_voice_conversion(&pool).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index cb462d17c9..c1100805b6 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -1,5 +1,6 @@ +pub mod model_weights; +pub mod seed_media_files; pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; -pub mod zero_shot_tts; -pub mod model_weights; \ No newline at end of file +pub mod zero_shot_tts; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs new file mode 100644 index 0000000000..5bca8981e2 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -0,0 +1,210 @@ +use std::path::Path; + +use log::{info, warn}; +use sqlx::{MySql, Pool}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use filesys::file_read_bytes::file_read_bytes; +use filesys::file_size::file_size; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs}; +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { + info!("Seeding media files..."); + + // NB: This is idempotent and will only install the media files once. + // Delete the records if you want to re-seed. + let media_files = [ + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"), + // Audio + (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), + (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), + (HANASHI_USER_TOKEN, "m_sq2vhrrmmbr079afk21543aa2jg53g", MediaFileType::Audio, "media/audio/voice-samples/brandon2.wav"), + (HANASHI_USER_TOKEN, "m_s61a8g3jnddr3xdg3gctsjxxk9jwja", MediaFileType::Audio, "media/audio/voice-samples/elon.wav"), + (HANASHI_USER_TOKEN, "m_e59fh75g76pvebx4qe4r4aymd93q30", MediaFileType::Audio, "media/audio/voice-samples/goku.wav"), + (HANASHI_USER_TOKEN, "m_ttq0ym8ht4v0z87zqz27fhadp0743c", MediaFileType::Audio, "media/audio/voice-samples/hillary.wav"), + (HANASHI_USER_TOKEN, "m_300bv6f141hv553fjf287p5gb76k37", MediaFileType::Audio, "media/audio/voice-samples/mario_impersonator.wav"), + (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), + (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), + (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + for (user_token, media_file_token, media_file_type, subdirectory_path) in media_files { + let user_token = UserToken::new_from_str(user_token); + let media_file_token = MediaFileToken::new_from_str(media_file_token); + + let mut media_file_path = seed_tool_data_root.clone(); + media_file_path.push(subdirectory_path); + + let result = seed_model( + &mysql_pool, + &media_file_token, + &user_token, + media_file_type, + &media_file_path, + maybe_bucket_clients, + ).await; + + match result { + Ok(_) => info!("Seeded {:?}", media_file_path), + Err(err) => warn!(r#" + Could not seed media file {} , {:?} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the media file on a previous invocation!) + "#, media_file_token, subdirectory_path, err), + } + } + + Ok(()) +} + +async fn seed_model( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, + user_token: &UserToken, + media_file_type: MediaFileType, + media_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients>, +) -> AnyhowResult<()> { + info!("Seeding Media file {:?} ...", media_file_path); + + let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut maybe_bucket_prefix = None; + let mut maybe_bucket_extension = None; + + if media_file_exists(mysql_pool, media_file_token).await? { + info!("Media file already seeded: {:?}", media_file_token); + return Ok(()) + } + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(media_file_path, bucket_clients).await?; + bucket_hash = bucket_details.bucket_hash; + maybe_bucket_prefix = bucket_details.maybe_bucket_prefix; + maybe_bucket_extension = bucket_details.maybe_bucket_extension; + } + + let maybe_mime_type = get_mimetype_for_file(media_file_path)?; + let file_size_bytes = file_size(media_file_path)?; + let sha256_checksum = sha256_hash_file(media_file_path)?; + + let maybe_filename = media_file_path + .file_name() + .map(|name| name.to_str()) + .flatten(); + + insert_media_file_from_cli_tool(InsertArgs { + pool: &mysql_pool, + media_file_type, + maybe_mime_type, + file_size_bytes, + sha256_checksum: &sha256_checksum, + maybe_origin_filename: maybe_filename, + maybe_creator_user_token: Some(user_token), + creator_set_visibility: Visibility::Public, + public_bucket_directory_hash: &bucket_hash, + maybe_public_bucket_prefix: maybe_bucket_prefix.as_deref(), + maybe_public_bucket_extension: maybe_bucket_extension.as_deref(), + }).await?; + + Ok(()) +} + +async fn media_file_exists( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, +) -> AnyhowResult { + + const CAN_SEE_DELETED : bool = true; + + let maybe_file = get_media_file( + &media_file_token, + CAN_SEE_DELETED, + mysql_pool + ).await?; + + Ok(maybe_file.is_some()) +} + +struct BucketDetails { + bucket_hash: String, + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +async fn seed_file_to_bucket( + media_file_path: &Path, + bucket_clients: &BucketClients, +) -> AnyhowResult { + + info!("Uploading media file {:?} ...", media_file_path); + + let maybe_bucket_prefix = Some("fakeyou_"); + + let maybe_bucket_extension = media_file_path + .extension() + .map(|extension| extension.to_str()) + .flatten(); + + let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension); + + let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf()); + + info!("Reading media file: {:?}", media_file_path); + + let bytes = file_read_bytes(media_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + info!("Uploading media file to bucket path: {:?}", bucket_path); + + let _r = bucket_clients.public.upload_file_with_content_type( + &bucket_path, + bytes.as_ref(), + mimetype + ).await?; + + Ok(BucketDetails { + bucket_hash: bucket_location.get_object_hash().to_string(), + maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()), + maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()), + }) +} + +// Temporary "test" to generate random tokens. +// #[cfg(test)] +// mod tests { +// use tokens::tokens::media_files::MediaFileToken; +// +// #[test] +// fn test() { +// // Print some randomly sampled tokens +// for _i in 0..50 { +// let token = MediaFileToken::generate(); +// println!("Token: {}", token); +// } +// // Deliberately fail the test to get the output +// assert_eq!(1, 2); +// } +// } diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 7a72bacf56..694780ffb9 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -123,6 +123,7 @@ pub async fn main() -> AnyhowResult<()> { maybe_mime_type, file_size_bytes: 0, // TODO sha256_checksum: "", // TODO + maybe_origin_filename: None, maybe_creator_user_token: None, creator_set_visibility, public_bucket_directory_hash: public_upload_path.get_object_hash(), diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index 43a543db7d..fb180eba35 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -19,7 +19,9 @@ pub struct InsertArgs<'a> { pub file_size_bytes: u64, pub sha256_checksum: &'a str, - pub maybe_creator_user_token: Option, + pub maybe_origin_filename: Option<&'a str>, + + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_set_visibility: Visibility, pub public_bucket_directory_hash: &'a str, @@ -58,6 +60,8 @@ SET file_size_bytes = ?, checksum_sha2 = ?, + maybe_origin_filename = ?, + public_bucket_directory_hash = ?, maybe_public_bucket_prefix = ?, maybe_public_bucket_extension = ?, @@ -77,6 +81,8 @@ SET args.file_size_bytes, args.sha256_checksum, + args.maybe_origin_filename, + args.public_bucket_directory_hash, args.maybe_public_bucket_prefix, args.maybe_public_bucket_extension, From 30d6bca9b2456faa8c05963a6437226adde225fb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 11 Dec 2023 21:53:31 -0500 Subject: [PATCH 107/136] seed video files + fix idempotency --- .../src/seeding/seed_media_files.rs | 30 ++++++++++++++----- crates/cli/dev_upload_media_file/src/main.rs | 1 + .../src/queries/media_files/get_media_file.rs | 2 +- .../insert_media_file_from_cli_tool.rs | 8 ++++- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs index 5bca8981e2..15f8959ef6 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -28,14 +28,6 @@ pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Op // NB: This is idempotent and will only install the media files once. // Delete the records if you want to re-seed. let media_files = [ - // Images - (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"), - (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"), - (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"), - (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"), - (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"), - (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"), - (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"), // Audio (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), @@ -47,6 +39,27 @@ pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Op (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/image/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/image/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/image/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/image/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/image/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/image/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/image/face-animator-inputs/zelda.jpg"), + // Video + (HANASHI_USER_TOKEN, "m_gedarcfdkxx3zv8zz2wmwsnc1r8jbp", MediaFileType::Video, "media/video/storyteller-studio-renders/girl-rotating-room.mp4"), + (HANASHI_USER_TOKEN, "m_3s74yyd4jwrjdp17yx1694n29jv5gr", MediaFileType::Video, "media/video/storyteller-studio-renders/hanashi-run.mp4"), + (HANASHI_USER_TOKEN, "m_p6d64gm1wawz25bmdahfded3366b9f", MediaFileType::Video, "media/video/storyteller-studio-renders/mocap-kiss.mp4"), + (HANASHI_USER_TOKEN, "m_szp2c77z6je965k44vc22xn9432a88", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-girl.mp4"), + (HANASHI_USER_TOKEN, "m_mdwspz81zymw9hvmb1d3jm830t25xe", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-m.mp4"), + (HANASHI_USER_TOKEN, "m_prcs6xkm8phs5y4a2dfwemwjy6n6z3", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-girl.mp4"), + (HANASHI_USER_TOKEN, "m_5j889dzw08vn9pbrscx1sncff43ekp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-jww.mp4"), + (HANASHI_USER_TOKEN, "m_fwm2w6wbssez274ayz6x8ta0h1a9qp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pascal.mp4"), + (HANASHI_USER_TOKEN, "m_yqqkrt2srep6a35xg78114jn9epwg1", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pop.mp4"), + (HANASHI_USER_TOKEN, "m_m4r043kez14pm6vaamsecpz8fafh7g", MediaFileType::Video, "media/video/video-style-transfer/man-explaining.mp4"), + (HANASHI_USER_TOKEN, "m_81gq3hygqvrahg6ncxbtcpjap4f9nv", MediaFileType::Video, "media/video/video-style-transfer/woman-gradient-background.mp4"), + (HANASHI_USER_TOKEN, "m_dp9ykj3yasjx69g2fx2bpd26wwjz0f", MediaFileType::Video, "media/video/video-style-transfer/woman-red-hair.mp4"), ]; let seed_tool_data_root = get_seed_tool_data_root(); @@ -117,6 +130,7 @@ async fn seed_model( insert_media_file_from_cli_tool(InsertArgs { pool: &mysql_pool, + maybe_use_apriori_media_token: Some(media_file_token), media_file_type, maybe_mime_type, file_size_bytes, diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 694780ffb9..4112093b85 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -119,6 +119,7 @@ pub async fn main() -> AnyhowResult<()> { let (media_file_token, _id) = insert_media_file_from_cli_tool(InsertArgs { pool: &pool, + maybe_use_apriori_media_token: None, media_file_type, maybe_mime_type, file_size_bytes: 0, // TODO diff --git a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs index ec94000555..8dc215e82f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs @@ -13,7 +13,7 @@ use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct MediaFile { pub token: MediaFileToken, diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index fb180eba35..e829ba53fd 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -14,6 +14,10 @@ use tokens::tokens::users::UserToken; pub struct InsertArgs<'a> { pub pool: &'a MySqlPool, + /// If supplied, use this media token rather than generating a new one. + /// This is good to support idempotency in development and testing. + pub maybe_use_apriori_media_token: Option<&'a MediaFileToken>, + pub media_file_type: MediaFileType, pub maybe_mime_type: Option<&'a str>, pub file_size_bytes: u64, @@ -35,7 +39,9 @@ pub async fn insert_media_file_from_cli_tool( args: InsertArgs<'_> ) -> AnyhowResult<(MediaFileToken, u64)> { - let media_file_token = MediaFileToken::generate(); + let media_file_token = args.maybe_use_apriori_media_token + .map(|token| token.clone()) + .unwrap_or_else(|| MediaFileToken::generate()); let mut transaction = args.pool.begin().await?; From 795eb7a141851fa63a18c739d5b369309608475d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 02:15:06 -0500 Subject: [PATCH 108/136] spacing nit --- .../inference_job/config/inference-job.development.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index c16128c8d6..a8427bea58 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -18,11 +18,11 @@ REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" # should rename my folder # abs path to the folder that contains the inference code need to get that container in the repo need basepath? relative to everyones env VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" -#VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" -VALL_E_X_INFERENCE_COMMAND = "python3 main.py" -VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .venv/bin/activate" +#VALL_E_X_INFERENCE_ROOT_DIRECTORY="/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" +VALL_E_X_INFERENCE_COMMAND="python3 main.py" +VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .venv/bin/activate" -# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE = "" # fill this out for dev? +# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev? #VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" #VALL_E_X_INFERENCE_COMMAND="python3 main.py" #VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate" From 07750fa122614d81f948a2ce655d90c4427259cb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 02:55:23 -0500 Subject: [PATCH 109/136] refactor bucket modules --- crates/_types/buckets/src/private/mod.rs | 3 ++- .../zs_voices/bucket_directory.rs} | 4 ++-- .../zs_voices/bucket_file_path.rs} | 15 ++++++--------- .../_types/buckets/src/private/zs_voices/mod.rs | 2 ++ .../{directory.rs => bucket_directory.rs} | 3 +-- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../_types/buckets/src/public/media_files/mod.rs | 4 ++-- .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../buckets/src/public/media_uploads/mod.rs | 4 ++-- crates/_types/buckets/src/public/mod.rs | 1 - .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 4 ++-- .../src/public/voice_conversion_results/mod.rs | 4 ++-- .../{directory.rs => bucket_directory.rs} | 2 +- .../{original_file.rs => bucket_file_path.rs} | 5 +++-- .../buckets/src/public/weight_uploads/mod.rs | 4 ++-- crates/_types/buckets/src/public/zs_voices/mod.rs | 2 -- .../src/seeding/model_weights.rs | 6 ++---- .../src/seeding/seed_media_files.rs | 2 +- .../src/seeding/zero_shot_tts.rs | 2 +- crates/cli/dev_upload_media_file/src/main.rs | 2 +- .../queries/media_uploads/insert_media_upload.rs | 2 +- .../queries/user_bookmarks/list_user_bookmarks.rs | 5 ++--- .../insert_dataset_sample_and_media_file.rs | 2 +- .../lipsync/sad_talker/download_audio_file.rs | 6 +++--- .../lipsync/sad_talker/download_image_file.rs | 4 ++-- .../job_types/lipsync/sad_talker/process_job.rs | 2 +- .../tacotron2_v2_early_fakeyou/upload_results.rs | 2 +- .../tts/vall_e_x/process_job_create_voice.rs | 6 +++--- .../tts/vall_e_x/process_job_inference_voice.rs | 6 +++--- .../src/job/job_types/vc/rvc_v2/process_job.rs | 4 ++-- .../job/job_types/vc/so_vits_svc/process_job.rs | 4 ++-- .../inference_job/get_inference_job_status.rs | 4 ++-- .../endpoints/media_files/get_media_file.rs | 2 +- .../media_uploads/common/handle_upload.rs | 2 +- .../endpoints/tts/get_tts_inference_job_status.rs | 2 +- .../voice_dataset_samples/upload_zs_sample.rs | 2 +- 38 files changed, 65 insertions(+), 71 deletions(-) rename crates/_types/buckets/src/{public/zs_voices/directory.rs => private/zs_voices/bucket_directory.rs} (96%) rename crates/_types/buckets/src/{public/zs_voices/file.rs => private/zs_voices/bucket_file_path.rs} (88%) create mode 100644 crates/_types/buckets/src/private/zs_voices/mod.rs rename crates/_types/buckets/src/public/media_files/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/media_files/{original_file.rs => bucket_file_path.rs} (96%) rename crates/_types/buckets/src/public/media_uploads/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/media_uploads/{original_file.rs => bucket_file_path.rs} (94%) rename crates/_types/buckets/src/public/voice_conversion_results/{directory.rs => bucket_directory.rs} (95%) rename crates/_types/buckets/src/public/voice_conversion_results/{original_file.rs => bucket_file_path.rs} (94%) rename crates/_types/buckets/src/public/weight_uploads/{directory.rs => bucket_directory.rs} (96%) rename crates/_types/buckets/src/public/weight_uploads/{original_file.rs => bucket_file_path.rs} (94%) delete mode 100644 crates/_types/buckets/src/public/zs_voices/mod.rs diff --git a/crates/_types/buckets/src/private/mod.rs b/crates/_types/buckets/src/private/mod.rs index 27089e4df1..d3090254af 100644 --- a/crates/_types/buckets/src/private/mod.rs +++ b/crates/_types/buckets/src/private/mod.rs @@ -1 +1,2 @@ -pub mod private_path; \ No newline at end of file +pub mod private_path; +pub mod zs_voices; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/directory.rs b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/zs_voices/directory.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_directory.rs index 1cfc740769..9d4ecee304 100644 --- a/crates/_types/buckets/src/public/zs_voices/directory.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs @@ -1,6 +1,6 @@ use crockford::crockford_entropy_lower; -use crate::private::private_path::PrivatePath; +use crate::private::private_path::PrivatePath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. @@ -79,7 +79,7 @@ impl ZeroShotVoiceEmbeddingBucketDirectory { #[cfg(test)] mod tests { - use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; + use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/zs_voices/file.rs b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs similarity index 88% rename from crates/_types/buckets/src/public/zs_voices/file.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs index 6428936c7c..47948912ec 100644 --- a/crates/_types/buckets/src/public/zs_voices/file.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs @@ -3,11 +3,10 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::private::private_path::PrivatePath; -use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; +use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; // TODO: Generate these from a macro. - /// The original user upload file. /// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] @@ -74,11 +73,9 @@ impl ZeroShotVoiceEmbeddingBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; - mod with_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; #[test] pub fn generate_new_entropy() { @@ -97,8 +94,8 @@ mod tests { } mod without_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; use super::*; @@ -135,7 +132,7 @@ mod tests { #[test] pub fn get_object_hash() { let hash = "abcdefghijk"; - let file = MediaFileBucketPath::from_object_hash(hash, None, None); + let file = ZeroShotVoiceEmbeddingBucketPath::from_object_hash(hash, ModelCategory::Tts, ModelType::VallEx, 1); assert_eq!(file.get_object_hash(), hash); } } diff --git a/crates/_types/buckets/src/private/zs_voices/mod.rs b/crates/_types/buckets/src/private/zs_voices/mod.rs new file mode 100644 index 0000000000..29c183a5c6 --- /dev/null +++ b/crates/_types/buckets/src/private/zs_voices/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_files/directory.rs b/crates/_types/buckets/src/public/media_files/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/directory.rs rename to crates/_types/buckets/src/public/media_files/bucket_directory.rs index e0083da128..855332bcf3 100644 --- a/crates/_types/buckets/src/public/media_files/directory.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_directory.rs @@ -4,7 +4,6 @@ use crate::public::public_path::PublicPath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. - // TODO: Use a central path registry for quick reference const MEDIA_FILE_DIRECTORY: &str = "/media"; @@ -47,7 +46,7 @@ impl MediaFileBucketDirectory { #[cfg(test)] mod tests { - use crate::public::media_files::directory::MediaFileBucketDirectory; + use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_files/original_file.rs b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/original_file.rs rename to crates/_types/buckets/src/public/media_files/bucket_file_path.rs index efd0fd9458..e503819440 100644 --- a/crates/_types/buckets/src/public/media_files/original_file.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_files::directory::MediaFileBucketDirectory; +use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "file.bin"; @@ -79,7 +79,7 @@ impl MediaFileBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; + use crate::public::media_files::bucket_file_path::MediaFileBucketPath; mod with_prefix_and_extension { use super::*; diff --git a/crates/_types/buckets/src/public/media_files/mod.rs b/crates/_types/buckets/src/public/media_files/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_files/mod.rs +++ b/crates/_types/buckets/src/public/media_files/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_uploads/directory.rs b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_uploads/directory.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_directory.rs index aa4ffc0470..77ff4ca662 100644 --- a/crates/_types/buckets/src/public/media_uploads/directory.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs @@ -47,7 +47,7 @@ impl MediaUploadDirectory { #[cfg(test)] mod tests { - use crate::public::media_uploads::directory::MediaUploadDirectory; + use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/original_file.rs b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/media_uploads/original_file.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs index 7f0a40a6b7..6867ff6e5a 100644 --- a/crates/_types/buckets/src/public/media_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; @@ -61,7 +61,7 @@ impl MediaUploadOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + use crate::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/mod.rs b/crates/_types/buckets/src/public/media_uploads/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_uploads/mod.rs +++ b/crates/_types/buckets/src/public/media_uploads/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index dc35c40ed4..fcceb68d9b 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,5 +2,4 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod zs_voices; pub mod weight_uploads; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs similarity index 95% rename from crates/_types/buckets/src/public/voice_conversion_results/directory.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs index c853cf2d2f..4f726b2c14 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs @@ -40,7 +40,7 @@ impl VoiceConversionResultDirectory { #[cfg(test)] mod tests { - use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; + use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; #[test] pub fn get_directory_path_str() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/voice_conversion_results/original_file.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs index 9e256e89d2..9163363e49 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::public::public_path::PublicPath; -use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; +use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; const ORIGINAL_FILE_PREFIX : &str = "fakeyou_"; const ORIGINAL_FILE_SUFFIX : &str = ".wav"; @@ -65,7 +65,7 @@ impl VoiceConversionResultOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; + use crate::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/directory.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/weight_uploads/directory.rs rename to crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs index fcedda62dd..1f3e016fb1 100644 --- a/crates/_types/buckets/src/public/weight_uploads/directory.rs +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs @@ -48,7 +48,7 @@ impl WeightUploadDirectory { #[cfg(test)] mod tests { - use crate::public::weight_uploads::directory::WeightUploadDirectory; + use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/weight_uploads/original_file.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/weight_uploads/original_file.rs rename to crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs index 42f203f849..69ca626486 100644 --- a/crates/_types/buckets/src/public/weight_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::weight_uploads::directory::WeightUploadDirectory; use crate::public::public_path::PublicPath; +use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; @@ -56,7 +56,8 @@ impl WeightUploadOriginalFilePath { #[cfg(test)] mod tests { use std::path::PathBuf; - use crate::public::weight_uploads::original_file::WeightUploadOriginalFilePath; + + use crate::public::weight_uploads::bucket_file_path::WeightUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/weight_uploads/mod.rs b/crates/_types/buckets/src/public/weight_uploads/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/weight_uploads/mod.rs +++ b/crates/_types/buckets/src/public/weight_uploads/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/mod.rs b/crates/_types/buckets/src/public/zs_voices/mod.rs deleted file mode 100644 index a5c0d4bfab..0000000000 --- a/crates/_types/buckets/src/public/zs_voices/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod directory; -pub mod file; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index e766086a0b..8860d323c7 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -16,7 +16,7 @@ use tokens::tokens::users::UserToken; use crate::seeding::users::HANASHI_USERNAME; -pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { +pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserToken) -> AnyhowResult<()> { let sd1_5_markdown_description = r#" # Dragonfruit AI Models Update and Workflow @@ -81,9 +81,7 @@ pub async fn seed_weights_for_paging( mysql_pool: &Pool,user_token: User *Thank you for your continued support and feedback!* "#; -let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; - - let user_token = user_token; + let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; // create a loop that loops from 1 to 100 for i in 1..=100 { diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs index 15f8959ef6..1e3f2c3c80 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -3,7 +3,7 @@ use std::path::Path; use log::{info, warn}; use sqlx::{MySql, Pool}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use errors::AnyhowResult; diff --git a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs index d01c359d13..196c36b6ef 100644 --- a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs @@ -1,7 +1,7 @@ use log::info; use sqlx::{MySql, Pool}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 4112093b85..3928071c5d 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -3,7 +3,7 @@ use std::time::Duration; use log::info; use sqlx::mysql::MySqlPoolOptions; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; use enums::by_table::media_files::media_file_type::MediaFileType; diff --git a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs index 043a3b81f3..667e3751f1 100644 --- a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index 1e1896a750..232c7344a6 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -4,10 +4,9 @@ use sqlx::MySqlPool; use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; -use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; -use tokens::tokens::user_bookmarks::UserBookmarkToken; +use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; +use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; pub async fn list_user_bookmarks( username: &str, diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs index c12bf4e62a..0abda7cd4b 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use sqlx::MySqlPool; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 4e6f8973e7..e90e3fc887 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -5,9 +5,9 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 3d401f0a7a..8285a78c5a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -5,8 +5,8 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource; diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index cb6d563156..57852d2b4b 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -4,7 +4,7 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs index 87e9747a3d..e2a7547f42 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/upload_results.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use log::info; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index f83b5b764c..e210b20259 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -5,9 +5,9 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; -use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; +use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; +use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index 20359f1e0d..76f4940796 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -5,9 +5,9 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; -use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; +use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; +use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 74496b6abc..158f304a2b 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -5,8 +5,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index f5d7fa1b89..e02716e323 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -4,8 +4,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 96be5ed7d7..ea3e6f8f15 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -9,8 +9,8 @@ use chrono::{DateTime, Utc}; use log::error; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::common::job_status_plus::JobStatusPlus; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs index 021d4db258..b18f8af8da 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::warn; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use mysql_queries::queries::media_files::get_media_file::get_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 792af74603..2d79cab2e7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -5,7 +5,7 @@ use actix_multipart::Multipart; use actix_web::{HttpRequest, web}; use log::{error, info, warn}; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index be43c33d8a..20dab038b7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::{error, log}; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index b257ce9c14..004d303f44 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -6,7 +6,7 @@ use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use hyper::StatusCode; use log::{error, info, warn}; use once_cell::sync::Lazy; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType::MediaUpload; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; From d9fe4e56a0556426ad041f35c07dae8007499ef8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 08:00:32 +0000 Subject: [PATCH 110/136] Bump version-sync from 0.8.1 to 0.9.5 Bumps [version-sync](https://github.com/mgeisler/version-sync) from 0.8.1 to 0.9.5. - [Release notes](https://github.com/mgeisler/version-sync/releases) - [Commits](https://github.com/mgeisler/version-sync/compare/0.8.1...0.9.5) --- updated-dependencies: - dependency-name: version-sync dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 460 +++++++++++++--------------- crates/vendor/limitation/Cargo.toml | 2 +- 2 files changed, 209 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fe281f68c..cfd5d64312 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,7 +78,7 @@ dependencies = [ "log", "mime", "mime_guess", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", ] @@ -110,7 +110,7 @@ dependencies = [ "language-tags", "local-channel", "mime", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "rand 0.8.5", "sha1 0.10.6", @@ -127,7 +127,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -164,8 +164,8 @@ checksum = "0a0a77f836d869f700e5b47ac7c3c8b9c8bc82e4aec861954c6198abee3ebd4d" dependencies = [ "darling 0.20.3", "parse-size", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -268,7 +268,7 @@ dependencies = [ "smallvec 1.11.2", "socket2 0.5.5", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -296,8 +296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -307,8 +307,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -416,8 +416,8 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9afe97decc070be1864d96ab118383abed1c58fc09696663666049431110c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -633,8 +633,8 @@ version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -681,7 +681,7 @@ dependencies = [ "native-tls", "serde", "serde_json", - "url 2.5.0", + "url", ] [[package]] @@ -715,7 +715,7 @@ dependencies = [ "serde", "thiserror", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -811,7 +811,7 @@ dependencies = [ "serde_json", "sqlx", "tokio", - "toml", + "toml 0.5.11", "url_config", ] @@ -1142,8 +1142,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1252,7 +1252,7 @@ dependencies = [ "anyhow", "easyenv", "my-workspace-hack", - "url 2.5.0", + "url", ] [[package]] @@ -1278,7 +1278,7 @@ dependencies = [ "rand 0.8.5", "serde", "tempfile", - "toml", + "toml 0.5.11", "uuid 0.8.2", ] @@ -1294,7 +1294,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "time 0.3.30", "version_check", ] @@ -1485,7 +1485,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1527,8 +1527,8 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1541,8 +1541,8 @@ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1555,8 +1555,8 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 2.0.39", ] @@ -1568,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1579,7 +1579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core 0.14.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1590,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1693,8 +1693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling 0.14.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -1715,8 +1715,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1788,8 +1788,8 @@ dependencies = [ "byteorder", "diesel_derives", "mysqlclient-sys", - "percent-encoding 2.3.1", - "url 2.5.0", + "percent-encoding", + "url", ] [[package]] @@ -1799,8 +1799,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ "diesel_table_macro_syntax", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1875,8 +1875,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -2030,13 +2030,13 @@ dependencies = [ "bytes 1.5.0", "dyn-clone", "lazy_static", - "percent-encoding 2.3.1", + "percent-encoding", "reqwest", "rustc_version 0.2.3", "serde", "serde_json", "serde_with", - "url 2.5.0", + "url", "void", ] @@ -2177,7 +2177,7 @@ dependencies = [ "sqlx-core", "sqlx-mysql", "strum", - "toml", + "toml 0.5.11", "utoipa", ] @@ -2347,7 +2347,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", ] [[package]] @@ -2475,8 +2475,8 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -2743,8 +2743,8 @@ dependencies = [ "log", "mac", "markup5ever", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -2828,7 +2828,7 @@ dependencies = [ "serde_json", "serde_qs 0.8.5", "serde_urlencoded", - "url 2.5.0", + "url", ] [[package]] @@ -2909,7 +2909,7 @@ dependencies = [ "http 1.0.0", "http-body 1.0.0", "pin-project-lite", - "tokio 1.34.0", + "tokio", "want", ] @@ -2937,7 +2937,7 @@ dependencies = [ "hyper 1.0.1", "hyper-util", "native-tls", - "tokio 1.34.0", + "tokio", "tokio-native-tls", "tower-service", ] @@ -2956,7 +2956,7 @@ dependencies = [ "hyper 1.0.1", "pin-project-lite", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "tower", "tower-service", "tracing", @@ -2999,17 +2999,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -3176,15 +3165,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.10.5" @@ -3330,7 +3310,7 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna 0.5.0", + "idna", "mime", "native-tls", "nom", @@ -3338,7 +3318,7 @@ dependencies = [ "quoted_printable", "socket2 0.5.5", "tokio", - "url 2.5.0", + "url", ] [[package]] @@ -3539,20 +3519,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "maybe-async" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3758,8 +3732,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3843,7 +3817,7 @@ dependencies = [ "twitch_oauth2", "unicode-bidi", "unicode-normalization", - "url 2.5.0", + "url", "void", "winapi 0.3.9", "windows-sys 0.48.0", @@ -3875,7 +3849,7 @@ dependencies = [ "sqlx", "tokens", "tokio", - "toml", + "toml 0.5.11", ] [[package]] @@ -4147,8 +4121,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4298,12 +4272,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -4367,8 +4335,8 @@ checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4405,8 +4373,8 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4517,8 +4485,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", "version_check", ] @@ -4529,20 +4497,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "version_check", ] -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid", -] - [[package]] name = "proc-macro2" version = "1.0.69" @@ -4561,17 +4520,6 @@ dependencies = [ "cc", ] -[[package]] -name = "pulldown-cmark" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1b74cc784b038a9921fd1a48310cc2e238101aa8ae0b94201e2d85121dd68b5" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4613,22 +4561,13 @@ dependencies = [ "memchr", ] -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.69", + "proc-macro2", ] [[package]] @@ -4797,7 +4736,7 @@ dependencies = [ "dtoa 0.4.8", "futures 0.1.31", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", "tokio-codec", "tokio-executor", @@ -4805,7 +4744,7 @@ dependencies = [ "tokio-sync", "tokio-tcp", "tokio-uds", - "url 2.5.0", + "url", ] [[package]] @@ -4818,9 +4757,9 @@ dependencies = [ "combine 4.6.6", "dtoa 0.4.8", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", - "url 2.5.0", + "url", ] [[package]] @@ -4834,13 +4773,13 @@ dependencies = [ "combine 4.6.6", "futures-util", "itoa 1.0.9", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "ryu", "sha1 0.6.1", "tokio", "tokio-util", - "url 2.5.0", + "url", ] [[package]] @@ -4955,7 +4894,7 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "serde", "serde_json", @@ -4965,7 +4904,7 @@ dependencies = [ "tokio-native-tls", "tokio-util", "tower-service", - "url 2.5.0", + "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", @@ -4982,7 +4921,7 @@ dependencies = [ "serde", "serde_derive", "sqlx", - "toml", + "toml 0.5.11", ] [[package]] @@ -5063,8 +5002,8 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rust-embed-utils", "shellexpand", "syn 2.0.39", @@ -5111,7 +5050,7 @@ dependencies = [ "maybe-async", "md5", "minidom", - "percent-encoding 2.3.1", + "percent-encoding", "quick-xml 0.26.0", "reqwest", "serde", @@ -5121,7 +5060,7 @@ dependencies = [ "time 0.3.30", "tokio", "tokio-stream", - "url 2.5.0", + "url", ] [[package]] @@ -5380,7 +5319,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0", + "semver-parser", ] [[package]] @@ -5395,12 +5334,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "semver-parser" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46e1121e8180c12ff69a742aabc4f310542b6ccb69f1691689ac17fdf8618aa" - [[package]] name = "serde" version = "1.0.193" @@ -5416,8 +5349,8 @@ version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5457,7 +5390,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5468,7 +5401,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cac3f1e2ca2fe333923a1ae72caca910b98ed0630bb35ef6f8c8517d6e81afa" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5479,11 +5412,20 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5513,8 +5455,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5538,8 +5480,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5692,8 +5634,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5824,7 +5766,7 @@ dependencies = [ "native-tls", "once_cell", "paste", - "percent-encoding 2.3.1", + "percent-encoding", "rustls 0.21.9", "rustls-pemfile", "serde", @@ -5836,7 +5778,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "url 2.5.0", + "url", "webpki-roots 0.25.3", ] @@ -5846,8 +5788,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "sqlx-core", "sqlx-macros-core", "syn 1.0.109", @@ -5865,8 +5807,8 @@ dependencies = [ "heck", "hex", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "serde", "serde_json", "sha2 0.10.8", @@ -5877,7 +5819,7 @@ dependencies = [ "syn 1.0.109", "tempfile", "tokio", - "url 2.5.0", + "url", ] [[package]] @@ -5909,7 +5851,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "rand 0.8.5", "rsa", "serde", @@ -5979,11 +5921,11 @@ dependencies = [ "futures-util", "libsqlite3-sys", "log", - "percent-encoding 2.3.1", + "percent-encoding", "serde", "sqlx-core", "tracing", - "url 2.5.0", + "url", "urlencoding", ] @@ -6086,7 +6028,7 @@ dependencies = [ "native-tls", "once_cell", "pin-project-lite", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_caching", @@ -6107,7 +6049,7 @@ dependencies = [ "time 0.3.30", "tokens", "tokio", - "toml", + "toml 0.5.11", "tts_common", "twitch_common", "twitch_oauth2", @@ -6150,8 +6092,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", ] [[package]] @@ -6193,8 +6135,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustversion", "syn 2.0.39", ] @@ -6390,25 +6332,14 @@ dependencies = [ "symphonia-metadata", ] -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid", -] - [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6418,8 +6349,8 @@ version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6525,8 +6456,8 @@ version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6619,7 +6550,7 @@ dependencies = [ "sqlx", "sqlx-core", "strum", - "toml", + "toml 0.5.11", "utoipa", ] @@ -6680,8 +6611,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6818,6 +6749,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -6828,7 +6793,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tower-layer", "tower-service", "tracing", @@ -6864,8 +6829,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6984,7 +6949,7 @@ dependencies = [ "rustls-native-certs", "sha-1", "thiserror", - "url 2.5.0", + "url", "utf-8", "webpki", ] @@ -7014,7 +6979,7 @@ dependencies = [ "my-workspace-hack", "mysql_queries", "native-tls", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_common", @@ -7029,7 +6994,7 @@ dependencies = [ "tokens", "tokio", "tokio-tungstenite", - "toml", + "toml 0.5.11", "tts_common", "twitch_api2", "twitch_common", @@ -7059,7 +7024,7 @@ dependencies = [ "thiserror", "twitch_oauth2", "typed-builder", - "url 2.5.0", + "url", "version_check", ] @@ -7095,7 +7060,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "url 2.5.0", + "url", ] [[package]] @@ -7104,8 +7069,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -7157,12 +7122,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -7190,17 +7149,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - [[package]] name = "url" version = "2.5.0" @@ -7208,8 +7156,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", - "percent-encoding 2.3.1", + "idna", + "percent-encoding", "serde", ] @@ -7236,7 +7184,7 @@ version = "0.0.1" dependencies = [ "my-workspace-hack", "once_cell", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "regex", "unicode-segmentation", ] @@ -7282,7 +7230,7 @@ dependencies = [ "strum_macros", "time 0.3.30", "tokens", - "toml", + "toml 0.5.11", "user_input_common", ] @@ -7317,8 +7265,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "regex", "syn 2.0.39", ] @@ -7368,18 +7316,17 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version-sync" -version = "0.8.1" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "844f3d3a2467f15cb999f5af7775f6e108ac546d4f42365832ed4c755404f806" +checksum = "835169da0173ea373ddf5987632aac1f918967fbbe58195e304342282efa6089" dependencies = [ - "itertools 0.8.2", - "proc-macro2 0.4.30", - "pulldown-cmark 0.4.1", + "proc-macro2", + "pulldown-cmark", "regex", - "semver-parser 0.9.0", - "syn 0.15.44", - "toml", - "url 1.7.2", + "semver 1.0.20", + "syn 2.0.39", + "toml 0.7.8", + "url", ] [[package]] @@ -7516,8 +7463,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-shared", ] @@ -7540,7 +7487,7 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ - "quote 1.0.33", + "quote", "wasm-bindgen-macro-support", ] @@ -7550,8 +7497,8 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7681,7 +7628,7 @@ dependencies = [ "sqlx", "time 0.3.30", "tokio", - "toml", + "toml 0.5.11", "twitch_api2", "twitch_common", "twitch_oauth2", @@ -7884,6 +7831,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -7919,8 +7875,8 @@ version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 51f1626635..d13d4d9d2d 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -28,4 +28,4 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] tokio = "1.34.0" -version-sync = "0.8.1" +version-sync = "0.9.5" From f9ff9b8c9ffd6ec520418d970bfd73ce5308e661 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 11:57:42 -0500 Subject: [PATCH 111/136] Kill newrelic. This is painful to keep upgrading Keeping the vendor directory because we may introduce later. We've already done some upkeep. --- Cargo.lock | 189 +++--------------- Cargo.toml | 3 +- .../service/job/email_sender_job/Cargo.toml | 3 - crates/service/job/inference_job/Cargo.toml | 3 - .../job/inference_job/src/job_dependencies.rs | 5 - crates/service/job/inference_job/src/main.rs | 9 - .../service/job/tts_inference_job/Cargo.toml | 3 - .../src/job_steps/job_args.rs | 5 - .../src/job_steps/process_single_job.rs | 47 +---- .../service/job/tts_inference_job/src/main.rs | 88 +++----- 10 files changed, 64 insertions(+), 291 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfd5d64312..25191d742f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http 0.2.11", + "http", "httparse", "httpdate", "itoa 1.0.9", @@ -176,7 +176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "http 0.2.11", + "http", "regex", "serde", "tracing", @@ -614,8 +614,8 @@ dependencies = [ "hex", "hmac 0.12.1", "http-types", - "hyper 0.14.27", - "hyper-tls 0.5.0", + "hyper", + "hyper-tls", "serde", "serde_json", "serde_path_to_error", @@ -676,7 +676,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ - "http 0.2.11", + "http", "log", "native-tls", "serde", @@ -1924,7 +1924,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "mimetypes", @@ -2114,7 +2114,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "images", "jobs_common", "lfu", @@ -2599,7 +2599,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", + "http", "indexmap", "slab", "tokio", @@ -2759,17 +2759,6 @@ dependencies = [ "itoa 1.0.9", ] -[[package]] -name = "http" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" -dependencies = [ - "bytes 1.5.0", - "fnv", - "itoa 1.0.9", -] - [[package]] name = "http-body" version = "0.4.5" @@ -2777,30 +2766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http 0.2.11", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" -dependencies = [ - "bytes 1.5.0", - "http 1.0.0", -] - -[[package]] -name = "http-body-util" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" -dependencies = [ - "bytes 1.5.0", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", + "http", "pin-project-lite", ] @@ -2820,7 +2786,7 @@ dependencies = [ "async-channel", "base64 0.13.1", "futures-lite", - "http 0.2.11", + "http", "infer 0.2.3", "pin-project-lite", "rand 0.7.3", @@ -2884,8 +2850,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", - "http-body 0.4.5", + "http", + "http-body", "httparse", "httpdate", "itoa 1.0.9", @@ -2897,22 +2863,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" -dependencies = [ - "bytes 1.5.0", - "futures-channel", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "pin-project-lite", - "tokio", - "want", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -2920,48 +2870,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.5.0", - "hyper 0.14.27", + "hyper", "native-tls", "tokio", "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes 1.5.0", - "http-body-util", - "hyper 1.0.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" -dependencies = [ - "bytes 1.5.0", - "futures-channel", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "hyper 1.0.1", - "pin-project-lite", - "socket2 0.5.5", - "tokio", - "tower", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.58" @@ -3095,7 +3009,7 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "images", "jobs_common", "lfu", @@ -3105,7 +3019,6 @@ dependencies = [ "mimetypes", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "once_cell", "r2d2_redis", "redis_common", @@ -3772,7 +3685,7 @@ dependencies = [ "hashbrown 0.14.2", "heck", "hmac 0.12.1", - "hyper 0.14.27", + "hyper", "indexmap", "lazy_static", "libc", @@ -3903,25 +3816,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" -[[package]] -name = "newrelic-telemetry" -version = "0.1.0" -dependencies = [ - "anyhow", - "env_logger", - "flate2", - "futures 0.3.29", - "hyper 0.14.27", - "hyper-tls 0.6.0", - "log", - "my-workspace-hack", - "rand 0.8.5", - "serde", - "serde_json", - "tokio", - "uuid 0.8.2", -] - [[package]] name = "nix" version = "0.26.4" @@ -4884,10 +4778,10 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", - "http-body 0.4.5", - "hyper 0.14.27", - "hyper-tls 0.5.0", + "http", + "http-body", + "hyper", + "hyper-tls", "ipnet", "js-sys", "log", @@ -5045,7 +4939,7 @@ dependencies = [ "futures 0.3.29", "hex", "hmac 0.12.1", - "http 0.2.11", + "http", "log", "maybe-async", "md5", @@ -6009,7 +5903,7 @@ dependencies = [ "hmac 0.10.1", "hostname", "http_server_common", - "hyper 0.14.27", + "hyper", "infer 0.5.0", "jwt", "language-tags", @@ -6783,28 +6677,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -6865,7 +6737,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -6896,14 +6768,13 @@ dependencies = [ "filesys", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "lfu", "log", "memory_caching", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "r2d2_redis", "serde", "serde_derive", @@ -6941,7 +6812,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http 0.2.11", + "http", "httparse", "log", "rand 0.8.5", @@ -6971,7 +6842,7 @@ dependencies = [ "futures 0.3.29", "futures-util", "hostname", - "hyper 0.14.27", + "hyper", "lazy_static", "log", "magic-crypt", @@ -7013,7 +6884,7 @@ dependencies = [ "chrono", "displaydoc", "futures 0.3.29", - "http 0.2.11", + "http", "once_cell", "reqwest", "serde", @@ -7053,7 +6924,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "displaydoc", - "http 0.2.11", + "http", "once_cell", "rand 0.8.5", "reqwest", @@ -7357,7 +7228,7 @@ dependencies = [ "google_drive_common", "hashing", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -7386,7 +7257,7 @@ dependencies = [ "filesys", "google_drive_common", "hostname", - "hyper 0.14.27", + "hyper", "jobs_common", "log", "my-workspace-hack", @@ -7606,7 +7477,7 @@ dependencies = [ "futures-util", "hostname", "http_server_common", - "hyper 0.14.27", + "hyper", "lazy_static", "limitation", "log", diff --git a/Cargo.toml b/Cargo.toml index 7649065a0d..1ea5d8fefc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,8 @@ members = [ # Vendored code "crates/vendor/limitation", - "crates/vendor/newrelic-telemetry-sdk-rust", + # NB(bt,2023-12-12): Newrelic keeps breaking with upgrades. Kill for now. + #"crates/vendor/newrelic-telemetry-sdk-rust", # cargo-hakari workspace hack (compilation speedup hack) "crates/my-workspace-hack", diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index f72b7f7fba..733388d420 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -44,9 +44,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -# newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 9f8205ca14..199d70f522 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -41,9 +41,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 504d2fb08d..79d8d703c1 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -22,7 +22,6 @@ use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmen use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; -use newrelic_telemetry::Client as NewRelicClient; use crate::job_specific_dependencies::JobSpecificDependencies; use crate::util::scoped_execution::ScopedExecution; @@ -135,10 +134,6 @@ pub struct ClientDependencies { pub job_progress_reporter: Box, pub firehose_publisher: FirehosePublisher, - - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, } pub struct FileSystemDetails { diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 7779535799..186e87c9f4 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -44,7 +44,6 @@ use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use memory_caching::ttl_key_counter::TtlKeyCounter; use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use newrelic_telemetry::ClientBuilder; use crate::http_server::run_http_server::CreateServerArgs; use crate::http_server::run_http_server::launch_http_server; @@ -149,12 +148,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -237,8 +230,6 @@ async fn main() -> AnyhowResult<()> { clients: ClientDependencies { job_progress_reporter, firehose_publisher, - newrelic_client, - newrelic_disabled, }, job: JobSystemDependencies { system: JobSystemControls { diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index cbfdd24de1..2d1f6fa732 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -30,9 +30,6 @@ tts_common = { path = "../../../lib/tts_common" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } diff --git a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs index 433c04771f..3b4296a0d3 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs @@ -10,7 +10,6 @@ use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::TtsModelForInferenceRecord; -use newrelic_telemetry::Client as NewRelicClient; use crate::caching::cache_miss_strategizer_multi::SyncMultiCacheMissStrategizer; use crate::caching::virtual_lfu_cache::SyncVirtualLfuCache; @@ -38,10 +37,6 @@ pub struct JobArgs { pub job_stats: JobStats, - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, - pub worker_details: JobWorkerDetails, // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index bb9cd522a8..1767a674e2 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -6,14 +6,13 @@ use std::fs::File; use std::io::Read; use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; use container_common::anyhow_result::AnyhowResult; -use container_common::token::random_uuid::generate_random_uuid; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; @@ -24,7 +23,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_done use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceRecord}; use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; -use newrelic_telemetry::Span; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; @@ -56,34 +54,13 @@ pub async fn process_single_job( job_args: &JobArgs, job: &AvailableTtsInferenceJob, cached_model_record: &TtsModelForInferenceRecord, -) -> Result<(Span, Span), ProcessSingleJobError> { +) -> Result<(), ProcessSingleJobError> { // NB: Hack to allow cache bypass for debug requests // This will let us iterate faster with model changes. let mut maybe_queried_model = None; let mut model_record = cached_model_record; - let start = Instant::now(); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - let maybe_user_token = job.maybe_creator_user_token.as_deref().unwrap_or(""); - - let mut job_iteration_span = Span::new(&span_id, &trace_id, get_timestamp_millis()) - .name("single job execution") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - - let created_at_timestamp = (job.created_at.timestamp() as u64) * 1000; - - let mut since_creation_span = Span::new(&span_id, &trace_id, created_at_timestamp) - .name("job since creation") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - let mut job_progress_reporter = job_args .job_progress_reporter .new_tts_inference(&job.inference_job_token) @@ -100,15 +77,7 @@ pub async fn process_single_job( if !lock_acquired { warn!("Could not acquire job lock for: {:?}", &job.id); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "failure"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "failure"); - job_iteration_span.set_duration(duration); - - return Ok((since_creation_span, job_iteration_span)); + return Ok(()); } info!("Lock acquired for job: {}", job.inference_job_token); @@ -448,15 +417,7 @@ pub async fn process_single_job( info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", job.id, id, &inference_result_token); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "success"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "success"); - job_iteration_span.set_duration(duration); - - Ok((since_creation_span, job_iteration_span)) + Ok(()) } fn get_timestamp_millis() -> u64 { diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index 9738caa13c..db471482ac 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -45,8 +45,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::list_available_tts_inferenc use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; -use newrelic_telemetry::ClientBuilder; -use newrelic_telemetry::Span; use crate::caching::cache_miss_strategizer::CacheMissStrategizer; use crate::caching::cache_miss_strategizer::CacheMissStrategy; @@ -253,12 +251,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("TTS_MODEL_RECORD_CACHE_MILLIS", 300_000)?, // Five minutes ); - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -301,8 +293,6 @@ async fn main() -> AnyhowResult<()> { tts_sidecar_health_check_client, }, job_stats: JobStats::new(), - newrelic_client, - newrelic_disabled, worker_details: JobWorkerDetails { is_on_prem, worker_hostname: server_hostname.clone(), @@ -344,8 +334,6 @@ async fn main_loop(job_args: JobArgs) { let mut noop_logger = NoOpLogger::new(job_args.no_op_logger_millis as i64); - let mut span_batch = Vec::new(); - let mut sort_by_priority = true; let mut sort_by_priority_count = 0; @@ -411,23 +399,11 @@ async fn main_loop(job_args: JobArgs) { needs_health_check_at_start = false; } - let mut spans = match batch_result { - Ok(spans) => spans, - Err(e) => { - warn!("Error running job batch: {:?}", e); - std::thread::sleep(Duration::from_millis(error_timeout_millis)); - error_timeout_millis += INCREASE_TIMEOUT_MILLIS; - continue; - } - }; - - if !job_args.newrelic_disabled { - span_batch.append(&mut spans); - - if span_batch.len() > 50 { - let spans_to_send = span_batch.split_off(0).into(); - job_args.newrelic_client.send_spans(spans_to_send).await; - } + if let Err(e) = batch_result { + warn!("Error running job batch: {:?}", e); + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; } error_timeout_millis = START_TIMEOUT_MILLIS; // reset @@ -441,7 +417,7 @@ async fn process_jobs( inferencer: &JobArgs, jobs: Vec, needs_health_check_at_start: bool, -) -> AnyhowResult> { +) -> AnyhowResult<()> { if needs_health_check_at_start { maybe_block_on_sidecar_health_check(&inferencer.http_clients.tts_sidecar_health_check_client).await; @@ -449,8 +425,6 @@ async fn process_jobs( let mut maybe_sidecar_health_issue = false; - let mut batch_spans = Vec::new(); - for job in jobs.into_iter() { let model_state_result = ModelState::query_model_and_check_filesystem( &job, @@ -555,44 +529,38 @@ async fn process_jobs( } let result = process_single_job(inferencer, &job, &model_state.model_record).await; - match result { - Ok((span1, span2)) => { - batch_spans.push(span1); - batch_spans.push(span2); - }, - Err(e) => { - warn!("Failure to process job: {:?}", e); + if let Err(e) = result { + warn!("Failure to process job: {:?}", e); - record_failure_and_maybe_slow_down(&inferencer.job_stats); + record_failure_and_maybe_slow_down(&inferencer.job_stats); - maybe_sidecar_health_issue = true; + maybe_sidecar_health_issue = true; - let failure_reason = "failure processing job"; - let internal_debugging_failure_reason = format!("job error: {:?}", e); + let failure_reason = "failure processing job"; + let internal_debugging_failure_reason = format!("job error: {:?}", e); - let _r = mark_tts_inference_job_failure( - &inferencer.mysql_pool, - &job, - failure_reason, - &internal_debugging_failure_reason, - inferencer.job_max_attempts, - &inferencer.get_worker_name(), - ).await; + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + &internal_debugging_failure_reason, + inferencer.job_max_attempts, + &inferencer.get_worker_name(), + ).await; - match e { - ProcessSingleJobError::Other(_) => {} // No-op - ProcessSingleJobError::FilesystemFull => { - // TODO: Refactor - we should stop processing all of these jobs as we'll lose out - // on this entire batch by attempting to clear the filesystem. This should be handled - // in the calling code. - delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; - } + match e { + ProcessSingleJobError::Other(_) => {} // No-op + ProcessSingleJobError::FilesystemFull => { + // TODO: Refactor - we should stop processing all of these jobs as we'll lose out + // on this entire batch by attempting to clear the filesystem. This should be handled + // in the calling code. + delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; } } } } - Ok(batch_spans) + Ok(()) } fn record_failure_and_maybe_slow_down(job_stats: &JobStats) { From 2d59c5c573ff4dae82705268517e41b7a88b4648 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 13:47:12 -0800 Subject: [PATCH 112/136] initial commit --- .env | 78 ----- Cargo.lock | 2 + .../inference_category.rs | 4 + .../inference_model_type.rs | 2 + .../generic_inference_args.rs | 10 + .../payloads/generic_inference_args/mod.rs | 3 +- .../videofilter_payload.rs | 36 ++ crates/service/job/inference_job/Cargo.toml | 2 + .../inference_job/src/job/job_types/mod.rs | 3 +- .../src/job/job_types/videofilter/mod.rs | 1 + .../rerender_a_video/categorize_error.rs | 5 + .../rerender_a_video/download_video_file.rs | 83 +++++ .../videofilter/rerender_a_video/mod.rs | 6 + .../rerender_a_video/process_jobs.rs | 316 ++++++++++++++++++ .../rerender_a_video/rerender_dependencies.rs | 15 + .../rerender_inference_command.rs | 233 +++++++++++++ .../rerender_a_video/validate_job.rs | 85 +++++ .../src/job_specific_dependencies.rs | 9 + .../service/web/storyteller_web/src/routes.rs | 1 + 19 files changed, 814 insertions(+), 80 deletions(-) delete mode 100644 .env create mode 100644 crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs diff --git a/.env b/.env deleted file mode 100644 index b205c3a6b3..0000000000 --- a/.env +++ /dev/null @@ -1,78 +0,0 @@ -SERVER_ENVIRONMENT=development - -DATABASE_URL=mysql://storyteller:password@localhost/storyteller -#DATABASE_URL=mysql://root:root@localhost/storyteller -#DATABASE_URL=mysql://root@localhost/storyteller - -# Diesel-cli reads this for the migration directory to use. -# We've renamed it so the migrations are sorted near the top of the project directory tree -# (since they kind of serve as documentation) -MIGRATION_DIRECTORY=_sql/migrations/ - -# SQLX normally requires a live database to reflect upon at build time. -# This setting turns that on and requires a `sql-data.json` cache file instead. -# See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme -#SQLX_OFFLINE=true - -# NB(bt, 2023-03-20): -# These are used in storyteller-web, image-generation-service, and websocket-gateway. -# I'm moving them to env configs that live under storyteller-web/.env. -# This will break the other services for now. -# # We can't serve HTTPS locally without self-signed certificate nonsense. -# COOKIE_SECURE=false -# COOKIE_DOMAIN=.jungle.horse -# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse - -# Bucket config (non-secret) -REGION_NAME=us-east1 - -TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads -TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models - -W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads -W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public - -PRIVATE_BUCKET_NAME=dev-vocodes-private-uploads -PUBLIC_BUCKET_NAME=dev-vocodes-public - -# Bucket roots -AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio -W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates -W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output - -W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip -W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py -W2L_MODEL_FILENAME=wav2lip_gan.pth - -#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth -W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth -W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 - -W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py - -TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py - -TACOTRON_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TACOTRON_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" -TACOTRON_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_tacotron.py" - -TALKNET_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TALKNET_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_talknet.py" - -HIFIGAN_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" -HIFIGAN_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_hifigan.py" - -TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 - -#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID -#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! -TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt - -TTS_HIFIGAN_VOCODER_MODEL_FILENAME=hifigan_universal_g_02500000.pt -TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt - -# Development key. Permissible to check in. -NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index bec222b29c..d8fddc06d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3063,6 +3063,7 @@ dependencies = [ "log", "media", "memory_caching", + "mime", "mimetypes", "my-workspace-hack", "mysql_queries", @@ -3077,6 +3078,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", + "time 0.3.30", "tokens", "tokio 1.34.0", "tts_common", diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index b38f181460..b96b6fb261 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -25,6 +25,9 @@ pub enum InferenceCategory { #[serde(rename = "voice_conversion")] VoiceConversion, + + #[serde(rename = "rerender_a_video")] + RerenderAVideo, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -38,6 +41,7 @@ impl InferenceCategory { Self::LipsyncAnimation => "lipsync_animation", Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", + Self::RerenderAVideo => "rerender_a_video", } } diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 2a939038d6..5401430bd1 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -31,6 +31,8 @@ pub enum InferenceModelType { Vits, #[serde(rename = "vall_e_x")] VallEX, + #[serde(rename = "rerender_a_video")] + RerenderAVideo, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 541de77aa4..7ba7b0b3ef 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -3,6 +3,7 @@ use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; use crate::payloads::generic_inference_args::tts_payload::TTSArgs; +use crate::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; /// Used to encode extra state for the `generic_inference_jobs` table. @@ -35,6 +36,10 @@ pub enum InferenceCategoryAbbreviated { #[serde(rename = "vc")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(alias = "voice_conversion")] VoiceConversion, + + #[serde(rename = "rr")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "rerender_a_video")] + RerenderAVideo, } #[derive(Serialize, Deserialize, Clone, Copy, Debug)] @@ -84,6 +89,9 @@ pub enum PolymorphicInferenceArgs { #[serde(skip_serializing_if = "Option::is_none")] transpose: Option, }, + + /// Rerender a video. (Short name to save space when serializing.) + Rr(RerenderArgs) } @@ -104,6 +112,7 @@ impl InferenceCategoryAbbreviated { InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, + InferenceCategory::RerenderAVideo => Self::RerenderAVideo, } } @@ -112,6 +121,7 @@ impl InferenceCategoryAbbreviated { Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, + Self::RerenderAVideo => InferenceCategory::RerenderAVideo, } } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs index 37980d38d1..e68fc5a71c 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs @@ -1,3 +1,4 @@ pub mod generic_inference_args; pub mod lipsync_payload; -pub mod tts_payload; \ No newline at end of file +pub mod tts_payload; +pub mod videofilter_payload; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs new file mode 100644 index 0000000000..85335cde79 --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -0,0 +1,36 @@ +/// Video sources can be one of several: +/// - F: media_files (todo) +/// - U: media_uploads (legacy) +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum VideofilterVideoSource { + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media File Token (media_files table) + /// Serde cannot yet rename enum variants. + F(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media Upload Token (media_uploads table) + /// Serde cannot yet rename enum variants. + U(String), +} + + +impl VideofilterVideoSource { + pub fn media_file_token(token: &str) -> Self { + VideofilterVideoSource::F(token.to_string()) + } + pub fn media_upload_token(token: &str) -> Self { + VideofilterVideoSource::U(token.to_string()) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct RerenderArgs { + #[serde(rename = "vs")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_video_source: Option, + + #[serde(rename = "sd")] + #[serde(skip_serializing_if = "Option::is_none")] + pub sd_model: String, +} diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index d0008b9eac..65713e40b0 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -79,3 +79,5 @@ serde_json = "1.0.108" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +time = "0.3.30" +mime = "0.3.17" diff --git a/crates/service/job/inference_job/src/job/job_types/mod.rs b/crates/service/job/inference_job/src/job/job_types/mod.rs index f21a93a492..a688e281ba 100644 --- a/crates/service/job/inference_job/src/job/job_types/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/mod.rs @@ -1,3 +1,4 @@ pub mod lipsync; pub mod tts; -pub mod vc; \ No newline at end of file +pub mod vc; +pub(crate) mod videofilter; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs new file mode 100644 index 0000000000..b8f301c73d --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -0,0 +1 @@ +pub(crate) mod rerender_a_video; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs new file mode 100644 index 0000000000..a1effe90c7 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs @@ -0,0 +1,5 @@ +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +pub fn categorize_error(stderr_contents: &str) -> Option { + None +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs new file mode 100644 index 0000000000..344592bd08 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -0,0 +1,83 @@ +use std::path::PathBuf; + +use anyhow::anyhow; +use log::{error, warn}; +use sqlx::MySqlPool; +use tempdir::TempDir; + +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use cloud_storage::bucket_client::BucketClient; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; + +pub struct VideoFile { + pub filesystem_path: PathBuf, +} + +pub async fn download_video_file( + video_source: &VideofilterVideoSource, + public_bucket_client: &BucketClient, + job_progress_reporter: &mut Box, + job: &AvailableInferenceJob, + temp_dir_creator: &ScopedTempDirCreator, + work_temp_dir: &TempDir, + mysql_pool: &MySqlPool, +) -> Result { + + let bucket_object_path; + + match video_source { + VideofilterVideoSource::F(_media_file_token) => { + // TODO(bt, 2023-09-08): Implement + return Err(ProcessSingleJobError::NotYetImplemented) + } + VideofilterVideoSource::U(media_upload_token) => { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + } + } + + let downloaded_filesystem_path = work_temp_dir.path().join("video.mp4"); + + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "video file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, + job_progress_reporter, + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; + + Ok(VideoFile { + filesystem_path: downloaded_filesystem_path, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs new file mode 100644 index 0000000000..152c2840da --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -0,0 +1,6 @@ +mod process_jobs; +mod validate_job; +mod download_video_file; +mod rerender_inference_command; +pub(crate) mod rerender_dependencies; +mod categorize_error; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs new file mode 100644 index 0000000000..f26de327c6 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs @@ -0,0 +1,316 @@ +use std::fs::read_to_string; +use std::time::Instant; + +use anyhow::anyhow; +use log::{error, info, warn}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; +use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; +use tokens::tokens::users::UserToken; + +use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; +// use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; +use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; +use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; +use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; +use crate::job_dependencies::JobDependencies; +use crate::util::common_commands::ffmpeg_logo_watermark_command; + +/// The maximum that either width or height can be +const MAX_DIMENSION : u32 = 1500; + +const BUCKET_FILE_PREFIX: &str = "fakeyou_"; +const BUCKET_FILE_EXTENSION: &str = ".mp4"; + +pub struct RerenderProcessJobArgs<'a> { + pub job_dependencies: &'a JobDependencies, + pub job: &'a AvailableInferenceJob, +} + +pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { + let job = args.job; + let deps = args.job_dependencies; + + let mut job_progress_reporter = args.job_dependencies + .clients + .job_progress_reporter + .new_generic_inference(job.inference_job_token.as_str()) + .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; + + let model_dependencies = args + .job_dependencies + .job + .job_specific_dependencies + .maybe_rerender_dependencies + .as_ref() + .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing Rerender dependencies".to_string())))?; + + // ==================== UNPACK + VALIDATE INFERENCE ARGS ==================== // + + let job_args = validate_job(job)?; + + // ==================== TEMP DIR ==================== // + + let work_temp_dir = format!("temp_rerender_inference_{}", job.id.0); + + // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem. + let work_temp_dir = args.job_dependencies + .fs + .scoped_temp_dir_creator_for_work + .new_tempdir(&work_temp_dir) + .map_err(|e| ProcessSingleJobError::from_io_error(e))?; + + + // ==================== QUERY AND DOWNLOAD FILES ==================== // + + let video_path = download_video_file( + &job_args.video_source, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir, + &deps.db.mysql_pool + ).await?; + + info!("Downloaded video file: {:?}", image_path.filesystem_path); + + // ==================== TRANSCODE MEDIA (IF NECESSARY) ==================== // + + let mut usable_video_path = video_path.filesystem_path.clone(); + // + // if let Some((width, height)) = job_args.width.zip(job_args.height) { + // info!("Requested image resize to {}x{} ...", width, height); + // + // let mut width = width.min(MAX_DIMENSION); + // let mut height = height.min(MAX_DIMENSION); + // + // if job_args.enhancer.is_some() { + // // The enhancer will double the size of the image, so we'll downsize the image by half + // // if it's enabled. + // width = width / 2; + // height = height / 2; + // info!("Enhancer will be doubling image size, so cutting frame size to {}x{}.", width, height) + // } + // + // info!("Resizing image to {}x{} ...", width, height); + // + // let result = resize_image( + // &image_path.filesystem_path, + // &work_temp_dir, + // width, + // height + // ); + // + // match result { + // Err(err) => { + // error!("Image resize failure: {:?}", err); + // safe_delete_temp_file(&audio_path.filesystem_path); + // safe_delete_temp_file(&image_path.filesystem_path); + // safe_delete_temp_directory(&work_temp_dir); + // return Err(ProcessSingleJobError::Other(anyhow!("image resize failure: {:?}", err))); + // } + // Ok(resized_image_path) => { + // usable_image_path = resized_image_path; + // } + // } + // } + + info!("Used video file: {:?}", usable_video_path); + + // ==================== SETUP FOR INFERENCE ==================== // + + info!("Ready for Rerender A Video inference..."); + + job_progress_reporter.log_status("running inference") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let output_video_fs_path = work_temp_dir.path().join("blend.mp4"); + + info!("Running Rerender A Video inference..."); + + info!("Expected output video filename: {:?}", &output_video_fs_path); + + // TODO: Limit output length for non-premium (???) + + let maybe_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + // ==================== WRITE CONFIG TO DISK ==================== // + + let config_path = work_temp_dir.path().join("config.json"); + + // TODO + + // ==================== RUN INFERENCE SCRIPT ==================== // + + let workdir = work_temp_dir.path().to_path_buf(); + let stderr_output_file = work_temp_dir.path().join("stderr.txt"); + let inference_start_time = Instant::now(); + + let command_exit_status = model_dependencies + .inference_command + .execute_inference(InferenceArgs { + config_file: &config_path, + stderr_output_file: &stderr_output_file, + }); + + let inference_duration = Instant::now().duration_since(inference_start_time); + + info!("Inference took duration to complete: {:?}", &inference_duration); + + if !command_exit_status.is_success() { + error!("Inference failed: {:?}", command_exit_status); + + let mut error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + + // match categorize_error(&contents) { + // Some(ProcessSingleJobError::FaceDetectionFailure) => { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + // } + } + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&config_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + return Err(error); + } + + // ==================== CHECK NON-WATERMARKED RESULT ==================== // + + info!("Checking that output file exists: {:?} ...", output_video_fs_path); + + check_file_exists(&output_video_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== OPTIONAL WATERMARK ==================== // + + let mut finished_file = output_video_fs_path.clone(); + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that output watermark file exists: {:?} ...", finished_file); + check_file_exists(&finished_file).map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Interrogating result file size ..."); + + let file_size_bytes = file_size(&finished_file) + .map_err(|err| ProcessSingleJobError::Other(err))?; + + info!("Interrogating result mimetype ..."); + + let mimetype = get_mimetype_for_file(&finished_file) + .map_err(|err| ProcessSingleJobError::from_io_error(err))? + .map(|mime| mime.to_string()) + .ok_or(ProcessSingleJobError::Other(anyhow!("Mimetype could not be determined")))?; + + info!("Calculating sha256..."); + + let file_checksum = sha256_hash_file(&finished_file) + .map_err(|err| { + ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err)) + })?; + + // ==================== UPLOAD AUDIO TO BUCKET ==================== // + + job_progress_reporter.log_status("uploading result") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let result_bucket_location = MediaFileBucketPath::generate_new( + Some(BUCKET_FILE_PREFIX), + Some(BUCKET_FILE_EXTENSION)); + + let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf(); + + info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf); + + info!("Uploading media ..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &result_bucket_object_pathbuf, + &finished_file, + &mimetype) // TODO: We should check the mimetype to make sure bad payloads can't get uploaded + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== DELETE TEMP FILES ==================== // + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&config_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&work_temp_dir); + + // ==================== SAVE RECORDS ==================== // + + info!("Saving SadTalker result (media_files table record) ..."); + + let (media_file_token, id) = insert_media_file_from_face_animation(InsertArgs { + pool: &args.job_dependencies.db.mysql_pool, + job: &job, + maybe_mime_type: Some(&mimetype), + file_size_bytes, + sha256_checksum: &file_checksum, + public_bucket_directory_hash: result_bucket_location.get_object_hash(), + maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX), + maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION), + is_on_prem: args.job_dependencies.job.info.container.is_on_prem, + worker_hostname: &args.job_dependencies.job.info.container.hostname, + worker_cluster: &args.job_dependencies.job.info.container.cluster_name, + }) + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("SadTalker Done."); + + // TODO: Update upstream to be strongly typed + let maybe_user_token = job.maybe_creator_user_token.as_deref() + .map(|token| UserToken::new_from_str(token)); + + args.job_dependencies.clients.firehose_publisher.lipsync_animation_finished( + maybe_user_token.as_ref(), + &job.inference_job_token, + media_file_token.as_str()) + .await + .map_err(|e| { + error!("error publishing event: {:?}", e); + ProcessSingleJobError::Other(anyhow!("error publishing event")) + })?; + + job_progress_reporter.log_status("done") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", + job.id, id, &media_file_token); + + Ok(JobSuccessResult { + maybe_result_entity: Some(ResultEntity { + entity_type: InferenceResultType::MediaFile, + entity_token: media_file_token.to_string(), + }), + inference_duration, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs new file mode 100644 index 0000000000..c50fe62ece --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs @@ -0,0 +1,15 @@ +use errors::AnyhowResult; + +use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::RerenderInferenceCommand; + +pub struct RerenderDependencies { + pub inference_command: RerenderInferenceCommand, +} + +impl RerenderDependencies { + pub fn setup() -> AnyhowResult { + Ok(Self { + inference_command: RerenderInferenceCommand::from_env()?, + }) + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs new file mode 100644 index 0000000000..2c221e2b28 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -0,0 +1,233 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use log::info; +use once_cell::sync::Lazy; +use subprocess::{Popen, PopenConfig, Redirection}; + +use errors::AnyhowResult; +use filesys::path_to_string::path_to_string; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; + +use crate::job::job_loop::command_exit_status::CommandExitStatus; + +// These environment vars are not copied over to the subprocess +// TODO/FIXME(bt, 2023-05-28): This is horrific security! +static IGNORED_ENVIRONMENT_VARS : Lazy> = Lazy::new(|| { + let env_var_names= [ + "MYSQL_URL", + "ACCESS_KEY", + "SECRET_KEY", + "NEWRELIC_API_KEY", + ]; + + env_var_names.iter() + .map(|value| value.to_string()) + .collect::>() +}); + +#[derive(Clone)] +pub struct RerenderInferenceCommand { + /// Where the code lives + rerender_root_code_directory: PathBuf, + + /// A single executable script or a much larger bash command. + executable_or_command: ExecutableOrCommand, + + /// Config file to use + config_path: PathBuf, + + /// eg. `source python/bin/activate` + maybe_virtual_env_activation_command: Option, + + /// If this is run under Docker (eg. in development), these are the options. + maybe_docker_options: Option, + + /// If the execution should be ended after a certain point. + maybe_execution_timeout: Option, + +} + +#[derive(Clone)] +pub enum ExecutableOrCommand { + /// Eg. `inference.py` + Executable(PathBuf), + + /// Eg. `python3 inference.py` + Command(String), +} + +pub struct InferenceArgs<'s, P: AsRef> { + pub config_file: P, + pub stderr_output_file: &'s Path, +} + +impl RerenderInferenceCommand { + pub fn new>( + rerender_root_code_directory: PathBuf, + executable_or_command: ExecutableOrCommand, + config_path: PathBuf, + maybe_virtual_env_activation_command: Option, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + ) -> Self { + Self { + rerender_root_code_directory: rerender_root_code_directory.as_ref().to_path_buf(), + executable_or_command, + config_path: config_path.as_ref().to_path_buf(), + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_docker_options, + maybe_execution_timeout, + } + } + + pub fn from_env() -> AnyhowResult { + let rerender_root_code_directory = easyenv::get_env_pathbuf_required( + "RERENDER_INFERENCE_ROOT_DIRECTORY")?; + + let config_path = easyenv::get_env_pathbuf_required( + "RERENDER_INFERENCE_CONFIG_PATH")?; + + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( + "RERENDER_INFERENCE_MAYBE_VENV_COMMAND"); + + let maybe_execution_timeout = + easyenv::get_env_duration_seconds_optional("RERENDER_TIMEOUT_SECONDS"); + + let maybe_docker_options = easyenv::get_env_string_optional( + "RERENDER_INFERENCE_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_environment_variables: None, + maybe_gpu: Some(DockerGpu::All), + } + }); + + Ok(Self { + rerender_root_code_directory, + executable_or_command, + config_path, + maybe_virtual_env_activation_command, + maybe_docker_options, + maybe_execution_timeout, + }) + } + + pub fn execute_inference>( + &self, + args: InferenceArgs

, + ) -> CommandExitStatus { + match self.do_execute_inference(args) { + Ok(exit_status) => exit_status, + Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) }, + } + } + + fn do_execute_inference>( + &self, + args: InferenceArgs

, + ) -> AnyhowResult { + + let mut command = String::new(); + command.push_str(&format!("cd {}", path_to_string(&self.rerender_root_code_directory))); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + + command.push_str(" && "); + + match self.executable_or_command { + ExecutableOrCommand::Executable(ref executable) => { + command.push_str(&path_to_string(executable)); + command.push_str(" "); + } + ExecutableOrCommand::Command(ref cmd) => { + command.push_str(cmd); + command.push_str(" "); + } + } + + // ===== Begin Python Args ===== + + command.push_str(" --cfg "); + command.push_str(&path_to_string(args.config_file.as_ref())); + + // ===== End Python Args ===== + + if let Some(docker_options) = self.maybe_docker_options.as_ref() { + command = docker_options.to_command_string(&command); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + let mut env_vars = Vec::new(); + + // Copy all environment variables from the parent process. + // This is necessary to send all the kubernetes settings for Nvidia / CUDA. + for (env_key, env_value) in env::vars() { + if IGNORED_ENVIRONMENT_VARS.contains(&env_key) { + continue; + } + env_vars.push(( + OsString::from(env_key), + OsString::from(env_value), + )); + } + + let mut config = PopenConfig::default(); + + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + + if !env_vars.is_empty() { + config.env = Some(env_vars); + } + + let mut p = Popen::create(&command_parts, config)?; + + info!("Subprocess PID: {:?}", p.pid()); + + match self.maybe_execution_timeout { + None => { + let exit_status = p.wait()?; + info!("Subprocess exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + Some(timeout) => { + info!("Executing with timeout: {:?}", &timeout); + let exit_status = p.wait_timeout(timeout.clone())?; + + match exit_status { + None => { + // NB: If the program didn't successfully terminate, kill it. + info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout); + let _r = p.terminate()?; + Ok(CommandExitStatus::Timeout) + } + Some(exit_status) => { + info!("Subprocess timed wait exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + } + } + } + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs new file mode 100644 index 0000000000..127a601f30 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs @@ -0,0 +1,85 @@ +use anyhow::anyhow; + +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::{VideofilterVideoSource}; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +pub struct JobArgs<'a> { + pub video_source: &'a VideofilterVideoSource, + pub sd_model: &'a str, +} + +pub fn validate_job(job: &AvailableInferenceJob) -> Result { + let inference_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + let inference_category = job.maybe_inference_args + .as_ref() + .map(|args| args.inference_category) + .flatten(); + + match inference_category { + Some(InferenceCategoryAbbreviated::RerenderAVideo) => {}, // Valid + Some(category) => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category))); + }, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference category for job!"))); + } + }; + + let inference_args = match inference_args { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!"))); + } + }; + + let inference_args = match inference_args { + PolymorphicInferenceArgs::Rr(inference_args) => inference_args, + _ => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!"))); + } + }; + + let video_source = match &inference_args.maybe_video_source { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no video source!"))); + } + }; + + let sd_model = match &inference_args.sd_model { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); + } + }; + + + // let preprocess = match inference_args.maybe_preprocess { + // None => Some("full".to_string()), + // Some(Preprocess::F) => Some("full".to_string()), + // Some(Preprocess::EF) => Some("extfull".to_string()), + // Some(Preprocess::C) => Some("crop".to_string()), + // Some(Preprocess::EC) => Some("extcrop".to_string()), + // }; + // + // let enhancer = match inference_args.maybe_face_enhancer { + // None => None, + // Some(FaceEnhancer::G) => Some("gfpgan".to_string()), + // Some(FaceEnhancer::R) => Some("RestoreFormer".to_string()), + // }; + // + // let mut width = inference_args.maybe_resize_width; + // let mut height = inference_args.maybe_resize_height; + + Ok(JobArgs { + video_source, + sd_model, + }) +} diff --git a/crates/service/job/inference_job/src/job_specific_dependencies.rs b/crates/service/job/inference_job/src/job_specific_dependencies.rs index ddd313a46a..a863b3872c 100644 --- a/crates/service/job/inference_job/src/job_specific_dependencies.rs +++ b/crates/service/job/inference_job/src/job_specific_dependencies.rs @@ -7,6 +7,7 @@ use crate::job::job_types::tts::vall_e_x::vall_e_x_dependencies::VallExDependenc use crate::job::job_types::tts::vits::vits_dependencies::VitsDependencies; use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies; use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies; +use crate::job::job_types::videofilter::rerender_a_video::rerender_dependencies::RerenderDependencies; use crate::util::scoped_execution::ScopedExecution; pub struct JobSpecificDependencies { @@ -16,6 +17,7 @@ pub struct JobSpecificDependencies { pub maybe_tacotron2_dependencies: Option, pub maybe_vall_e_x_dependencies: Option, pub maybe_vits_dependencies: Option, + pub maybe_rerender_dependencies: Option, } impl JobSpecificDependencies { @@ -27,6 +29,7 @@ impl JobSpecificDependencies { let mut maybe_tacotron2_dependencies = None; let mut maybe_vall_e_x_dependencies = None; let mut maybe_vits_dependencies = None; + let mut maybe_rerender_dependencies = None; if scoped_execution.can_run_job(InferenceModelType::RvcV2) { print_with_space("Setting RVCv2 dependencies..."); @@ -58,6 +61,11 @@ impl JobSpecificDependencies { maybe_vits_dependencies = Some(VitsDependencies::setup()?); } + if scoped_execution.can_run_job(InferenceModelType::RerenderAVideo) { + print_with_space("Setting Rerender dependencies..."); + maybe_rerender_dependencies = Some(RerenderDependencies::setup()?); + } + Ok(JobSpecificDependencies { maybe_rvc_v2_dependencies, maybe_sad_talker_dependencies, @@ -65,6 +73,7 @@ impl JobSpecificDependencies { maybe_tacotron2_dependencies, maybe_vall_e_x_dependencies, maybe_vits_dependencies, + maybe_rerender_dependencies, }) } } diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9bc7775457..f612d034f7 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -206,6 +206,7 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> let mut app = RouteBuilder::from_app(app) .add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler) + .add_post("/v1/animation/rerender/create", enqueue_rerender_animation_handler) .into_app(); // ==================== "Generic" Inference ==================== From 43030402ca20f41b72ff73b9f22b2c33e3b371fb Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 14:50:50 -0800 Subject: [PATCH 113/136] update --- Cargo.lock | 1 + .../inference_model_type.rs | 3 +- .../generic_inference_args.rs | 2 +- .../videofilter_payload.rs | 2 +- .../web/get_inference_job_status.rs | 2 + .../src/job/job_loop/process_single_job.rs | 3 + .../job/job_types/vc/rvc_v2/process_job.rs | 2 + .../job_types/vc/so_vits_svc/process_job.rs | 2 + .../src/job/job_types/videofilter/mod.rs | 3 +- .../videofilter/process_single_vf_job.rs | 63 +++++ .../videofilter/rerender_a_video/mod.rs | 2 +- .../rerender_a_video/process_jobs.rs | 2 +- .../rerender_a_video/validate_job.rs | 2 +- .../traits/internal_session_cache_purge.rs | 1 + crates/service/web/storyteller_web/Cargo.toml | 1 + .../storyteller_web/src/configs/plans/plan.rs | 19 ++ .../animation/enqueue_rerender_animation.rs | 237 ++++++++++++++++++ .../http_server/endpoints/animation/mod.rs | 3 +- .../inference_job/get_inference_job_status.rs | 8 + .../service/web/storyteller_web/src/routes.rs | 1 + 20 files changed, 351 insertions(+), 8 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs diff --git a/Cargo.lock b/Cargo.lock index d8fddc06d1..215225d64d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6025,6 +6025,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http", "http_server_common", "hyper", "infer 0.5.0", diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 5401430bd1..0a30156539 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -48,7 +48,8 @@ impl InferenceModelType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", - Self::VallEX => "vall_e_x" + Self::VallEX => "vall_e_x", + Self::RerenderAVideo => "rerender_a_video", } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 7ba7b0b3ef..f34a4469c3 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -3,7 +3,7 @@ use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; use crate::payloads::generic_inference_args::tts_payload::TTSArgs; -use crate::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; +use crate::payloads::generic_inference_args::videofilter_payload::{RerenderArgs}; /// Used to encode extra state for the `generic_inference_jobs` table. diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs index 85335cde79..12d316933f 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -32,5 +32,5 @@ pub struct RerenderArgs { #[serde(rename = "sd")] #[serde(skip_serializing_if = "Option::is_none")] - pub sd_model: String, + pub maybe_sd_model: Option, } diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index a59fd2e5d2..247eb2b9c0 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -177,6 +177,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), + InferenceCategory::RerenderAVideo => Some("rerender animation"), }; // NB: A bit of a hack. We store TTS results with a full path. @@ -185,6 +186,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), + InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), }; // NB: We've moved voice conversion out of their own table and into media_files diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 2d8f04efa9..6dc5a4f7d4 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -182,6 +182,9 @@ async fn do_process_single_job( InferenceCategory::VoiceConversion => { process_single_vc_job(job_dependencies, job).await? } + InferenceCategory::RerenderAVideo => { + process_single_rerender_job(job_dependencies, job).await? + } }; let maybe_entity_type = job_success_result.maybe_result_entity diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 74496b6abc..082dd95eac 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -239,6 +239,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { override_f0_method, .. } => *override_f0_method, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); @@ -247,6 +248,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index f5d7fa1b89..69337d467f 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -168,6 +168,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten() .unwrap_or(false); @@ -177,6 +178,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs index b8f301c73d..7f70f41199 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -1 +1,2 @@ -pub(crate) mod rerender_a_video; \ No newline at end of file +pub(crate) mod rerender_a_video; +mod process_single_vf_job; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs new file mode 100644 index 0000000000..eee4f2cd8d --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -0,0 +1,63 @@ +use std::time::Duration; + +use anyhow::anyhow; +use log::{error, info}; + +use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::job::job_loop::job_success_result::JobSuccessResult; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; +use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; +use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job_dependencies::JobDependencies; + +pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { + let maybe_media_upload_token = job.maybe_input_source_token + .as_deref() + .map(|token| MediaUploadToken::new_from_str(token)); + + let media_upload_token = match maybe_media_upload_token { + None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media upload for vc job: {:?}", job.inference_job_token))), + Some(token) => token, + }; + + let maybe_media_upload_result = + get_media_upload_for_inference(&media_upload_token, &job_dependencies.db.mysql_pool).await; + + let media_upload = match maybe_media_upload_result { + Ok(Some(media_upload)) => media_upload, + Ok(None) => { + error!("no media upload record found for token: {:?}", media_upload_token); + return Err(ProcessSingleJobError::Other(anyhow!("no media upload record found for token: {:?}", media_upload_token))); + }, + Err(err) => { + error!("error fetching media upload record from db: {:?}", err); + return Err(ProcessSingleJobError::Other(err)); + }, + }; + + info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); + info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); + info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); + + + + let job_success_result = match vc_model.model_type { + VoiceConversionModelType::RvcV2 => { + rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + job_dependencies, + job, + vc_model: &vc_model, + media_upload_token: &media_upload_token, + media_upload: &media_upload, + }).await? + } + }; + + Ok(job_success_result) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 152c2840da..711397cfb1 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -3,4 +3,4 @@ mod validate_job; mod download_video_file; mod rerender_inference_command; pub(crate) mod rerender_dependencies; -mod categorize_error; \ No newline at end of file +mod categorize_error; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs index f26de327c6..affec1aee3 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs @@ -83,7 +83,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result Result args, None => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index d125c53d7f..69abd86b95 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,4 +1,5 @@ use actix_web::HttpRequest; +use mockall::automock; /// Allows us to purge session caches that might keep stale premium plan/billing data around #[cfg_attr(test, automock)] diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index df7cab336c..f015262c17 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -164,6 +164,7 @@ pulldown-cmark = "0.9.3" async-trait = "0.1.58" utoipa = { version = "4", features = ["actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } +http = "0.2.11" [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 67dd178cc1..80d100a9fe 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -49,6 +49,9 @@ pub struct Plan { can_remove_visual_watermarks: bool, + // ========== Features for Videofilters ========== + rerender_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -91,6 +94,7 @@ impl Plan { is_development_plan: builder.is_development_plan, is_synthetic_plan: builder.is_synthetic_plan, lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive, + rerender_requires_frontend_keepalive: builder.rerender_requires_frontend_keepalive, can_remove_visual_watermarks: builder.can_remove_visual_watermarks, tts_base_priority_level: builder.tts_base_priority_level, tts_max_duration: builder.tts_max_duration, @@ -146,6 +150,10 @@ impl Plan { self.lipsync_requires_frontend_keepalive } + pub fn rerender_requires_frontent_keepalive(&self) -> bool { + self.rerender_requires_frontend_keepalive + } + pub fn can_remove_visual_watermarks(&self) -> bool { self.can_remove_visual_watermarks } @@ -192,6 +200,9 @@ pub struct PlanBuilder { can_remove_visual_watermarks: bool, + // ========== Features for Videofilters ========== + rerender_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -239,6 +250,9 @@ impl PlanBuilder { lipsync_requires_frontend_keepalive: true, can_remove_visual_watermarks: false, + // Videofilters + rerender_requires_frontend_keepalive: true, + // TTS tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL, tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS), @@ -304,6 +318,11 @@ impl PlanBuilder { self } + pub fn rerender_requires_frontend_keepalive(mut self, value: bool) -> Self { + self.rerender_requires_frontend_keepalive = value; + self + } + pub fn can_remove_visual_watermarks(mut self, value: bool) -> Self { self.can_remove_visual_watermarks = value; self diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs new file mode 100644 index 0000000000..1519e5332b --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -0,0 +1,237 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use log::{info, warn}; + +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use enums::common::visibility::Visibility; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::{RerenderArgs, VideofilterVideoSource}; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::media_uploads::MediaUploadToken; +use tokens::tokens::users::UserToken; + +use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// Debug requests can get routed to special "debug-only" workers, which can +/// be used to trial new code, run debugging, etc. +const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; + +/// The routing tag header can send workloads to particular k8s hosts. +/// This is useful for catching the live logs or intercepting the job. +const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; + +#[derive(Deserialize)] +pub struct EnqueueRerenderAnimationRequest { + uuid_idempotency_token: String, + + video_source: VideoSource, + sd_model: String, + creator_set_visibility: Option, +} + +/// Treated as an enum. Only one of these may be set. +#[derive(Deserialize)] +pub struct VideoSource { + maybe_media_file_token: Option, + maybe_media_upload_token: Option, +} + +#[derive(Serialize)] +pub struct EnqueueRerenderAnimationSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueRerenderAnimationError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueRerenderAnimationError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueRerenderAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueRerenderAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueRerenderAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueRerenderAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueRerenderAnimationError::BadInput(reason) => reason.to_string(), + EnqueueRerenderAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueRerenderAnimationError::ServerError => "server error".to_string(), + EnqueueRerenderAnimationError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl std::fmt::Display for EnqueueRerenderAnimationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn enqueue_rerender_animation_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let mut maybe_user_token : Option = None; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueRerenderAnimationError::ServerError + })?; + + let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); + + // ==================== USER SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EnqueueRerenderAnimationError::ServerError + })?; + + if let Some(user_session) = maybe_user_session.as_ref() { + maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token)); + } + + // TODO: Plan should handle "first anonymous use" and "investor" cases. + let plan = get_correct_plan_for_session( + server_state.server_environment, + maybe_user_session.as_ref()); + + // TODO: Separate priority for animation. + let priority_level = plan.web_vc_base_priority_level(); + + // ==================== DEBUG MODE + ROUTING TAG ==================== // + + let is_debug_request = + get_request_header_optional(&http_request, DEBUG_HEADER_NAME) + .is_some(); + + let maybe_routing_tag= + get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) + .map(|routing_tag| routing_tag.trim().to_string()); + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueRerenderAnimationError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueRerenderAnimationError::RateLimited); + } + + // ==================== LOOK UP MODEL INFO ==================== // + + // TODO(bt): CHECK DATABASE FOR TOKENS! + + let video_source: VideofilterVideoSource = { + if let Some(ref token) = request.video_source.maybe_media_file_token { + VideofilterVideoSource::media_file_token(token.as_str()) + } else if let Some(ref token) = request.video_source.maybe_media_upload_token { + VideofilterVideoSource::media_upload_token(token.as_str()) + } else { + return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); + } + }; + + let ip_address = get_request_ip(&http_request); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session: &users_component::utils::user_session_extended::UserSessionExtended| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(Visibility::Public); + + let inference_args = RerenderArgs { + maybe_video_source: Some(video_source), + maybe_sd_model: Some(request.sd_model.clone()), + }; + + info!("Creating rerender animation job record..."); + + let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_category: InferenceCategory::RerenderAVideo, + maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference + maybe_model_token: None, // NB: Model is static during inference + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? + maybe_raw_inference_text: None, // No text + maybe_max_duration_seconds: None, + maybe_inference_args: Some(GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo), + args: Some(PolymorphicInferenceArgs::Rr(inference_args)), + }), + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + requires_keepalive: plan.rerender_requires_frontent_keepalive(), + is_debug_request, + maybe_routing_tag: maybe_routing_tag.as_deref(), + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match query_result { + Ok((job_token, _id)) => job_token, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueRerenderAnimationError::ServerError); + } + }; + + let response: EnqueueRerenderAnimationSuccessResponse = EnqueueRerenderAnimationSuccessResponse { + success: true, + inference_job_token: job_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| EnqueueRerenderAnimationError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs index 1102d9357d..5a9c5c080e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -1 +1,2 @@ -pub mod enqueue_lipsync_animation; \ No newline at end of file +pub mod enqueue_lipsync_animation; +pub mod enqueue_rerender_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 96be5ed7d7..d227585c20 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -256,6 +256,14 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } + InferenceCategory::RerenderAVideo => { + MediaFileBucketPath::from_object_hash( + &result_details.public_bucket_location_or_hash, + result_details.maybe_media_file_public_bucket_prefix.as_deref(), + result_details.maybe_media_file_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string() + } InferenceCategory::TextToSpeech => { match result_details.entity_type.as_str() { "media_file" => { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index f612d034f7..96be089846 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -12,6 +12,7 @@ use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler; +use crate::http_server::endpoints::animation::enqueue_rerender_animation::enqueue_rerender_animation_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; From 14432cce83a8dde04613d0a8fc45a8b7bbf0e6fd Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 15:12:34 -0800 Subject: [PATCH 114/136] fix errors --- .../src/job/job_loop/process_single_job.rs | 3 +- .../src/job/job_types/videofilter/mod.rs | 2 +- .../videofilter/process_single_vf_job.rs | 35 ++++++++++--------- .../videofilter/rerender_a_video/mod.rs | 2 +- .../{process_jobs.rs => process_job.rs} | 0 .../rerender_inference_command.rs | 20 +++++++++-- .../traits/internal_session_cache_purge.rs | 4 +-- 7 files changed, 41 insertions(+), 25 deletions(-) rename crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/{process_jobs.rs => process_job.rs} (100%) diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 6dc5a4f7d4..c7edcdbb7f 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -16,6 +16,7 @@ use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSucce use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job; +use crate::job::job_types::videofilter::process_single_vf_job::process_single_rr_job; use crate::job_dependencies::JobDependencies; pub async fn process_single_job( @@ -183,7 +184,7 @@ async fn do_process_single_job( process_single_vc_job(job_dependencies, job).await? } InferenceCategory::RerenderAVideo => { - process_single_rerender_job(job_dependencies, job).await? + process_single_rr_job(job_dependencies, job).await? } }; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs index 7f70f41199..305ceb8a28 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -1,2 +1,2 @@ pub(crate) mod rerender_a_video; -mod process_single_vf_job; +pub(crate) mod process_single_vf_job; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index eee4f2cd8d..6a35427d0e 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,19 +1,13 @@ -use std::time::Duration; - use anyhow::anyhow; use log::{error, info}; -use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; -use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; -use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; -use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job::job_types::videofilter::rerender_a_video; use crate::job_dependencies::JobDependencies; pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { @@ -47,17 +41,24 @@ pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &Ava - let job_success_result = match vc_model.model_type { - VoiceConversionModelType::RvcV2 => { - rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { - job_dependencies, - job, - vc_model: &vc_model, - media_upload_token: &media_upload_token, - media_upload: &media_upload, - }).await? + // let job_success_result = match vc_model.model_type { + // VoiceConversionModelType::RvcV2 => { + // rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + // job_dependencies, + // job, + // vc_model: &vc_model, + // media_upload_token: &media_upload_token, + // media_upload: &media_upload, + // }).await? + // } + // }; + + let job_success_result = rerender_a_video::process_job::process_job( + rerender_a_video::process_job::RerenderProcessJobArgs { + job_dependencies, + job } - }; + ).await?; Ok(job_success_result) } diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 711397cfb1..28c9dea48a 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -1,4 +1,4 @@ -mod process_jobs; +pub(crate) mod process_job; mod validate_job; mod download_video_file; mod rerender_inference_command; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs similarity index 100% rename from crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs rename to crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs index 2c221e2b28..8b0b011893 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -77,9 +77,9 @@ impl RerenderInferenceCommand { maybe_execution_timeout: Option, ) -> Self { Self { - rerender_root_code_directory: rerender_root_code_directory.as_ref().to_path_buf(), + rerender_root_code_directory: rerender_root_code_directory.clone(), executable_or_command, - config_path: config_path.as_ref().to_path_buf(), + config_path: config_path.clone(), maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), maybe_docker_options, maybe_execution_timeout, @@ -93,6 +93,20 @@ impl RerenderInferenceCommand { let config_path = easyenv::get_env_pathbuf_required( "RERENDER_INFERENCE_CONFIG_PATH")?; + let executable_or_command = match easyenv::get_env_string_optional( + "RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND") { + None => { + return Err(anyhow!("RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND is required")); + } + Some(executable_or_command) => { + if executable_or_command.contains(" ") { + ExecutableOrCommand::Command(executable_or_command) + } else { + ExecutableOrCommand::Executable(PathBuf::from(executable_or_command)) + } + } + }; + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( "RERENDER_INFERENCE_MAYBE_VENV_COMMAND"); @@ -192,7 +206,7 @@ impl RerenderInferenceCommand { let mut config = PopenConfig::default(); - info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_os_str()); let stderr_file = File::create(&args.stderr_output_file)?; config.stderr = Redirection::File(stderr_file); diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index 69abd86b95..cc4df2852d 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,8 +1,8 @@ use actix_web::HttpRequest; -use mockall::automock; +// use mockall::automock; /// Allows us to purge session caches that might keep stale premium plan/billing data around -#[cfg_attr(test, automock)] +// #[cfg_attr(test, automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. From e5a85c752821c75f610f0dc2bd794dbc4974edfd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 18:56:02 -0500 Subject: [PATCH 115/136] fix failing doctest --- crates/vendor/limitation/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs index 649b30eb46..b40cf6b1fe 100644 --- a/crates/vendor/limitation/src/lib.rs +++ b/crates/vendor/limitation/src/lib.rs @@ -42,9 +42,12 @@ //! // The key to count and track //! let key = "10.0.0.5"; //! +//! // TODO(bt,2023-12-12): This doctest is so far behind in `futures` versions and needs to be updated. +//! // Tokio should be executing the future and evaluating the result. //! // Start and run a Tokio runtime to drive the Future to completion -//! tokio::run( -//! limiter +//! //let mut rt = tokio::runtime::Runtime::new().unwrap(); +//! //rt.block_on(...)//! +//! let _r = limiter //! // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` //! // containing a Status if the limit has been exceeded //! .count(key) @@ -57,8 +60,9 @@ //! .and_then(|status| { //! println!("ok: {:?}", status); //! Ok(()) -//! }), -//! ); +//! }) +//! .wait(); +//! //); //! # Ok::<(), limitation::Error>(()) //! ``` //! From e375c8090c658bd52ee6d8cd73b0078bdf457343 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 19:21:42 -0500 Subject: [PATCH 116/136] fix IDE handling (code compiled and tested, but IDE unhappy) --- .../billing/src/stripe/traits/internal_session_cache_purge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index d125c53d7f..df5bae0773 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; /// Allows us to purge session caches that might keep stale premium plan/billing data around -#[cfg_attr(test, automock)] +#[cfg_attr(test, mockall::automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. From d51ee67d4c066b111da06352dde0835777d85c0d Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 17:35:36 -0800 Subject: [PATCH 117/136] complete failure --- .../inference_category.rs | 8 ++++++- .../inference_model_type.rs | 6 +++-- .../config/inference-job.development.env | 9 +++++++ .../rerender_inference_command.rs | 24 +++---------------- .../animation/enqueue_rerender_animation.rs | 12 ++-------- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index b96b6fb261..d1a420544b 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -50,6 +50,7 @@ impl InferenceCategory { "lipsync_animation" => Ok(Self::LipsyncAnimation), "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), + "rerender_a_video" => Ok(Self::RerenderAVideo), _ => Err(format!("invalid value: {:?}", value)), } } @@ -61,6 +62,7 @@ impl InferenceCategory { Self::LipsyncAnimation, Self::TextToSpeech, Self::VoiceConversion, + Self::RerenderAVideo, ]) } } @@ -75,6 +77,7 @@ mod tests { assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation"); assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); + assert_serialization(InferenceCategory::RerenderAVideo, "rerender_a_video"); } #[test] @@ -82,6 +85,7 @@ mod tests { assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation"); assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); + assert_eq!(InferenceCategory::RerenderAVideo.to_str(), "rerender_a_video"); } #[test] @@ -89,16 +93,18 @@ mod tests { assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation); assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); + assert_eq!(InferenceCategory::from_str("rerender_a_video").unwrap(), InferenceCategory::RerenderAVideo); } #[test] fn all_variants() { // Static check let mut variants = InferenceCategory::all_variants(); - assert_eq!(variants.len(), 3); + assert_eq!(variants.len(), 4); assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::RerenderAVideo)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 0a30156539..c5c312b300 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -49,7 +49,7 @@ impl InferenceModelType { Self::Tacotron2 => "tacotron2", Self::Vits => "vits", Self::VallEX => "vall_e_x", - Self::RerenderAVideo => "rerender_a_video", + Self::RerenderAVideo => "rerender_a_video", } } @@ -61,6 +61,7 @@ impl InferenceModelType { "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), "vall_e_x" => Ok(Self::VallEX), + "rerender_a_video" => Ok(Self::RerenderAVideo), _ => Err(format!("invalid value: {:?}", value)), } } @@ -74,7 +75,8 @@ impl InferenceModelType { InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, - InferenceModelType::VallEX + InferenceModelType::VallEX, + InferenceModelType::RerenderAVideo ]) } } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 0df22ea669..359c8edf3e 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -94,6 +94,15 @@ VITS_INFERENCE_SCRIPT="infer_ts_job.py" VITS_INFERENCE_MAYBE_PYTHON_INTERPRETER="python3" VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b" + +# ================================= # +# # +# Rerender # +# # +# ================================= # +RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/" +RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="conda activate rerender && python rerender.py" + # ================================= # # # # CACHE SETTINGS # diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs index 8b0b011893..a3ba8953ae 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -40,7 +40,7 @@ pub struct RerenderInferenceCommand { executable_or_command: ExecutableOrCommand, /// Config file to use - config_path: PathBuf, + config_path: Option, /// eg. `source python/bin/activate` maybe_virtual_env_activation_command: Option, @@ -68,30 +68,12 @@ pub struct InferenceArgs<'s, P: AsRef> { } impl RerenderInferenceCommand { - pub fn new>( - rerender_root_code_directory: PathBuf, - executable_or_command: ExecutableOrCommand, - config_path: PathBuf, - maybe_virtual_env_activation_command: Option, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - ) -> Self { - Self { - rerender_root_code_directory: rerender_root_code_directory.clone(), - executable_or_command, - config_path: config_path.clone(), - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_docker_options, - maybe_execution_timeout, - } - } - pub fn from_env() -> AnyhowResult { let rerender_root_code_directory = easyenv::get_env_pathbuf_required( "RERENDER_INFERENCE_ROOT_DIRECTORY")?; - let config_path = easyenv::get_env_pathbuf_required( - "RERENDER_INFERENCE_CONFIG_PATH")?; + let config_path = easyenv::get_env_pathbuf_optional( + "RERENDER_INFERENCE_CONFIG_PATH"); let executable_or_command = match easyenv::get_env_string_optional( "RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND") { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs index 1519e5332b..d90caa3b8b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -20,7 +20,6 @@ use mysql_queries::payloads::generic_inference_args::videofilter_payload::{Reren use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; -use tokens::tokens::media_uploads::MediaUploadToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; @@ -39,17 +38,12 @@ const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; pub struct EnqueueRerenderAnimationRequest { uuid_idempotency_token: String, - video_source: VideoSource, + video_source: Option, sd_model: String, creator_set_visibility: Option, } /// Treated as an enum. Only one of these may be set. -#[derive(Deserialize)] -pub struct VideoSource { - maybe_media_file_token: Option, - maybe_media_upload_token: Option, -} #[derive(Serialize)] pub struct EnqueueRerenderAnimationSuccessResponse { @@ -165,10 +159,8 @@ pub async fn enqueue_rerender_animation_handler( // TODO(bt): CHECK DATABASE FOR TOKENS! let video_source: VideofilterVideoSource = { - if let Some(ref token) = request.video_source.maybe_media_file_token { + if let Some(ref token) = request.video_source { VideofilterVideoSource::media_file_token(token.as_str()) - } else if let Some(ref token) = request.video_source.maybe_media_upload_token { - VideofilterVideoSource::media_upload_token(token.as_str()) } else { return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); } From 292b831d1697d4912632a02cba080620fba55854 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 17:36:53 -0800 Subject: [PATCH 118/136] update tests --- .../by_table/generic_inference_jobs/inference_model_type.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index c5c312b300..a3ea0bcd29 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -94,6 +94,7 @@ mod tests { assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); assert_serialization(InferenceModelType::VallEX, "vall_e_x"); + assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video"); } #[test] @@ -104,6 +105,7 @@ mod tests { assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x"); + assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video"); } #[test] @@ -114,19 +116,21 @@ mod tests { assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX); + assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo); } #[test] fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 6); + assert_eq!(variants.len(), 7); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo)); assert_eq!(variants.pop_first(), None); // Generated check From 57e2f70707a2ace6f5ead0407b0ca3c66bc1fb46 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 23:07:27 -0800 Subject: [PATCH 119/136] WORKING! --- Cargo.lock | 1 + .../generic_synthetic_ids/id_category.rs | 6 + .../media_file_origin_model_type.rs | 7 +- .../media_file_origin_product_category.rs | 6 + .../insert_media_file_from_rerender.rs | 158 ++++++++++++ .../src/queries/media_files/mod.rs | 1 + crates/service/job/inference_job/Cargo.toml | 1 + .../config/inference-job.development.env | 2 +- .../videofilter/process_single_vf_job.rs | 76 +++--- .../rerender_a_video/download_video_file.rs | 32 ++- .../rerender_a_video/process_job.rs | 229 ++++++++++++------ 11 files changed, 415 insertions(+), 104 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs diff --git a/Cargo.lock b/Cargo.lock index 215225d64d..4193767e2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3070,6 +3070,7 @@ dependencies = [ "newrelic-telemetry", "once_cell", "r2d2_redis", + "rand 0.8.5", "redis_common", "serde", "serde_derive", diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index eccff8cf4d..01bddb94dd 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,6 +21,10 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimationResult, + // Results from rerender a video + #[serde(rename = "rerender")] + RerenderResult, + /// Results from tacotron2 /// Applies for RVC and SVC #[serde(rename = "tts_result")] @@ -62,6 +66,7 @@ impl IdCategory { Self::ZeroShotVoiceDataset => "zs_dataset", Self::ZeroShotVoiceEmbedding => "zs_voice", Self::ZeroShotTtsResult => "zs_tts_result", + Self::RerenderResult => "rerender", Self::ModelWeights => "model_weights", } } @@ -75,6 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), + "rerender" => Ok(Self::RerenderResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index 8cb9685bc7..a79930d8c6 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -30,6 +30,9 @@ pub enum MediaFileOriginModelType { #[serde(rename = "vall_e_x")] VallEX, + + #[serde(rename = "rerender")] + Rerender, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -45,7 +48,8 @@ impl MediaFileOriginModelType { Self::SadTalker => "sad_talker", Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", - Self::VallEX => "vall_e_x" + Self::VallEX => "vall_e_x", + Self::Rerender => "rerender", } } @@ -56,6 +60,7 @@ impl MediaFileOriginModelType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 7b6f2a029a..5965f9b3a9 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -38,6 +38,10 @@ pub enum MediaFileOriginProductCategory { /// Media files created by (or uploaded for) the Zero Shot voice product. #[serde(rename = "zs_voice")] ZeroShotVoice, + + // Media files for Rerender + #[serde(rename = "rerender")] + Rerender, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -54,6 +58,7 @@ impl MediaFileOriginProductCategory { Self::Unknown => "unknown", Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", + Self::Rerender => "rerender", } } @@ -64,6 +69,7 @@ impl MediaFileOriginProductCategory { "unknown" => Ok(Self::Unknown), "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs new file mode 100644 index 0000000000..c599a4fad0 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs @@ -0,0 +1,158 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub struct InsertArgs<'a> { + pub pool: &'a MySqlPool, + pub job: &'a AvailableInferenceJob, + + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub sha256_checksum: &'a str, + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, + + pub is_on_prem: bool, + pub worker_hostname: &'a str, + pub worker_cluster: &'a str, +} + +pub async fn insert_media_file_from_rerender( + args: InsertArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let result_token = MediaFileToken::generate(); + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() { + let user_token = UserToken::new_from_str(creator_user_token); + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_rerender_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::RerenderResult, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_rerender_id); + } + + let creator_ip_address = args.job.creator_ip_address.as_str(); + let creator_set_visibility = args.job.creator_set_visibility.clone(); + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Rerender; + const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::Rerender; + const MEDIA_TYPE : MediaFileType = MediaFileType::Video; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + maybe_origin_model_type = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ?, + + is_generated_on_prem = ?, + generated_by_worker = ?, + generated_by_cluster = ? + + "#, + result_token.as_str(), + + ORIGIN_CATEGORY.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + ORIGIN_MODEL_TYPE.to_str(), + + MEDIA_TYPE.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.job.maybe_creator_user_token, + args.job.maybe_creator_anonymous_visitor_token, + args.job.creator_ip_address, + + args.job.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + + args.is_on_prem, + args.worker_hostname, + args.worker_cluster + ) + .execute(&mut *transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((result_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 8930a3ae7c..ff431d1cb4 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -8,3 +8,4 @@ pub mod insert_media_file_from_zero_shot_tts; pub mod list_media_files; pub mod list_media_files_for_user; pub mod update_media_file; +pub mod insert_media_file_from_rerender; diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 65713e40b0..fb451aa8a1 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -81,3 +81,4 @@ async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } time = "0.3.30" mime = "0.3.17" +rand = { version = "0.8.5", features = [] } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 359c8edf3e..15ab7f6497 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -101,7 +101,7 @@ VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b" # # # ================================= # RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/" -RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="conda activate rerender && python rerender.py" +RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py" # ================================= # # # diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index 6a35427d0e..9d7cc0163c 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,43 +1,56 @@ use anyhow::anyhow; -use log::{error, info}; +use log::info; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; -use tokens::tokens::media_uploads::MediaUploadToken; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use tokens::tokens::media_files::MediaFileToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::videofilter::rerender_a_video; +use crate::job::job_types::videofilter::rerender_a_video::process_job::RerenderProcessJobArgs; use crate::job_dependencies::JobDependencies; pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { - let maybe_media_upload_token = job.maybe_input_source_token - .as_deref() - .map(|token| MediaUploadToken::new_from_str(token)); + // let maybe_inference_args = job.maybe_inference_args.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Inference args not found")))?; + // + // let maybe_rerender_args = maybe_inference_args.args.as_ref().map(|args| match args { + // Rr(args) => Some(args), + // _ => None, + // }).flatten(); + // + // let rerender_args = match maybe_rerender_args { + // None => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + // Some(args) => args, + // }; + // + // let videofilter_source = rerender_args.maybe_video_source.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Video source not found")))?; + // + // let media_file_token = match videofilter_source { + // VideofilterVideoSource::F(token) => token, + // _ => return Err(ProcessSingleJobError::Other(anyhow!("Video source not found"))), + // }; + // + // let media_file_token = MediaFileToken::new_from_str(media_file_token); + // + // info!("media_file_token: {:?}", media_file_token); + // + // + // let media_file = match media_file_result { + // Ok(Some(result)) => result, + // Ok(None) => { + // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); + // } + // Err(e) => { + // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); + // } + // }; - let media_upload_token = match maybe_media_upload_token { - None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media upload for vc job: {:?}", job.inference_job_token))), - Some(token) => token, - }; - - let maybe_media_upload_result = - get_media_upload_for_inference(&media_upload_token, &job_dependencies.db.mysql_pool).await; - - let media_upload = match maybe_media_upload_result { - Ok(Some(media_upload)) => media_upload, - Ok(None) => { - error!("no media upload record found for token: {:?}", media_upload_token); - return Err(ProcessSingleJobError::Other(anyhow!("no media upload record found for token: {:?}", media_upload_token))); - }, - Err(err) => { - error!("error fetching media upload record from db: {:?}", err); - return Err(ProcessSingleJobError::Other(err)); - }, - }; - - info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); - info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); - info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); + // info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); + // info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); + // info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); @@ -54,9 +67,10 @@ pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &Ava // }; let job_success_result = rerender_a_video::process_job::process_job( - rerender_a_video::process_job::RerenderProcessJobArgs { + RerenderProcessJobArgs { job_dependencies, - job + job, + // media_file } ).await?; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs index 344592bd08..e05d695841 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -4,13 +4,16 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -34,9 +37,32 @@ pub async fn download_video_file( let bucket_object_path; match video_source { - VideofilterVideoSource::F(_media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + VideofilterVideoSource::F(media_file_token) => { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + false, + mysql_pool + ).await; + match media_file_result { + Ok(Some(result)) => { + let media_file_bucket_path = + MediaFileBucketPath::from_object_hash( + &result.public_bucket_directory_hash, + result.maybe_public_bucket_prefix.as_deref(), + result.maybe_public_bucket_extension.as_deref(), + ); + bucket_object_path = media_file_bucket_path.to_full_object_pathbuf(); + } + Ok(None) => { + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file: {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + } } VideofilterVideoSource::U(media_upload_token) => { let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index affec1aee3..0df0d80325 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -1,8 +1,9 @@ use std::fs::read_to_string; use std::time::Instant; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use log::{error, info, warn}; +use serde::{Deserialize, Serialize}; use buckets::public::media_files::original_file::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; @@ -12,19 +13,16 @@ use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; -use tokens::tokens::users::UserToken; +use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs}; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; -// use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; use crate::job_dependencies::JobDependencies; -use crate::util::common_commands::ffmpeg_logo_watermark_command; /// The maximum that either width or height can be const MAX_DIMENSION : u32 = 1500; @@ -35,6 +33,7 @@ const BUCKET_FILE_EXTENSION: &str = ".mp4"; pub struct RerenderProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, + // pub media_file: MediaFile, } pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { @@ -88,43 +87,8 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { - // error!("Image resize failure: {:?}", err); - // safe_delete_temp_file(&audio_path.filesystem_path); - // safe_delete_temp_file(&image_path.filesystem_path); - // safe_delete_temp_directory(&work_temp_dir); - // return Err(ProcessSingleJobError::Other(anyhow!("image resize failure: {:?}", err))); - // } - // Ok(resized_image_path) => { - // usable_image_path = resized_image_path; - // } - // } - // } + + //TODO: re encode with ffmpeg info!("Used video file: {:?}", usable_video_path); @@ -148,15 +112,43 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + Some(args) => args, + }; + + let rr_args = match poly_args { + Rr(args) => args, + _ => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + }; + + let sd_model = rr_args.maybe_sd_model.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + // ============== CREATE AND WRITE CONFIG TO DISK ============== // + + // create seed if not exists + // let seed = rr_args.seed.unwrap_or_else(|| rand::random::()); + let seed = rand::random::(); + + let config = RerenderConfig { + input: Some(usable_video_path.to_str().unwrap().to_string()), + output: Some(output_video_fs_path.to_str().unwrap().to_string()), + work_dir: Some(work_temp_dir.path().to_str().unwrap().to_string()), + sd_model: Some(sd_model.to_string()), + prompt: Some("1girl, anime style, red hair".to_string()), + a_prompt: Some("1girl, 3d, realistic, read hair".to_string()), + n_prompt: Some("Bad quality, low quality, photo, artifacts, watermark, signature, faceless".to_string()), + seed: Some(seed as i64), + ..RerenderConfig::new() + }; + + config.validate().map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; let config_path = work_temp_dir.path().join("config.json"); - - // TODO + let config_json = serde_json::to_string(&config).unwrap(); + std::fs::write(&config_path, config_json).unwrap(); // ==================== RUN INFERENCE SCRIPT ==================== // - - let workdir = work_temp_dir.path().to_path_buf(); let stderr_output_file = work_temp_dir.path().join("stderr.txt"); let inference_start_time = Instant::now(); @@ -174,7 +166,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, + + #[serde(rename = "output")] + pub output: Option, + + #[serde(rename = "work_dir")] + pub work_dir: Option, + + #[serde(rename = "key_subdir")] + pub key_subdir: Option, + + #[serde(rename = "sd_model")] + pub sd_model: Option, + + #[serde(rename = "lora_path")] + pub lora_path: Option, + + #[serde(rename = "interval")] + pub interval: Option, + + #[serde(rename = "prompt")] + pub prompt: Option, + + #[serde(rename = "a_prompt")] + pub a_prompt: Option, + + #[serde(rename = "n_prompt")] + pub n_prompt: Option, + + #[serde(rename = "x0_strength")] + pub x0_strength: Option, + + #[serde(rename = "control_type")] + pub control_type: Option, + + #[serde(rename = "canny_low")] + pub canny_low: Option, + + #[serde(rename = "canny_high")] + pub canny_high: Option, + + #[serde(rename = "control_strength")] + pub control_strength: Option, + + #[serde(rename = "seed")] + pub seed: Option, + + #[serde(rename = "warp_period")] + pub warp_period: Option>, + + #[serde(rename = "ada_period")] + pub ada_period: Option>, +} + +impl RerenderConfig { + pub fn new() -> Self { + Self { + input: None, + output: None, + work_dir: None, + key_subdir: Some("keys".to_string()), + sd_model: None, + lora_path: None, + interval: Some(8), + prompt: None, + a_prompt: None, + n_prompt: None, + x0_strength: Some(0.95), + control_type: Some("canny".to_string()), + canny_low: Some(50), + canny_high: Some(100), + control_strength: Some(0.7), + seed: None, + warp_period: Some(vec![0., 0.1]), + ada_period: Some(vec![0.8, 1.]), + } + } + + pub fn validate(&self) -> Result<(), Error> { + let fields = &[ + (self.input.is_some(), "input"), + (self.output.is_some(), "output"), + (self.work_dir.is_some(), "work_dir"), + (self.key_subdir.is_some(), "key_subdir"), + (self.sd_model.is_some(), "sd_model"), + (self.interval.is_some(), "interval"), + (self.prompt.is_some(), "prompt"), + (self.a_prompt.is_some(), "a_prompt"), + (self.n_prompt.is_some(), "n_prompt"), + (self.x0_strength.is_some(), "x0_strength"), + (self.control_type.is_some(), "control_type"), + (self.canny_low.is_some(), "canny_low"), + (self.canny_high.is_some(), "canny_high"), + (self.control_strength.is_some(), "control_strength"), + (self.seed.is_some(), "seed"), + (self.warp_period.is_some(), "warp_period"), + (self.ada_period.is_some(), "ada_period"), + ]; + + let missing_fields: Vec<&str> = fields + .iter() + .filter_map(|(is_some, field_name)| if *is_some { None } else { Some(*field_name) }) + .collect(); + + if missing_fields.is_empty() { + Ok(()) + } else { + Err(anyhow!("Missing fields: {}", missing_fields.join(", "))) + } + } +} From 196be25fb5a8ccbb660b3f11c85d14d724d7cf59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 11:04:13 -0500 Subject: [PATCH 120/136] voice takedown requests --- _sql/actions/voice_conversion_models.sql | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 8520aa609e..9fd7b4ee20 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -344,6 +344,36 @@ where token IN ( 'vcm_zmxv2k1q9jd8' ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'takedown request' +where token IN ( + 'vcm_3mzcjkf78x7m', + 'vcm_7k2t1wnyqxsc', + 'vcm_w2edpe6q2nya', + 'vcm_vz9cg294xa29', + 'vcm_0z1ys58589k7', + 'vcm_4x8a30h1r26z', + 'vcm_0vf9prxsww60', + 'vcm_yxbadkvqhg16', + 'vcm_p0ezrc6wvzak', + 'vcm_6r5pbtexyvtp', + 'vcm_pb0ewrf7ych5', + 'vcm_7frb2xk0cxk3', + 'vcm_x129c9s2af06', + 'vcm_e0snndfd59b6', + 'vcm_nb8zb0djqwqw', + 'vcm_2erz5hbh8tpf', + 'vcm_e9wn2ccqe3bk', + 'vcm_f8jbxdbjgcjb', + 'vcm_x02h7z3jqxb0', + 'vcm_xy97xcgnk9y6', + 'vcm_a47wtfbsjrfc', + 'vcm_kygenhbg5mgj', + 'vcm_s8gndv19yt7p' +); + From 3766758cfb33ac0cedf93ced058c117130419883 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 13:33:48 -0500 Subject: [PATCH 121/136] Rate limiter support for AI streamers --- .../config/storyteller-web.development.env | 3 + .../src/configs/app_startup/mod.rs | 2 + .../app_startup/redis_rate_limiters.rs | 102 ++++++++++++++++++ .../src/configs/app_startup/username_set.rs | 102 ++++++++++++++++++ .../web/storyteller_web/src/configs/mod.rs | 1 + .../enqueue_infer_tts_handler.rs | 19 ++++ .../service/web/storyteller_web/src/main.rs | 67 +----------- .../web/storyteller_web/src/server_state.rs | 7 ++ 8 files changed, 238 insertions(+), 65 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/mod.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs diff --git a/crates/service/web/storyteller_web/config/storyteller-web.development.env b/crates/service/web/storyteller_web/config/storyteller-web.development.env index 01ffcc593e..c890b5ef3d 100644 --- a/crates/service/web/storyteller_web/config/storyteller-web.development.env +++ b/crates/service/web/storyteller_web/config/storyteller-web.development.env @@ -5,6 +5,9 @@ COOKIE_SECURE=false COOKIE_DOMAIN=.jungle.horse WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse +# Test the rate limiter bypass (obviously these are dummy values) +AI_STREAMER_USERNAMES="PewDiePie,xQc" + # Stripe configs # All secrets are development and okay to check in STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs new file mode 100644 index 0000000000..824be656d0 --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs @@ -0,0 +1,2 @@ +pub mod redis_rate_limiters; +pub mod username_set; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs new file mode 100644 index 0000000000..1dc2b4d5cd --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -0,0 +1,102 @@ +use std::time::Duration; + +use log::info; + +use config::common_env::CommonEnv; +use errors::AnyhowResult; +use limitation::Limiter; +use crate::configs::app_startup::username_set::UsernameSet; + +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::server_state::RedisRateLimiters; + + +/// Build the various rate limiters +pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult { + // TODO(bt,2023-12-13): Kill CommonEnv. + + info!("Setting up Redis rate limiters..."); + + let logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) + }; + + let logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) + }; + + let api_high_priority_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) + }; + + // This is for the following users: + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let api_ai_streamers_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_AI_STREAMERS_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_ai_streamers", limiter_enabled) + }; + + let ai_streamer_usernames = + UsernameSet::from_comma_separated(&easyenv::get_env_string_or_default("AI_STREAMER_USERNAMES", "")); + + info!("AI Streamers that can bypass the normal rate limiter ({}): {:?}", + ai_streamer_usernames.len(), + ai_streamer_usernames.list_names()); + + let model_upload_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) + }; + + Ok(RedisRateLimiters { + logged_out: logged_out_redis_rate_limiter, + logged_in: logged_in_redis_rate_limiter, + api_high_priority: api_high_priority_redis_rate_limiter, + api_ai_streamers: api_ai_streamers_redis_rate_limiter, + api_ai_streamer_username_set: ai_streamer_usernames, + model_upload: model_upload_rate_limiter, + }) +} diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs new file mode 100644 index 0000000000..f9858a201b --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -0,0 +1,102 @@ +use std::collections::{BTreeSet, HashSet}; + +#[derive(Clone)] +pub struct UsernameSet { + /// "usernames" in our service are always lowercase. + /// "display names" are case sensitive. + usernames: HashSet, +} + +impl UsernameSet { + pub fn from_comma_separated(comma_separated_list: &str) -> Self { + let usernames = comma_separated_list.split(",") + .into_iter() + .map(|item| item.trim()) + .map(|item| item.to_lowercase()) + .filter(|item| !item.is_empty()) + .collect::>(); + Self { + usernames + } + } + + pub fn len(&self) -> usize { + self.usernames.len() + } + + pub fn list_names(&self) -> BTreeSet<&str> { + // NB: BTreeSet for sorting, which is good for logging at app startup + self.usernames.iter().map(|item| item.as_str()).collect::<_>() + } + + pub fn username_is_in_set(&self, username: &str) -> bool { + // We lowercase the input username just in case a display name is passed + let username = username.to_lowercase(); + self.usernames.contains(&username) + } +} + +#[cfg(test)] +mod tests { + use crate::configs::app_startup::username_set::UsernameSet; + fn build_set() -> UsernameSet { + UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") + } + #[test] + fn test_deduplication() { + let set = build_set(); + assert_eq!(set.len(), 3); + } + + #[test] + fn test_in_set() { + let set = build_set(); + assert!(set.username_is_in_set("bob")); + assert!(set.username_is_in_set("alice")); + assert!(set.username_is_in_set("john")); + } + + #[test] + fn test_in_set_case_insensitive() { + let set = build_set(); + + assert!(set.username_is_in_set("Bob")); + assert!(set.username_is_in_set("BOB")); + + assert!(set.username_is_in_set("AlIcE")); + assert!(set.username_is_in_set("ALICE")); + + assert!(set.username_is_in_set("jOHN")); + assert!(set.username_is_in_set("JOHN")); + } + + #[test] + fn test_not_in_set() { + let set = build_set(); + + // NB: The build_set() function might have attempted to introduce these, + // but the struct constructor should filter blank spaces out + assert!(!set.username_is_in_set("")); + assert!(!set.username_is_in_set(" ")); + + // Obviously not in set + assert!(!set.username_is_in_set("mary")); + assert!(!set.username_is_in_set("bill")); + + // These usernames are in the set, but not with padding + assert!(!set.username_is_in_set(" alice ")); + assert!(!set.username_is_in_set("bob ")); + assert!(!set.username_is_in_set(" john")); + } + + #[test] + fn test_username_return() { + let set = build_set(); + let usernames = set.list_names(); + assert_eq!(usernames.len(), 3); + assert!(usernames.contains(&"alice")); + assert!(usernames.contains(&"bob")); + assert!(usernames.contains(&"john")); + assert!(!usernames.contains(&"mary")); + } +} diff --git a/crates/service/web/storyteller_web/src/configs/mod.rs b/crates/service/web/storyteller_web/src/configs/mod.rs index 0a735a5b5c..806e1e3209 100644 --- a/crates/service/web/storyteller_web/src/configs/mod.rs +++ b/crates/service/web/storyteller_web/src/configs/mod.rs @@ -1,2 +1,3 @@ +pub mod app_startup; pub mod plans; pub mod static_api_tokens; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index b01dd37a5d..cb417f5906 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -242,7 +242,12 @@ pub async fn enqueue_infer_tts_handler( // ==================== RATE LIMIT ==================== // + // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { @@ -250,6 +255,15 @@ pub async fn enqueue_infer_tts_handler( warn!("User is not authorized to use TTS model because they are banned."); return Err(InferTtsError::NotAuthorized); } + + if server_state + .redis_rate_limiters + .api_ai_streamer_username_set + .username_is_in_set(&user.user.username) + { + is_ai_streamer = true; + } + &server_state.redis_rate_limiters.logged_in }, }; @@ -264,6 +278,11 @@ pub async fn enqueue_infer_tts_handler( rate_limiter = &server_state.redis_rate_limiters.api_high_priority; } + if is_ai_streamer { + info!("Using AI streamer rate limiter"); + rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; + } + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index e77ed85093..57f09a82c8 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -75,6 +75,7 @@ use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStr use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; @@ -173,65 +174,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, ); - info!("Setting up Redis rate limiters..."); - - // Old env vars: - // - // "LIMITER_ENABLED" - // "LIMITER_MAX_REQUESTS" - // "LIMITER_WINDOW_SECONDS" - // - let logged_out_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) - }; - - let logged_in_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) - }; - - let api_high_priority_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) - }; - - let model_upload_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) - }; info!("Connecting to elasticsearch..."); @@ -466,12 +408,7 @@ async fn main() -> AnyhowResult<()> { elasticsearch, redis_pool, redis_ttl_cache, - redis_rate_limiters: RedisRateLimiters { - logged_out: logged_out_redis_rate_limiter, - logged_in: logged_in_redis_rate_limiter, - api_high_priority: api_high_priority_redis_rate_limiter, - model_upload: model_upload_rate_limiter, - }, + redis_rate_limiters: configure_redis_rate_limiters(&common_env)?, firehose_publisher, badge_granter, avt_cookie_manager, diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index 41ddadf1d3..a8f86133e2 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -20,6 +20,7 @@ use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; +use crate::configs::app_startup::username_set::UsernameSet; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo; @@ -140,6 +141,12 @@ pub struct RedisRateLimiters { /// constructions) pub api_high_priority: RedisRateLimiter, + /// API rate limit for AI streamers + pub api_ai_streamers: RedisRateLimiter, + + /// Usernames of AI streamers + pub api_ai_streamer_username_set: UsernameSet, + /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, } From 81dd377599b08f62d0f7a6d483ebb1238d6c512c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 15:27:34 -0500 Subject: [PATCH 122/136] clean up and test rate limiter choice code --- .../src/configs/app_startup/username_set.rs | 1 + .../enqueue_infer_tts_handler.rs | 148 +++++++++++++----- 2 files changed, 114 insertions(+), 35 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs index f9858a201b..0f48cb5350 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -42,6 +42,7 @@ mod tests { fn build_set() -> UsernameSet { UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") } + #[test] fn test_deduplication() { let set = build_set(); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index cb417f5906..2001562038 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -32,6 +32,7 @@ use tts_common::priority::{FAKEYOU_DEFAULT_VALID_API_TOKEN_PRIORITY_LEVEL, FAKEY use user_input_common::check_for_slurs::contains_slurs; use users_component::utils::user_session_extended::UserSessionExtended; +use crate::configs::app_startup::username_set::UsernameSet; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; use crate::configs::plans::plan::Plan; use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; @@ -240,49 +241,44 @@ pub async fn enqueue_infer_tts_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + warn!("User is not authorized to use TTS model because they are banned."); + return Err(InferTtsError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // - // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { - // For AI live streamers - // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 - let mut is_ai_streamer = false; + let maybe_username = maybe_user_session + .as_ref() + .map(|session| session.user.username.as_str()); - let mut rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - warn!("User is not authorized to use TTS model because they are banned."); - return Err(InferTtsError::NotAuthorized); - } + // FIXME(bt,2023-12-13): These boolean flags are a bit dangerous. + let rate_limiter_type = get_rate_limiter_type( + maybe_username, + &server_state.redis_rate_limiters.api_ai_streamer_username_set, + is_investor, + is_from_api, + use_high_priority_rate_limiter + ); - if server_state - .redis_rate_limiters - .api_ai_streamer_username_set - .username_is_in_set(&user.user.username) - { - is_ai_streamer = true; - } - - &server_state.redis_rate_limiters.logged_in + let rate_limiter = match rate_limiter_type { + RateLimiterType::LoggedOut => &server_state.redis_rate_limiters.logged_out, + RateLimiterType::LoggedIn => &server_state.redis_rate_limiters.logged_in, + RateLimiterType::ApiHighPriority => { + info!("Using API high priority rate limiter"); + &server_state.redis_rate_limiters.api_high_priority + }, + RateLimiterType::ApiAiStreamer => { + info!("Using AI streamer rate limiter"); + &server_state.redis_rate_limiters.api_ai_streamers }, }; - // TODO/TEMP - if is_investor || is_from_api { - rate_limiter = &server_state.redis_rate_limiters.logged_in; - } - - // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters - if use_high_priority_rate_limiter { - rate_limiter = &server_state.redis_rate_limiters.api_high_priority; - } - - if is_ai_streamer { - info!("Using AI streamer rate limiter"); - rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; - } - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } @@ -495,6 +491,54 @@ pub async fn enqueue_infer_tts_handler( .body(body)) } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RateLimiterType { + LoggedOut, + LoggedIn, + ApiHighPriority, + ApiAiStreamer, +} + +// FIXME(bt,2023-12-13): The boolean flags and disjoint decision making are kind of terrible. +fn get_rate_limiter_type( + maybe_username: Option<&str>, + ai_streamer_username_set: &UsernameSet, + is_investor: bool, + is_from_api: bool, + use_high_priority_rate_limiter: bool, +) -> RateLimiterType { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + + let mut rate_limiter = match maybe_username { + None => RateLimiterType::LoggedOut, + Some(username) => { + if ai_streamer_username_set.username_is_in_set(username) { + is_ai_streamer = true; + } + + RateLimiterType::LoggedIn + }, + }; + + // TODO/TEMP + if is_investor || is_from_api { + rate_limiter = RateLimiterType::LoggedIn; + } + + // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters + if use_high_priority_rate_limiter { + rate_limiter = RateLimiterType::ApiHighPriority; + } + + if is_ai_streamer { + rate_limiter = RateLimiterType::ApiAiStreamer; + } + + rate_limiter +} + pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> { if text.len() < 3 { return Err("text is too short".to_string()); @@ -561,3 +605,37 @@ fn random_troll_text() -> &'static str { let random = texts.choose(&mut rand::thread_rng()); random.map(|r| *r).unwrap_or(texts[0]) } + + +#[cfg(test)] +mod tests { + + mod rate_limiter_tests { + use crate::configs::app_startup::username_set::UsernameSet; + use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::{get_rate_limiter_type, RateLimiterType}; + + fn build_username_set() -> UsernameSet { + UsernameSet::from_comma_separated("ai_streamer") + } + + #[test] + fn test_logged_out() { + let rate_limiter = get_rate_limiter_type(None, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedOut); + } + + #[test] + fn test_logged_in() { + let username = Some("not_ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedIn); + } + + #[test] + fn test_ai_streamer() { + let username = Some("ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::ApiAiStreamer); + } + } +} From 9ab8e5221190980666a1510fdf699171149bfcba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 16:58:37 -0500 Subject: [PATCH 123/136] more rate limiter fixes --- .../app_startup/redis_rate_limiters.rs | 28 +++++++++++++++++++ .../animation/enqueue_face_animation.rs | 16 +++++++---- .../media_uploads/common/handle_upload.rs | 21 +++++++------- .../enqueue_voice_conversion_inference.rs | 15 ++++++---- .../inference/enqueue_tts_request.rs | 17 ++++++----- .../voice_dataset_samples/upload_zs_sample.rs | 18 +++++++----- .../voice_designer/voices/create_voice.rs | 16 +++++++---- .../w2l/enqueue_infer_w2l_with_uploads.rs | 15 ++++++---- .../web/storyteller_web/src/server_state.rs | 6 ++++ 9 files changed, 103 insertions(+), 49 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs index 1dc2b4d5cd..3aa3df32cf 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -91,6 +91,32 @@ pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult AnyhowResult &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueFaceAnimationError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 2d79cab2e7..7fe2872d84 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -66,26 +66,25 @@ pub async fn handle_upload( UploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(UploadError::RateLimited); } - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - // ==================== READ MULTIPART REQUEST ==================== // let upload_media_request = drain_multipart_request(multipart_payload) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index 22131729c0..7be7cec00d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -205,17 +205,20 @@ pub async fn enqueue_voice_conversion_inference_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _session) => &server_state.redis_rate_limiters.logged_in }; // TODO/TEMP diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 41adb52569..7e09f40782 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,7 +1,6 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] -use utoipa::ToSchema; use std::fmt::Debug; use std::sync::Arc; @@ -12,6 +11,7 @@ use actix_web::http::StatusCode; use log::warn; use serde::Deserialize; use serde::Serialize; +use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; @@ -153,16 +153,19 @@ pub async fn enqueue_tts_request( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueTTSRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueTTSRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index 004d303f44..e89e18eca9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -26,6 +26,7 @@ use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; @@ -111,16 +112,19 @@ pub async fn upload_zs_sample_handler( UploadSampleError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadSampleError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadSampleError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _user) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs index b47de2c732..6654843007 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs @@ -30,6 +30,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -138,16 +139,19 @@ pub async fn create_voice_handler( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueCreateVoiceRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueCreateVoiceRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index eee83b0dcc..8eea038050 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -102,16 +102,19 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_session { + if user.is_banned { + return Err(InferW2lWithUploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(InferW2lWithUploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index a8f86133e2..c7ba601993 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -149,6 +149,12 @@ pub struct RedisRateLimiters { /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_out: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_in: RedisRateLimiter, } /// In-memory caches of several types. From b30c352d79bad5ad5a4b07bd219412455168ac69 Mon Sep 17 00:00:00 2001 From: saltacc Date: Wed, 13 Dec 2023 16:29:45 -0800 Subject: [PATCH 124/136] pass tests --- .../src/by_table/generic_synthetic_ids/id_category.rs | 9 +++++++-- .../media_files/media_file_origin_model_type.rs | 10 +++++++--- .../media_files/media_file_origin_product_category.rs | 7 ++++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index 01bddb94dd..e5b2e90771 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -80,7 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), - "rerender" => Ok(Self::RerenderResult), + "rerender" => Ok(Self::RerenderResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } @@ -92,6 +92,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimationResult, + Self::RerenderResult, Self::TtsResult, Self::VoiceConversionResult, Self::ZeroShotTtsResult, @@ -114,6 +115,7 @@ mod tests { fn test_serialization() { assert_serialization(IdCategory::MediaFile, "media_file"); assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation"); + assert_serialization(IdCategory::RerenderResult, "rerender"); assert_serialization(IdCategory::TtsResult, "tts_result"); assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion"); assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); @@ -126,6 +128,7 @@ mod tests { fn to_str() { assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation"); + assert_eq!(IdCategory::RerenderResult.to_str(), "rerender"); assert_eq!(IdCategory::TtsResult.to_str(), "tts_result"); assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion"); assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset"); @@ -138,6 +141,7 @@ mod tests { fn from_str() { assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult); + assert_eq!(IdCategory::from_str("rerender").unwrap(), IdCategory::RerenderResult); assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult); assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult); assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset); @@ -150,9 +154,10 @@ mod tests { fn all_variants() { // Static check let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 8); + assert_eq!(variants.len(), 9); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); + assert_eq!(variants.pop_first(), Some(IdCategory::RerenderResult)); assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult)); diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index a79930d8c6..fa75c7f791 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -60,7 +60,7 @@ impl MediaFileOriginModelType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), - "rerender" => Ok(Self::Rerender), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } @@ -73,7 +73,8 @@ impl MediaFileOriginModelType { Self::SadTalker, Self::SoVitsSvc, Self::Tacotron2, - Self::VallEX + Self::VallEX, + Self::Rerender, ]) } } @@ -98,6 +99,7 @@ mod tests { assert_eq!(MediaFileOriginModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(MediaFileOriginModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(MediaFileOriginModelType::VallEX.to_str(), "vall_e_x"); + assert_eq!(MediaFileOriginModelType::Rerender.to_str(), "rerender"); } #[test] @@ -107,18 +109,20 @@ mod tests { assert_eq!(MediaFileOriginModelType::from_str("so_vits_svc").unwrap(), MediaFileOriginModelType::SoVitsSvc); assert_eq!(MediaFileOriginModelType::from_str("tacotron2").unwrap(), MediaFileOriginModelType::Tacotron2); assert_eq!(MediaFileOriginModelType::from_str("vall_e_x").unwrap(), MediaFileOriginModelType::VallEX); + assert_eq!(MediaFileOriginModelType::from_str("rerender").unwrap(), MediaFileOriginModelType::Rerender); assert!(MediaFileOriginModelType::from_str("foo").is_err()); } #[test] fn all_variants() { let mut variants = MediaFileOriginModelType::all_variants(); - assert_eq!(variants.len(), 5); + assert_eq!(variants.len(), 6); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::VallEX)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Rerender)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 5965f9b3a9..103bab0042 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -80,6 +80,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::TextToSpeech, + Self::Rerender, Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, @@ -102,6 +103,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); + assert_serialization(MediaFileOriginProductCategory::Rerender, "rerender") } #[test] @@ -111,6 +113,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); + assert_eq!(MediaFileOriginProductCategory::Rerender.to_str(), "rerender"); } #[test] @@ -120,17 +123,19 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); + assert_eq!(MediaFileOriginProductCategory::from_str("rerender").unwrap(), MediaFileOriginProductCategory::Rerender); } #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 5); + assert_eq!(variants.len(), 6); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Rerender)); assert_eq!(variants.pop_first(), None); } } From 0ed04839d3a36c0270905aa08a7c4ab8473ff35a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 20:27:04 -0500 Subject: [PATCH 125/136] takedown request --- _sql/actions/voice_conversion_models.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 9fd7b4ee20..1c040d57da 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -371,7 +371,8 @@ where token IN ( 'vcm_xy97xcgnk9y6', 'vcm_a47wtfbsjrfc', 'vcm_kygenhbg5mgj', - 'vcm_s8gndv19yt7p' + 'vcm_s8gndv19yt7p', + 'vcm_ny9pj57w5eyh' ); From a25e9aba9115b0c462c3090564912b7c0376c770 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 13 Dec 2023 23:18:14 -0500 Subject: [PATCH 126/136] Seeding model weights --- .env | 2 +- Cargo.lock | 1 + crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_database_seed/src/main.rs | 6 +- .../cli/dev_database_seed/src/seeding/mod.rs | 3 +- .../src/seeding/seed_model_weights.rs | 241 ++++++++++++++++++ 6 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs diff --git a/.env b/.env index a39f1b1d8d..c41492d472 100644 --- a/.env +++ b/.env @@ -76,4 +76,4 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt # Development key. Permissible to check in. NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. +STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. diff --git a/Cargo.lock b/Cargo.lock index 25191d742f..2c3aad99e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1751,6 +1751,7 @@ dependencies = [ "mimetypes", "my-workspace-hack", "mysql_queries", + "rand 0.8.5", "sqlx", "storyteller_root", "tokens", diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index a771241cf5..a6e1bfc853 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -34,7 +34,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated - +rand = "0.8.5" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 558478772e..6a2290bea5 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -7,7 +7,8 @@ use errors::AnyhowResult; use crate::bucket_clients::get_bucket_clients; use crate::cli_args::parse_cli_args; -use crate::seeding::model_weights::seed_weights; +use crate::seeding::model_weights::seed_weights; // V1 +use crate::seeding::seed_model_weights::seed_weights_files; // V2 v3 will come later on when everything is more conslidated and clear. use crate::seeding::seed_media_files::seed_media_files; use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; @@ -60,6 +61,9 @@ pub async fn main() -> AnyhowResult<()> { seed_weights(&pool).await?; seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?; + + // should seed the weights with a few files for hanashi + seed_weights_files(&pool, maybe_bucket_clients.as_ref()).await?; info!("Done!"); Ok(()) diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index c1100805b6..7bc49a4b9a 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -3,4 +3,5 @@ pub mod seed_media_files; pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; -pub mod zero_shot_tts; \ No newline at end of file +pub mod zero_shot_tts; +pub mod seed_model_weights; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs new file mode 100644 index 0000000000..72ab9badd3 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs @@ -0,0 +1,241 @@ +use std::path::Path; + +use log::{ info, warn }; +use sqlx::{ MySql, Pool }; + +use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self }, private }; + +use filesys::file_read_bytes::file_read_bytes; +use filesys::file_size::file_size; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_for_file::get_mimetype_for_file; + +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::users::UserToken; +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +use enums::{ + by_table::model_weights::{ weights_category::WeightsCategory, weights_types::WeightsType }, + common::visibility::Visibility, +}; + +use rand::Rng; +use errors::{ anyhow, AnyhowResult }; +use mysql_queries::queries::model_weights::create_weight::{ + create_weight, + CreateModelWeightsArgs, + self, +}; +use tokens::tokens::model_weights::ModelWeightToken; + +pub async fn seed_weights_files( + mysql_pool: &Pool, + maybe_bucket_clients: Option<&BucketClients> +) -> AnyhowResult<()> { + info!("Seeding weights files..."); + + let seeded_weights = [ + ( + HANASHI_USER_TOKEN, + "m_21jxt406aattq2cna1xtgf5m4mjyy1", + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + "Niji Mecha", + "models/imagegen/loRA/nijiMecha.safetensors", + ), + ( + HANASHI_USER_TOKEN, + "m_cxzabpxf88h10rshqraksxbqmhs412", + WeightsType::StableDiffusion15, + WeightsCategory::ImageGeneration, + "Niji Diffused Mix", + "models/imagegen/sd15/nijidiffusedmix_v40.safetensors", + ), + ( + HANASHI_USER_TOKEN, + "m_21jxt406aattq2cna1xtgf5m4mjyy9", + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + "Niji Mecha", + "models/imagegen/loRA/nijiMecha.safetensors", + ), + ( + HANASHI_USER_TOKEN, + "m_cxzabpxf88h10rshqraksxbqmhs413", + WeightsType::StableDiffusion15, + WeightsCategory::ImageGeneration, + "Niji Diffused Mix", + "models/imagegen/sd15/nijidiffusedmix_v40.safetensors", + ), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + for ( + user_token, + weights_file_token, + weights_type, + weights_category, + model_name, + subdirectory_path, + ) in seeded_weights { + let user_token = UserToken::new_from_str(user_token); + // prepare the inputs for a model + let mut rng = rand::thread_rng(); + let n: u32 = rng.gen(); + + let model_weight_token = ModelWeightToken(format!("{}", n)); // random token + let model_weight_token = model_weight_token; + + let title = model_name; + let description = "This is a description"; + let description_rendered_html = "This is a description rendered html"; + let original_filename = "model.safetensors"; + let original_download_url = "https://example.com/model.safetensors"; + let thumbnail_token = format!("{}", n); + + let public_bucket_hash: String = weights_file_token.to_string(); + let public_bucket_prefix = weights_type.to_str(); + let public_bucket_extension = weights_category.to_str(); + + let cached_user_ratings_total_count = n.clone(); + let cached_user_ratings_positive_count = n.clone(); + let cached_user_ratings_negative_count = n.clone(); + let cached_user_ratings_ratio = 100.0 as f32; + let version = 1; + + // get the abs path of the weights file to upload. + let mut weight_file_path = seed_tool_data_root.clone(); + weight_file_path.push(subdirectory_path); + + let file_size_bytes = file_size(weight_file_path.clone())?; + let sha256_checksum = sha256_hash_file(weight_file_path.clone())?; + + let result = seed_model( + &mysql_pool, + &model_weight_token, + &user_token, + weights_type, + &weight_file_path, + maybe_bucket_clients + ).await; + + let create_model_weights_args = CreateModelWeightsArgs { + token: &model_weight_token, + weights_type: weights_type, // Assuming weights_type is defined elsewhere + weights_category: weights_category, // Assuming weights_category is defined elsewhere + title: title.to_string(), + maybe_thumbnail_token: Some(thumbnail_token), + description_markdown: description.to_string(), + description_rendered_html: description_rendered_html.to_string(), + creator_user_token: Some(&user_token), + creator_ip_address: "127.0.0.1", // Assuming the IP address is localhost + creator_set_visibility: Visibility::Public, // Assuming the visibility is public + maybe_last_update_user_token: None, // Assuming there is no last update user token + original_download_url: Some(original_download_url.to_string()), + original_filename: Some(original_filename.to_string()), + file_size_bytes: file_size_bytes as i32, // Assuming the file size is 0 + file_checksum_sha2: sha256_checksum.to_string(), // Assuming the file checksum is an empty string + private_bucket_hash: public_bucket_hash, + maybe_private_bucket_prefix: Some(public_bucket_prefix.to_string()), + maybe_private_bucket_extension: Some(public_bucket_extension.to_string()), + cached_user_ratings_total_count: cached_user_ratings_total_count, + cached_user_ratings_positive_count: cached_user_ratings_positive_count, + cached_user_ratings_negative_count: cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio), + version: version, + mysql_pool: &mysql_pool, // Assuming mysql_pool is defined elsewhere + }; + + create_weight(create_model_weights_args).await?; + + match result { + Ok(_) => info!("Seeded {:?}", weight_file_path), + Err(err) => + warn!( + r#" + Could not seed media file {} , {:?} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the media file on a previous invocation!) + "#, + model_weight_token, + subdirectory_path, + err + ), + } + } + + Ok(()) +} +struct BucketDetails { + bucket_hash: String, + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +async fn seed_model( + mysql_pool: &Pool, + model_weight_token: &ModelWeightToken, + user_token: &UserToken, + weight_type: WeightsType, + weight_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients> +) -> AnyhowResult<()> { + info!("Seeding Media file {:?} ...", weight_file_path); + // mutable variables + let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut maybe_bucket_prefix = ""; + let mut maybe_bucket_extension = ""; + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(weight_file_path, bucket_clients).await?; + bucket_hash = bucket_details.bucket_hash; + maybe_bucket_prefix = bucket_details.maybe_bucket_prefix.as_deref().unwrap(); + maybe_bucket_extension = bucket_details.maybe_bucket_extension.as_deref().unwrap(); + } + + Ok(()) +} + +async fn seed_file_to_bucket( + weight_file_path: &Path, + bucket_clients: &BucketClients +) -> AnyhowResult { + info!("Uploading weights file {:?} ...", weight_file_path); + + let maybe_bucket_prefix = Some("fakeyou_"); + + let maybe_bucket_extension = weight_file_path + .extension() + .map(|extension| extension.to_str()) + .flatten(); + // we should just have this be file bucket path and then we can use a file descriptor to turn it into a specific type of path + let bucket_location = MediaFileBucketPath::generate_new( + maybe_bucket_prefix, + maybe_bucket_extension + ); + + let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf()); + + info!("Reading weights file: {:?}", weight_file_path); + + let bytes = file_read_bytes(weight_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + info!("Uploading weights file to bucket path: {:?}", bucket_path); + + let _r = bucket_clients.public.upload_file_with_content_type( + &bucket_path, + bytes.as_ref(), + mimetype + ).await?; + + Ok(BucketDetails { + bucket_hash: bucket_location.get_object_hash().to_string(), + maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()), + maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()), + }) +} From aad6b4e30457cd516a5c9668af0673fda8a8da17 Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 13 Dec 2023 23:21:55 -0500 Subject: [PATCH 127/136] Update .env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index c41492d472..a39f1b1d8d 100644 --- a/.env +++ b/.env @@ -76,4 +76,4 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt # Development key. Permissible to check in. NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. +#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. From a0375f10e21cc6bf89902620f3978c58c47e32aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 10:48:58 -0500 Subject: [PATCH 128/136] Featured weights endpoint --- .../model_weights/weights_category.rs | 5 +- .../by_table/model_weights/weights_types.rs | 2 +- .../model_weights/list_weights_by_tokens.rs | 264 ++++++++++++++++++ .../src/queries/model_weights/mod.rs | 3 +- .../endpoints/weights/get_weight.rs | 4 +- .../weights/list_featured_weights.rs | 156 +++++++++++ .../src/http_server/endpoints/weights/mod.rs | 7 +- .../service/web/storyteller_web/src/routes.rs | 7 +- 8 files changed, 434 insertions(+), 14 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index f6c4e6ed3f..f2b10e67d2 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -4,9 +4,8 @@ use std::collections::BTreeSet; use strum::EnumCount; #[cfg(test)] use strum::EnumIter; - - use utoipa::ToSchema; + #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { @@ -51,7 +50,7 @@ impl WeightsCategory { } impl_enum_display_and_debug_using_to_str!(WeightsCategory); impl_mysql_enum_coders!(WeightsCategory); - +impl_mysql_from_row!(WeightsCategory); #[cfg(test)] mod tests { diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 1b6bbcb132..12f00955df 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -72,7 +72,7 @@ impl WeightsType { impl_enum_display_and_debug_using_to_str!(WeightsType); impl_mysql_enum_coders!(WeightsType); - +impl_mysql_from_row!(WeightsType); #[cfg(test)] mod tests { diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs new file mode 100644 index 0000000000..f08a9534a9 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs @@ -0,0 +1,264 @@ +use chrono::{DateTime, Utc}; +use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +#[derive(Serialize)] +pub struct WeightsByTokensRecord { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn list_weights_by_tokens( + mysql_pool: &MySqlPool, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let mut connection = mysql_pool.acquire().await?; + + let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + + Ok(map_to_weights(raw_weights)) +} + +async fn get_raw_weights_by_tokens( + connection: &mut MySqlConnection, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let connection = connection.acquire().await?; + + let mut query_builder : QueryBuilder = if can_see_deleted { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.token IN ( + "#, + ) + + } else { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.user_deleted_at IS NULL + AND mw.mod_deleted_at IS NULL + AND mw.token IN ( + "# + ) + }; + + query_builder.push(token_predicate(weight_tokens)); + + query_builder.push(")"); + + let query = query_builder.build_query_as::(); + + let results = query.fetch_all(connection).await?; + + Ok(results) +} + +/// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. +/// Issue: https://github.com/launchbadge/sqlx/issues/875 +fn token_predicate(tokens: &[ModelWeightToken]) -> String { + tokens.iter() + .map(|ty| ty.as_str()) + .map(|ty| format!("\"{}\"", ty)) + .collect::>() + .join(", ") +} + +fn map_to_weights(dataset:Vec) -> Vec { + let weights: Vec = dataset + .into_iter() + .map(|dataset: RawWeightJoinUser| { + WeightsByTokensRecord { + token: dataset.token, + title: dataset.title, + weights_type: dataset.weights_type, + weights_category: dataset.weights_category, + maybe_thumbnail_token: dataset.maybe_thumbnail_token, + + creator_user_token: dataset.creator_user_token, + creator_username:dataset.creator_username, + creator_display_name:dataset.creator_display_name, + creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + + private_bucket_hash: dataset.private_bucket_hash, + maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, + maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + + cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + + maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, + + created_at: dataset.created_at, + updated_at: dataset.updated_at, + user_deleted_at: dataset.user_deleted_at, + mod_deleted_at: dataset.mod_deleted_at, + } + }).collect(); + + weights +} + + #[derive(Serialize)] + pub struct RawWeightJoinUser { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for RawWeightJoinUser { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + token: ModelWeightToken::new(row.try_get("token")?), + weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, + weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, + title: row.try_get("title")?, + maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, + creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), + creator_username: row.try_get("creator_username")?, + creator_display_name: row.try_get("creator_display_name")?, + creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, + private_bucket_hash: row.try_get("private_bucket_hash")?, + maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, + maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, + cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, + cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, + cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, + maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, + cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + user_deleted_at: row.try_get("user_deleted_at")?, + mod_deleted_at: row.try_get("mod_deleted_at")?, + }) + } +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs index 898685caa6..06aeec0264 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs @@ -1,7 +1,8 @@ pub mod create_weight; pub mod delete_weights; pub mod get_weight; -pub mod update_weight; +pub mod list_weights_by_tokens; pub mod list_weights_by_user; pub mod list_weights_query_builder; pub mod tests; +pub mod update_weight; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 02be96fef3..0b3f059aae 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -109,12 +109,12 @@ pub async fn get_weight_handler( } }; - let weight_token = path.weight_token.clone(); + let weight_token = ModelWeightToken::new_from_str(&path.weight_token); let creator_user_token = user_session.user_token.clone(); let is_mod = user_session.can_ban_users; let weight_lookup_result = get_weight_by_token( - &ModelWeightToken::new(weight_token.clone()), + &weight_token, is_mod, &server_state.mysql_pool, ).await; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs new file mode 100644 index 0000000000..33f921b722 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs @@ -0,0 +1,156 @@ +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{DateTime, Utc}; +use log::{debug, error, warn}; +use r2d2_redis::redis::Commands; +use utoipa::ToSchema; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, ToSchema)] +pub struct ListFeaturedWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, +} + +#[derive(Serialize, ToSchema)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator: UserDetailsLight, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// The key we store featured weights tokens under +const REDIS_KEY : &str = "featured_weights_list"; + +#[derive(Debug, ToSchema)] +pub enum ListFeaturedWeightsError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListFeaturedWeightsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListFeaturedWeightsError { + fn status_code(&self) -> StatusCode { + match *self { + ListFeaturedWeightsError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListFeaturedWeightsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/weights/list_featured", + responses( + (status = 200, description = "List Weights", body = ListFeaturedWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListFeaturedWeightsError), + (status = 500, description = "Server error", body = ListFeaturedWeightsError), + ), +)] +pub async fn list_featured_weights_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let mut redis = server_state.redis_pool.get() + .map_err(|err| { + error!("Could not obtain redis: {err}"); + ListFeaturedWeightsError::ServerError + })?; + + let token_list : Option = redis.get(REDIS_KEY) + .map_err(|err| { + error!("Could not get redis result: {err}"); + ListFeaturedWeightsError::ServerError + })?; + + let weight_tokens = token_list + .unwrap_or_else(|| "".to_string()) + .split(",") + .into_iter() + .map(|item| item.trim()) + .filter(|item| !item.is_empty()) + .map(|item| ModelWeightToken::new_from_str(item)) + .collect::>(); + + debug!("Weight tokens from Redis: {:?}", weight_tokens); + + let mut weights = Vec::new(); + + if !weight_tokens.is_empty() { + let query_results = + list_weights_by_tokens(&server_state.mysql_pool, &weight_tokens, false).await; + + weights = match query_results { + Ok(weights) => weights, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListFeaturedWeightsError::ServerError); + } + }; + } + + // generate parse a response + let response = ListFeaturedWeightsSuccessResponse { + success: true, + weights: weights.into_iter() + .map(|w| ModelWeightForList { + weight_token: w.token, + title: w.title, + weights_type: w.weights_type, + weights_category: w.weights_category, + maybe_thumbnail_token: w.maybe_thumbnail_token, + creator: UserDetailsLight::from_db_fields( + &w.creator_user_token, + &w.creator_username, + &w.creator_display_name, + &w.creator_email_gravatar_hash + ), + cached_user_ratings_total_count: w.cached_user_ratings_total_count, + cached_user_ratings_positive_count: w.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: w.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: w.maybe_cached_user_ratings_ratio, + created_at: w.created_at, + updated_at: w.updated_at, + }).collect::>(), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListFeaturedWeightsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index 3fbd1cb09d..f52604fe31 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -1,7 +1,8 @@ -pub mod get_weight; pub mod delete_weight; -pub mod update_weight; -pub mod list_weights_by_user; +pub mod get_weight; pub mod list_available_weights; +pub mod list_featured_weights; +pub mod list_weights_by_user; +pub mod update_weight; // pub mod create_weight; this is a private api call internally // pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9dcdce3e35..0d13225c1a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -156,6 +156,7 @@ use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; +use crate::http_server::endpoints::weights::list_featured_weights::list_featured_weights_handler; use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; @@ -1265,9 +1266,7 @@ fn add_weights_routes(app: App) -> App .route(web::delete().to(delete_weight_handler)) ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) - .route( - "/list", - web::get().to(list_available_weights_handler) - ) + .route("/list", web::get().to(list_available_weights_handler)) + .route("/list_featured", web::get().to(list_featured_weights_handler)) ) } From faff80fcc64747cd56d71e9cc9ab380f2341e17b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 03:20:47 -0500 Subject: [PATCH 129/136] deterministic RNGs for tests --- Cargo.lock | 1 + crates/_types/tokens/Cargo.toml | 4 +- crates/_types/tokens/src/deterministic_rng.rs | 43 +++++++++++++++++++ crates/_types/tokens/src/lib.rs | 1 + crates/_types/tokens/src/macros.rs | 35 +++++++++++++++ .../tokens/tests/media_file_token_tests.rs | 24 +++++++++++ .../weights/list_featured_weights.rs | 1 - 7 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 crates/_types/tokens/src/deterministic_rng.rs diff --git a/Cargo.lock b/Cargo.lock index 19da97d557..af9894f874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6442,6 +6442,7 @@ name = "tokens" version = "0.0.1" dependencies = [ "my-workspace-hack", + "once_cell", "rand 0.8.5", "serde", "serde_derive", diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index 581f8c65de..94152dee1b 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -25,7 +25,9 @@ serde = "1.0.125" serde_derive = "1.0.125" # Random -rand = "0.8.5" +rand = { version = "0.8.5", features = ["std_rng"] } + +once_cell = { version = "1.18.0", features = [] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } diff --git a/crates/_types/tokens/src/deterministic_rng.rs b/crates/_types/tokens/src/deterministic_rng.rs new file mode 100644 index 0000000000..90ad47f5ab --- /dev/null +++ b/crates/_types/tokens/src/deterministic_rng.rs @@ -0,0 +1,43 @@ +use std::sync::{Arc, LockResult, RwLock, RwLockWriteGuard}; + +use once_cell::sync::Lazy; +use rand::rngs::StdRng; +use rand::SeedableRng; + +// FIXME(bt,2023-12-14): Is this threadsafe? Looks like it should be fine. +/// Only use this in tests (and dev seed tools), never in production! +#[derive(Clone)] +pub struct DeterministicRng { + rng : Arc> +} + +impl DeterministicRng { + pub fn get_instance() -> LockResult> { + static DETERMINISTIC_RNG : Lazy>> = Lazy::new(|| { + Arc::new(RwLock::new(DeterministicRng::new())) + }); + + DETERMINISTIC_RNG.write() + } + + fn new() -> Self { + Self { + rng: Arc::new(RwLock::new(StdRng::seed_from_u64(0))) + } + } + + /// Reset the deterministic RNG's seed. + pub fn reset_rng(&self) { + match self.rng.write() { + Err(err) => panic!(format!("test panic due to rng failure {err}")), + Ok(mut lock) => { + *lock = StdRng::seed_from_u64(0); + } + } + } + + /// Grab the deterministic RNG + pub fn get_rng(&self) -> LockResult>{ + self.rng.write() + } +} diff --git a/crates/_types/tokens/src/lib.rs b/crates/_types/tokens/src/lib.rs index df6e81b4f2..b220bdb751 100644 --- a/crates/_types/tokens/src/lib.rs +++ b/crates/_types/tokens/src/lib.rs @@ -49,6 +49,7 @@ pub(crate) static MINIMUM_CHARACTER_ENTROPY : usize = 8; #[macro_use] mod macros; +pub (crate) mod deterministic_rng; pub (crate) mod prefixes; pub mod tokens; diff --git a/crates/_types/tokens/src/macros.rs b/crates/_types/tokens/src/macros.rs index 5964b6109a..4df28db41a 100644 --- a/crates/_types/tokens/src/macros.rs +++ b/crates/_types/tokens/src/macros.rs @@ -60,6 +60,41 @@ macro_rules! impl_crockford_generator { $t(token) } + /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. + /// This will not use a real RNG, so do not use in production code! + /// Once called, it will output the same random numbers in sequence. + pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously() { + let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); + deterministic.reset_rng(); + } + + /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. + /// This will not use a real RNG, so do not use in production code! + /// Once called, it will output the same random numbers in sequence. + pub fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously() -> Self { + use rand::Rng; + + // FIXME(bt,2023-12-14): This is bad error handling. (Maybe panic if lock is poisoned?) + let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); + let mut rng = deterministic.get_rng().expect("tests should not fail due to mutex"); + + let charset = Self::token_character_set(); + let entropy_length = Self::entropic_character_len(); + + let entropy_part: String = (0..entropy_length) + .map(|_| { + let idx = rng.gen_range(0..charset.len()); + charset[idx] as char + }) + .collect(); + + let token_prefix = Self::token_prefix(); + + let token = format!("{}{}", token_prefix, entropy_part); + + $t(token) + } + #[inline] pub fn entropy_suffix(&self) -> &str { use crate::prefixes::PrefixGenerator; diff --git a/crates/_types/tokens/tests/media_file_token_tests.rs b/crates/_types/tokens/tests/media_file_token_tests.rs index dc09692485..9a8704e1b2 100644 --- a/crates/_types/tokens/tests/media_file_token_tests.rs +++ b/crates/_types/tokens/tests/media_file_token_tests.rs @@ -21,6 +21,30 @@ mod interface { assert!(token.to_string().starts_with("m_")); } + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_1() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_2() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_3() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + #[test] fn new() { let token = MediaFileToken::new("m_foo".to_string()); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs index 33f921b722..87d75eed0f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs @@ -122,7 +122,6 @@ pub async fn list_featured_weights_handler( }; } - // generate parse a response let response = ListFeaturedWeightsSuccessResponse { success: true, weights: weights.into_iter() From d447ad7d8676f1ea2f3e5304b9c3fed77278acd8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 03:49:58 -0500 Subject: [PATCH 130/136] deterministic test tokens it'd be good to use the same token values in tests, dev seed tools, etc. this should be a threadsafe, resettable, deterministic RNG --- crates/_types/tokens/src/deterministic_rng.rs | 4 +- crates/_types/tokens/src/macros.rs | 4 +- .../tokens/tests/media_file_token_tests.rs | 17 +++---- .../src/seeding/model_weights.rs | 44 ++++++++++--------- .../src/seeding/seed_model_weights.rs | 44 +++++++++---------- 5 files changed, 54 insertions(+), 59 deletions(-) diff --git a/crates/_types/tokens/src/deterministic_rng.rs b/crates/_types/tokens/src/deterministic_rng.rs index 90ad47f5ab..affd9fda00 100644 --- a/crates/_types/tokens/src/deterministic_rng.rs +++ b/crates/_types/tokens/src/deterministic_rng.rs @@ -27,11 +27,11 @@ impl DeterministicRng { } /// Reset the deterministic RNG's seed. - pub fn reset_rng(&self) { + pub fn reset_rng(&self, state: u64) { match self.rng.write() { Err(err) => panic!(format!("test panic due to rng failure {err}")), Ok(mut lock) => { - *lock = StdRng::seed_from_u64(0); + *lock = StdRng::seed_from_u64(state); } } } diff --git a/crates/_types/tokens/src/macros.rs b/crates/_types/tokens/src/macros.rs index 4df28db41a..44fc14d6df 100644 --- a/crates/_types/tokens/src/macros.rs +++ b/crates/_types/tokens/src/macros.rs @@ -63,9 +63,9 @@ macro_rules! impl_crockford_generator { /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. /// This will not use a real RNG, so do not use in production code! /// Once called, it will output the same random numbers in sequence. - pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously() { + pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(state: u64) { let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); - deterministic.reset_rng(); + deterministic.reset_rng(state); } /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. diff --git a/crates/_types/tokens/tests/media_file_token_tests.rs b/crates/_types/tokens/tests/media_file_token_tests.rs index 9a8704e1b2..722f89c0a5 100644 --- a/crates/_types/tokens/tests/media_file_token_tests.rs +++ b/crates/_types/tokens/tests/media_file_token_tests.rs @@ -23,23 +23,18 @@ mod interface { #[test] fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_1() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // NB: Using the same reset seed will produce the same results each time + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); - } - - #[test] - fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_2() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // NB: Same seed -> same tokens generated + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); - } - - #[test] - fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_3() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // Once more... + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 8860d323c7..13e23d6014 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -132,7 +132,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT - Gawr Gura MOS 4 "#; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura: {}", i); description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); description_rendered_html = rvc_markdown_description; @@ -155,7 +155,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT }, 21..=40 =>{ // TTS - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("dragonfruitGTv1: {}", i); description = format!("Description Number {}:", i); @@ -187,7 +187,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT }, 41..=60 => { // SD 1.5 - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("dragonfruitGTv1: {}", i); description = format!("Description Number {}:", i); @@ -232,7 +232,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT "#; let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("SDXL_Niji_Special Edition: {}", i); description = format!("Description Number {}:", i); @@ -263,7 +263,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT "#; let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg"; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura LoRA: {}", i); description = format!("Description Number {}:", i); @@ -295,7 +295,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT - Gawr Gura MOS 4 "#; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura: {}", i); description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); description_rendered_html = rvc_markdown_description; @@ -356,11 +356,13 @@ pub async fn seed_weights_for_user_token( ) -> AnyhowResult<()> { info!("Seeding weights..."); - let model_weight_token1 = ModelWeightToken("11".to_string()); - let model_weight_token2 = ModelWeightToken("22".to_string()); - let model_weight_token3 = ModelWeightToken("33".to_string()); - let model_weight_token4 = ModelWeightToken("44".to_string()); - let model_weight_token5 = ModelWeightToken("55".to_string()); + ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(54321); + + let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); let model_weights_args = vec![ CreateModelWeightsArgs { @@ -503,20 +505,20 @@ pub async fn seed_weights_for_user_token( pub async fn original_seed_weights(mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { - let model_weight_token1 = ModelWeightToken("1".to_string()); - let creator_token1 = UserToken("creatorToken1".to_string()); + let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token1 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token2 = ModelWeightToken("2".to_string()); - let creator_token2 = UserToken("creatorToken2".to_string()); + let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token2 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token3 = ModelWeightToken("3".to_string()); - let creator_token3 = UserToken("creatorToken3".to_string()); + let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token3 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token4 = ModelWeightToken("4".to_string()); - let creator_token4 = UserToken("creatorToken4".to_string()); + let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token4 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token5 = ModelWeightToken("5".to_string()); - let creator_token5 = UserToken("creatorToken5".to_string()); + let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token5 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); let model_weights_args = vec![ CreateModelWeightsArgs { diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs index 72ab9badd3..ac6388b877 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs @@ -1,35 +1,31 @@ use std::path::Path; -use log::{ info, warn }; -use sqlx::{ MySql, Pool }; - -use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self }, private }; +use log::{info, warn}; +use rand::Rng; +use sqlx::{MySql, Pool}; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::{ + by_table::model_weights::{weights_category::WeightsCategory, weights_types::WeightsType}, + common::visibility::Visibility, +}; +use errors::AnyhowResult; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; -use mimetypes::mimetype_for_file::get_mimetype_for_file; - -use storyteller_root::get_seed_tool_data_root; -use tokens::tokens::users::UserToken; -use crate::bucket_clients::BucketClients; -use crate::seeding::users::HANASHI_USER_TOKEN; - -use enums::{ - by_table::model_weights::{ weights_category::WeightsCategory, weights_types::WeightsType }, - common::visibility::Visibility, -}; - -use rand::Rng; -use errors::{ anyhow, AnyhowResult }; use mysql_queries::queries::model_weights::create_weight::{ create_weight, - CreateModelWeightsArgs, - self, + CreateModelWeightsArgs + , }; +use storyteller_root::get_seed_tool_data_root; use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; pub async fn seed_weights_files( mysql_pool: &Pool, @@ -37,6 +33,9 @@ pub async fn seed_weights_files( ) -> AnyhowResult<()> { info!("Seeding weights files..."); + // NB: Stable token generation. + ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(123); + let seeded_weights = [ ( HANASHI_USER_TOKEN, @@ -87,9 +86,8 @@ pub async fn seed_weights_files( let mut rng = rand::thread_rng(); let n: u32 = rng.gen(); - let model_weight_token = ModelWeightToken(format!("{}", n)); // random token - let model_weight_token = model_weight_token; - + let model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let title = model_name; let description = "This is a description"; let description_rendered_html = "This is a description rendered html"; From eee75f0711b2e76a9b90fed588207f38ce9bc119 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 04:37:44 -0500 Subject: [PATCH 131/136] polymorphic token checks (thanks kasisnu!) + new token types --- .../by_table/comments/comment_entity_type.rs | 18 ++++++++++ .../user_bookmark_entity_type.rs | 9 +++++ .../src/by_table/user_ratings/entity_type.rs | 18 ++++++++++ .../user_ratings/user_rating_entity.rs | 8 +++++ .../queries/comments/comment_entity_token.rs | 8 +++++ .../user_bookmarks/list_user_bookmarks.rs | 4 +++ .../list_user_bookmarks_result.rs | 2 ++ .../user_bookmark_entity_token.rs | 4 +++ .../comments/create_comment_handler.rs | 27 +++++++++++++++ .../create_user_bookmark_handler.rs | 34 ++++++++++++++++++- .../user_ratings/get_user_rating_handler.rs | 7 +++- .../user_ratings/set_user_rating_handler.rs | 32 +++++++++++++++-- 12 files changed, 167 insertions(+), 4 deletions(-) diff --git a/crates/_types/enums/src/by_table/comments/comment_entity_type.rs b/crates/_types/enums/src/by_table/comments/comment_entity_type.rs index 72294e2b27..014e04e17f 100644 --- a/crates/_types/enums/src/by_table/comments/comment_entity_type.rs +++ b/crates/_types/enums/src/by_table/comments/comment_entity_type.rs @@ -13,6 +13,14 @@ pub enum CommentEntityType { #[serde(rename = "user")] User, + /// Media files + #[serde(rename = "media_file")] + MediaFile, + + /// Model weights + #[serde(rename = "model_weight")] + ModelWeight, + /// TTS model (architecture does not matter) #[serde(rename = "tts_model")] TtsModel, @@ -39,6 +47,8 @@ impl CommentEntityType { pub fn to_str(&self) -> &'static str { match self { Self::User => "user", + Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::TtsModel => "tts_model", Self::TtsResult => "tts_result", Self::W2lTemplate => "w2l_template", @@ -49,6 +59,8 @@ impl CommentEntityType { pub fn from_str(value: &str) -> Result { match value { "user" => Ok(Self::User), + "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "tts_model" => Ok(Self::TtsModel), "tts_result" => Ok(Self::TtsResult), "w2l_template" => Ok(Self::W2lTemplate), @@ -69,6 +81,8 @@ mod tests { #[test] fn test_serialization() { assert_serialization(CommentEntityType::User, "user"); + assert_serialization(CommentEntityType::MediaFile, "media_file"); + assert_serialization(CommentEntityType::ModelWeight, "model_weight"); assert_serialization(CommentEntityType::TtsModel, "tts_model"); assert_serialization(CommentEntityType::TtsResult, "tts_result"); assert_serialization(CommentEntityType::W2lTemplate, "w2l_template"); @@ -82,6 +96,8 @@ mod tests { #[test] fn test_to_str() { assert_eq!(CommentEntityType::User.to_str(), "user"); + assert_eq!(CommentEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(CommentEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(CommentEntityType::TtsModel.to_str(), "tts_model"); assert_eq!(CommentEntityType::TtsResult.to_str(), "tts_result"); assert_eq!(CommentEntityType::W2lTemplate.to_str(), "w2l_template"); @@ -91,6 +107,8 @@ mod tests { #[test] fn test_from_str() { assert_eq!(CommentEntityType::from_str("user").unwrap(), CommentEntityType::User); + assert_eq!(CommentEntityType::from_str("media_file").unwrap(), CommentEntityType::MediaFile); + assert_eq!(CommentEntityType::from_str("model_weight").unwrap(), CommentEntityType::ModelWeight); assert_eq!(CommentEntityType::from_str("tts_model").unwrap(), CommentEntityType::TtsModel); assert_eq!(CommentEntityType::from_str("tts_result").unwrap(), CommentEntityType::TtsResult); assert_eq!(CommentEntityType::from_str("w2l_template").unwrap(), CommentEntityType::W2lTemplate); diff --git a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs index 69bf47cf5e..1e3fa6692c 100644 --- a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs +++ b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs @@ -33,6 +33,10 @@ pub enum UserBookmarkEntityType { #[serde(rename = "media_file")] MediaFile, + /// ModelWeight (the new way to store models) + #[serde(rename = "model_weight")] + ModelWeight, + /// VoiceConversionModel #[serde(rename = "voice_conversion_model")] VoiceConversionModel, @@ -56,6 +60,7 @@ impl UserBookmarkEntityType { Self::W2lTemplate => "w2l_template", Self::W2lResult => "w2l_result", Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::VoiceConversionModel => "voice_conversion_model", Self::ZsVoice => "zs_voice", } @@ -69,6 +74,7 @@ impl UserBookmarkEntityType { "w2l_template" => Ok(Self::W2lTemplate), "w2l_result" => Ok(Self::W2lResult), "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "voice_conversion_model" => Ok(Self::VoiceConversionModel), "zs_voice" => Ok(Self::ZsVoice), _ => Err(format!("invalid value: {:?}", value)), @@ -92,6 +98,7 @@ mod tests { assert_serialization(UserBookmarkEntityType::W2lTemplate, "w2l_template"); assert_serialization(UserBookmarkEntityType::W2lResult, "w2l_result"); assert_serialization(UserBookmarkEntityType::MediaFile, "media_file"); + assert_serialization(UserBookmarkEntityType::ModelWeight, "model_weight"); assert_serialization(UserBookmarkEntityType::VoiceConversionModel, "voice_conversion_model"); assert_serialization(UserBookmarkEntityType::ZsVoice, "zs_voice"); } @@ -108,6 +115,7 @@ mod tests { assert_eq!(UserBookmarkEntityType::W2lTemplate.to_str(), "w2l_template"); assert_eq!(UserBookmarkEntityType::W2lResult.to_str(), "w2l_result"); assert_eq!(UserBookmarkEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserBookmarkEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(UserBookmarkEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); assert_eq!(UserBookmarkEntityType::ZsVoice.to_str(), "zs_voice"); } @@ -120,6 +128,7 @@ mod tests { assert_eq!(UserBookmarkEntityType::from_str("w2l_template").unwrap(), UserBookmarkEntityType::W2lTemplate); assert_eq!(UserBookmarkEntityType::from_str("w2l_result").unwrap(), UserBookmarkEntityType::W2lResult); assert_eq!(UserBookmarkEntityType::from_str("media_file").unwrap(), UserBookmarkEntityType::MediaFile); + assert_eq!(UserBookmarkEntityType::from_str("model_weight").unwrap(), UserBookmarkEntityType::ModelWeight); assert_eq!(UserBookmarkEntityType::from_str("voice_conversion_model").unwrap(), UserBookmarkEntityType::VoiceConversionModel); assert_eq!(UserBookmarkEntityType::from_str("zs_voice").unwrap(), UserBookmarkEntityType::ZsVoice); assert!(UserBookmarkEntityType::from_str("foo").is_err()); diff --git a/crates/_types/enums/src/by_table/user_ratings/entity_type.rs b/crates/_types/enums/src/by_table/user_ratings/entity_type.rs index f463471a98..025a8e4d6c 100644 --- a/crates/_types/enums/src/by_table/user_ratings/entity_type.rs +++ b/crates/_types/enums/src/by_table/user_ratings/entity_type.rs @@ -9,6 +9,14 @@ use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] pub enum UserRatingEntityType { + /// Media files (inference results, uploads, etc.) + #[serde(rename = "media_file")] + MediaFile, + + /// Model weights (modern, polymorphic, type agnostic) + #[serde(rename = "model_weight")] + ModelWeight, + /// TTS model (architecture does not matter) #[serde(rename = "tts_model")] TtsModel, @@ -34,6 +42,8 @@ impl_mysql_enum_coders!(UserRatingEntityType); impl UserRatingEntityType { pub fn to_str(&self) -> &'static str { match self { + Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::TtsModel => "tts_model", Self::TtsResult => "tts_result", Self::W2lTemplate => "w2l_template", @@ -43,6 +53,8 @@ impl UserRatingEntityType { pub fn from_str(value: &str) -> Result { match value { + "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "tts_model" => Ok(Self::TtsModel), "tts_result" => Ok(Self::TtsResult), "w2l_template" => Ok(Self::W2lTemplate), @@ -62,6 +74,8 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(UserRatingEntityType::MediaFile, "media_file"); + assert_serialization(UserRatingEntityType::ModelWeight, "model_weight"); assert_serialization(UserRatingEntityType::TtsModel, "tts_model"); assert_serialization(UserRatingEntityType::TtsResult, "tts_result"); assert_serialization(UserRatingEntityType::W2lTemplate, "w2l_template"); @@ -74,6 +88,8 @@ mod tests { #[test] fn test_to_str() { + assert_eq!(UserRatingEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserRatingEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(UserRatingEntityType::TtsModel.to_str(), "tts_model"); assert_eq!(UserRatingEntityType::TtsResult.to_str(), "tts_result"); assert_eq!(UserRatingEntityType::W2lTemplate.to_str(), "w2l_template"); @@ -82,6 +98,8 @@ mod tests { #[test] fn test_from_str() { + assert_eq!(UserRatingEntityType::from_str("media_file").unwrap(), UserRatingEntityType::MediaFile); + assert_eq!(UserRatingEntityType::from_str("model_weight").unwrap(), UserRatingEntityType::ModelWeight); assert_eq!(UserRatingEntityType::from_str("tts_model").unwrap(), UserRatingEntityType::TtsModel); assert_eq!(UserRatingEntityType::from_str("tts_result").unwrap(), UserRatingEntityType::TtsResult); assert_eq!(UserRatingEntityType::from_str("w2l_template").unwrap(), UserRatingEntityType::W2lTemplate); diff --git a/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs b/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs index a5fe4d14b5..a100f7df30 100644 --- a/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs +++ b/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs @@ -1,4 +1,6 @@ use enums::by_table::user_ratings::entity_type::UserRatingEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; @@ -12,6 +14,8 @@ use tokens::tokens::w2l_templates::W2lTemplateToken; /// NB: We probably don't want to expose this type of interface to web clients because it's /// monotonous as a JSON interface. pub enum UserRatingEntity { + MediaFile(MediaFileToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), W2lTemplate(W2lTemplateToken), } @@ -20,6 +24,8 @@ pub enum UserRatingEntity { impl UserRatingEntity { pub fn get_entity_type(&self) -> UserRatingEntityType { match self { + UserRatingEntity::MediaFile(_) => UserRatingEntityType::MediaFile, + UserRatingEntity::ModelWeight(_) => UserRatingEntityType::ModelWeight, UserRatingEntity::TtsModel(_) => UserRatingEntityType::TtsModel, UserRatingEntity::W2lTemplate(_) => UserRatingEntityType::W2lTemplate, } @@ -27,6 +33,8 @@ impl UserRatingEntity { pub fn get_entity_token_str(&self) -> &str { match self { + UserRatingEntity::MediaFile(token) => token.as_str(), + UserRatingEntity::ModelWeight(token) => token.as_str(), UserRatingEntity::TtsModel(token) => token.as_str(), UserRatingEntity::W2lTemplate(token) => token.as_str(), } diff --git a/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs b/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs index 47f9189413..885ceac73a 100644 --- a/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs +++ b/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs @@ -1,10 +1,14 @@ use enums::by_table::comments::comment_entity_type::CommentEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; use tokens::tokens::users::UserToken; pub enum CommentEntityToken { User(UserToken), + MediaFile(MediaFileToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), TtsResult(String), // TODO: Strong type W2lTemplate(W2lTemplateToken), @@ -20,6 +24,8 @@ impl CommentEntityToken { CommentEntityType::TtsResult => Self::TtsResult(token.to_string()), CommentEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), CommentEntityType::W2lResult => Self::W2lResult(token.to_string()), + CommentEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), + CommentEntityType::ModelWeight => Self::ModelWeight(ModelWeightToken::new_from_str(token)), } } @@ -30,6 +36,8 @@ impl CommentEntityToken { CommentEntityToken::TtsResult(tts_result_token) => (CommentEntityType::TtsResult, tts_result_token.as_str()), CommentEntityToken::W2lTemplate(w2l_template_token) => (CommentEntityType::W2lTemplate, w2l_template_token.as_str()), CommentEntityToken::W2lResult(w2l_result_token) => (CommentEntityType::W2lResult, w2l_result_token.as_str()), + CommentEntityToken::MediaFile(media_file_token) => (CommentEntityType::MediaFile, media_file_token.as_str()), + CommentEntityToken::ModelWeight(model_weight_token) => (CommentEntityType::ModelWeight, model_weight_token.as_str()), }; (entity_type, entity_token) } diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index 232c7344a6..c28ab16129 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -33,6 +33,7 @@ SELECT media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`, media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + model_weights.title as maybe_descriptive_text_model_weight_title, tts_models.title as maybe_descriptive_text_tts_model_title, tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text, users.display_name as maybe_descriptive_text_user_display_name, @@ -44,6 +45,7 @@ FROM JOIN users AS u ON f.user_token = u.token +LEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token LEFT OUTER JOIN media_files ON media_files.token = f.entity_token LEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token LEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token @@ -100,6 +102,7 @@ SELECT media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`, media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + model_weights.title as maybe_descriptive_text_model_weight_title, tts_models.title as maybe_descriptive_text_tts_model_title, tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text, users.display_name as maybe_descriptive_text_user_display_name, @@ -111,6 +114,7 @@ FROM JOIN users AS u ON f.user_token = u.token +LEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token LEFT OUTER JOIN media_files ON media_files.token = f.entity_token LEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token LEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index e56d1de9d8..f384fbcd7e 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -44,6 +44,7 @@ pub struct RawUserBookmarkRecord { pub (crate) maybe_media_file_type: Option, pub (crate) maybe_media_file_origin_category: Option, + pub (crate) maybe_descriptive_text_model_weight_title: Option, pub (crate) maybe_descriptive_text_tts_model_title: Option, pub (crate) maybe_descriptive_text_tts_result_inference_text: Option, pub (crate) maybe_descriptive_text_user_display_name: Option, @@ -59,6 +60,7 @@ impl RawUserBookmarkRecord { entity_token: self.entity_token, maybe_entity_descriptive_text: match self.entity_type { UserBookmarkEntityType::User => self.maybe_descriptive_text_user_display_name, + UserBookmarkEntityType::ModelWeight => self.maybe_descriptive_text_model_weight_title, UserBookmarkEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, UserBookmarkEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, UserBookmarkEntityType::W2lTemplate => None, diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs index 5fbf8224ad..aee34d7deb 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs @@ -1,5 +1,6 @@ use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::tts_results::TtsResultToken; use tokens::tokens::users::UserToken; @@ -10,6 +11,7 @@ use tokens::tokens::zs_voices::ZsVoiceToken; pub enum UserBookmarkEntityToken { User(UserToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), TtsResult(TtsResultToken), W2lTemplate(W2lTemplateToken), @@ -23,6 +25,7 @@ impl UserBookmarkEntityToken { pub fn from_entity_type_and_token(entity_type: UserBookmarkEntityType, token: &str) -> Self { match entity_type { UserBookmarkEntityType::User => Self::User(UserToken::new_from_str(token)), + UserBookmarkEntityType::ModelWeight => Self::ModelWeight(ModelWeightToken::new_from_str(token)), UserBookmarkEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), UserBookmarkEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), UserBookmarkEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), @@ -36,6 +39,7 @@ impl UserBookmarkEntityToken { pub fn get_composite_keys(&self) -> (UserBookmarkEntityType, &str) { match self { UserBookmarkEntityToken::User(user_token) => (UserBookmarkEntityType::User, user_token.as_str()), + UserBookmarkEntityToken::ModelWeight(model_weight_token) => (UserBookmarkEntityType::ModelWeight, model_weight_token.as_str()), UserBookmarkEntityToken::TtsModel(tts_model_token) => (UserBookmarkEntityType::TtsModel, tts_model_token.as_str()), UserBookmarkEntityToken::TtsResult(tts_result_token) => (UserBookmarkEntityType::TtsResult, tts_result_token.as_str()), UserBookmarkEntityToken::W2lTemplate(w2l_template_token) => (UserBookmarkEntityType::W2lTemplate, w2l_template_token.as_str()), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs index 09acffdb8b..8e6f98b2ee 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs @@ -16,6 +16,13 @@ use http_server_common::request::get_request_ip::get_request_ip; use mysql_queries::queries::comments::comment_entity_token::CommentEntityToken; use mysql_queries::queries::comments::insert_comment::{insert_comment, InsertCommentArgs}; use tokens::tokens::comments::CommentToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; use user_input_common::check_for_slurs::contains_slurs; use user_input_common::markdown_to_html::markdown_to_html; @@ -76,6 +83,26 @@ pub async fn create_comment_handler( server_state: web::Data>, ) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + CommentEntityType::User => token.starts_with(UserToken::token_prefix()) || token.starts_with("U:"), + CommentEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + CommentEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + CommentEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + CommentEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + CommentEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + CommentEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(CreateCommentError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool .acquire() .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs index b62fe2feeb..43f0fe6ac1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs @@ -12,9 +12,19 @@ use actix_web::http::StatusCode; use log::warn; use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; -use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs}; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; use tokens::tokens::user_bookmarks::UserBookmarkToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use tokens::tokens::zs_voices::ZsVoiceToken; + use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -70,6 +80,28 @@ pub async fn create_user_bookmark_handler( server_state: web::Data>, ) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + UserBookmarkEntityType::User => token.starts_with(UserToken::token_prefix()) || token.starts_with("U:"), + UserBookmarkEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + UserBookmarkEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + UserBookmarkEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + UserBookmarkEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + UserBookmarkEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + UserBookmarkEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + UserBookmarkEntityType::VoiceConversionModel => token.starts_with(VoiceConversionModelToken::token_prefix()), + UserBookmarkEntityType::ZsVoice => token.starts_with(ZsVoiceToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(CreateUserBookmarkError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool .acquire() .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs index 228b053afa..5c7f077910 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs @@ -9,6 +9,8 @@ use enums::by_table::user_ratings::rating_value::UserRatingValue; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use mysql_queries::composite_keys::by_table::user_ratings::user_rating_entity::UserRatingEntity; use mysql_queries::queries::users::user_ratings::get_user_rating::{Args, get_user_rating}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; @@ -105,9 +107,12 @@ pub async fn get_user_rating_handler( UserRatingEntityType::W2lTemplate => UserRatingEntity::W2lTemplate( W2lTemplateToken::new_from_str(&path.entity_token)), + UserRatingEntityType::MediaFile => UserRatingEntity::MediaFile(MediaFileToken::new_from_str(&path.entity_token)), + UserRatingEntityType::ModelWeight => UserRatingEntity::ModelWeight(ModelWeightToken::new_from_str(&path.entity_token)), + // TODO: We'll handle ratings of more types in the future. UserRatingEntityType::W2lResult | UserRatingEntityType::TtsResult => - return Err(GetUserRatingError::BadInput("type not yet supported".to_string())) + return Err(GetUserRatingError::BadInput("type not yet supported".to_string())), }; let maybe_rating = get_user_rating(Args { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs index 8d5af40699..e25a3d6e85 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use actix_web::http::StatusCode; -use log::{error, info}; +use log::{error, info, warn}; use enums::by_table::user_ratings::entity_type::UserRatingEntityType; use enums::by_table::user_ratings::rating_value::UserRatingValue; @@ -11,7 +11,11 @@ use http_server_common::response::serialize_as_json_error::serialize_as_json_err use mysql_queries::composite_keys::by_table::user_ratings::user_rating_entity::UserRatingEntity; use mysql_queries::queries::users::user_ratings::update_tts_model_ratings::update_tts_model_ratings; use mysql_queries::queries::users::user_ratings::upsert_user_rating::{Args, upsert_user_rating}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::w2l_results::W2lResultToken; use tokens::tokens::w2l_templates::W2lTemplateToken; use crate::server_state::ServerState; @@ -73,6 +77,27 @@ pub async fn set_user_rating_handler( request: web::Json, server_state: web::Data>) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + UserRatingEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + UserRatingEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + UserRatingEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + UserRatingEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + UserRatingEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + UserRatingEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + //UserRatingEntityType::VoiceConversionModel => token.starts_with(VoiceConversionModelToken::token_prefix()), + //UserRatingEntityType::ZsVoice => token.starts_with(ZsVoiceToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(SetUserRatingError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool.acquire() .await .map_err(|e| { @@ -100,6 +125,9 @@ pub async fn set_user_rating_handler( let ip_address = get_request_ip(&http_request); let entity= match request.entity_type { + UserRatingEntityType::MediaFile => UserRatingEntity::MediaFile(MediaFileToken::new_from_str(&request.entity_token)), + UserRatingEntityType::ModelWeight => UserRatingEntity::ModelWeight(ModelWeightToken::new_from_str(&request.entity_token)), + UserRatingEntityType::TtsModel => UserRatingEntity::TtsModel( TtsModelToken::new_from_str(&request.entity_token)), @@ -108,7 +136,7 @@ pub async fn set_user_rating_handler( // TODO: We'll handle ratings of more types in the future. UserRatingEntityType::W2lResult | UserRatingEntityType::TtsResult => - return Err(SetUserRatingError::BadInput("type not yet supported".to_string())) + return Err(SetUserRatingError::BadInput("type not yet supported".to_string())), }; let _r = upsert_user_rating(Args { From 8618e774e7bfa7af95f531eb1d737f94c0140646 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 13:37:25 -0500 Subject: [PATCH 132/136] codegen db --- ...663d0eb3d969365f6b2fd048ca41c5c455b2.json} | 23 +- ...44666e6c7c0f24b5da79dde828ffdb2c8302.json} | 23 +- .../media_files/list_media_files_by_tokens.rs | 264 ++++++++++++++++++ 3 files changed, 298 insertions(+), 12 deletions(-) rename .sqlx/{query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json => query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json} (78%) rename .sqlx/{query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json => query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json} (79%) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs diff --git a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json b/.sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json similarity index 78% rename from .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json rename to .sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json index 33334d1769..921e62dbe0 100644 --- a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json +++ b/.sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json @@ -1,6 +1,6 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n model_weights.title as maybe_descriptive_text_model_weight_title,\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { @@ -125,7 +125,7 @@ }, { "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", + "name": "maybe_descriptive_text_model_weight_title", "type_info": { "type": "VarString", "flags": "BINARY | NO_DEFAULT_VALUE", @@ -135,6 +135,16 @@ }, { "ordinal": 13, + "name": "maybe_descriptive_text_tts_model_title", + "type_info": { + "type": "VarString", + "flags": "BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, "name": "maybe_descriptive_text_tts_result_inference_text", "type_info": { "type": "Blob", @@ -144,7 +154,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "maybe_descriptive_text_user_display_name", "type_info": { "type": "VarString", @@ -154,7 +164,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "maybe_descriptive_text_voice_conversion_model_title", "type_info": { "type": "VarString", @@ -164,7 +174,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "maybe_descriptive_text_zs_voice_title", "type_info": { "type": "VarString", @@ -194,8 +204,9 @@ true, true, true, + true, true ] }, - "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" + "hash": "3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2" } diff --git a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json b/.sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json similarity index 79% rename from .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json rename to .sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json index 55d497beee..0b00758da8 100644 --- a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json +++ b/.sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json @@ -1,6 +1,6 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n model_weights.title as maybe_descriptive_text_model_weight_title,\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { @@ -125,7 +125,7 @@ }, { "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", + "name": "maybe_descriptive_text_model_weight_title", "type_info": { "type": "VarString", "flags": "BINARY | NO_DEFAULT_VALUE", @@ -135,6 +135,16 @@ }, { "ordinal": 13, + "name": "maybe_descriptive_text_tts_model_title", + "type_info": { + "type": "VarString", + "flags": "BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, "name": "maybe_descriptive_text_tts_result_inference_text", "type_info": { "type": "Blob", @@ -144,7 +154,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "maybe_descriptive_text_user_display_name", "type_info": { "type": "VarString", @@ -154,7 +164,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "maybe_descriptive_text_voice_conversion_model_title", "type_info": { "type": "VarString", @@ -164,7 +174,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "maybe_descriptive_text_zs_voice_title", "type_info": { "type": "VarString", @@ -194,8 +204,9 @@ true, true, true, + true, true ] }, - "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" + "hash": "707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302" } diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs new file mode 100644 index 0000000000..f08a9534a9 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -0,0 +1,264 @@ +use chrono::{DateTime, Utc}; +use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +#[derive(Serialize)] +pub struct WeightsByTokensRecord { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn list_weights_by_tokens( + mysql_pool: &MySqlPool, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let mut connection = mysql_pool.acquire().await?; + + let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + + Ok(map_to_weights(raw_weights)) +} + +async fn get_raw_weights_by_tokens( + connection: &mut MySqlConnection, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let connection = connection.acquire().await?; + + let mut query_builder : QueryBuilder = if can_see_deleted { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.token IN ( + "#, + ) + + } else { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.user_deleted_at IS NULL + AND mw.mod_deleted_at IS NULL + AND mw.token IN ( + "# + ) + }; + + query_builder.push(token_predicate(weight_tokens)); + + query_builder.push(")"); + + let query = query_builder.build_query_as::(); + + let results = query.fetch_all(connection).await?; + + Ok(results) +} + +/// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. +/// Issue: https://github.com/launchbadge/sqlx/issues/875 +fn token_predicate(tokens: &[ModelWeightToken]) -> String { + tokens.iter() + .map(|ty| ty.as_str()) + .map(|ty| format!("\"{}\"", ty)) + .collect::>() + .join(", ") +} + +fn map_to_weights(dataset:Vec) -> Vec { + let weights: Vec = dataset + .into_iter() + .map(|dataset: RawWeightJoinUser| { + WeightsByTokensRecord { + token: dataset.token, + title: dataset.title, + weights_type: dataset.weights_type, + weights_category: dataset.weights_category, + maybe_thumbnail_token: dataset.maybe_thumbnail_token, + + creator_user_token: dataset.creator_user_token, + creator_username:dataset.creator_username, + creator_display_name:dataset.creator_display_name, + creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + + private_bucket_hash: dataset.private_bucket_hash, + maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, + maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + + cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + + maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, + + created_at: dataset.created_at, + updated_at: dataset.updated_at, + user_deleted_at: dataset.user_deleted_at, + mod_deleted_at: dataset.mod_deleted_at, + } + }).collect(); + + weights +} + + #[derive(Serialize)] + pub struct RawWeightJoinUser { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for RawWeightJoinUser { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + token: ModelWeightToken::new(row.try_get("token")?), + weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, + weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, + title: row.try_get("title")?, + maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, + creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), + creator_username: row.try_get("creator_username")?, + creator_display_name: row.try_get("creator_display_name")?, + creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, + private_bucket_hash: row.try_get("private_bucket_hash")?, + maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, + maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, + cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, + cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, + cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, + maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, + cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + user_deleted_at: row.try_get("user_deleted_at")?, + mod_deleted_at: row.try_get("mod_deleted_at")?, + }) + } +} From 32b1786b59c8456ea382c511ab50b9be1d906d35 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 14:53:17 -0500 Subject: [PATCH 133/136] No hateful / antisemitic models on our service (hitler) --- _sql/actions/voice_conversion_models.sql | 9 +++++++++ crates/lib/mysql_queries/src/queries/media_files/mod.rs | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 1c040d57da..7f97f2da68 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -376,6 +376,15 @@ where token IN ( ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'abuse' +where token IN ( + 'vcm_qajyd528p8kw', + 'vcm_f5pcdecmzcc5' +); + diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index ff431d1cb4..cedc16583c 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -2,10 +2,11 @@ pub mod delete_media_file; pub mod get_media_file; pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_rerender; pub mod insert_media_file_from_tacotron2; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; pub mod list_media_files; +pub mod list_media_files_by_tokens; pub mod list_media_files_for_user; pub mod update_media_file; -pub mod insert_media_file_from_rerender; From 5a4d2a8c065e208f994749ddf421ff715e347b72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 15:15:31 -0500 Subject: [PATCH 134/136] media files query by tokens (WHERE IN clause) --- .../media_files/list_media_files_by_tokens.rs | 296 ++++++++---------- 1 file changed, 132 insertions(+), 164 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs index f08a9534a9..a6e06e49f8 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -2,142 +2,131 @@ use chrono::{DateTime, Utc}; use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; -use enums::by_table::model_weights::{ - weights_category::WeightsCategory, - weights_types::WeightsType, -}; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; -use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; #[derive(Serialize)] -pub struct WeightsByTokensRecord { - pub token: ModelWeightToken, +pub struct MediaFilesByTokensRecord { + pub token: MediaFileToken, - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, - pub title: String, + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, - pub maybe_thumbnail_token: Option, - - pub creator_user_token: UserToken, - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, - - pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, - pub maybe_private_bucket_extension: Option, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_email_gravatar_hash: Option, pub created_at: DateTime, pub updated_at: DateTime, - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, } -pub async fn list_weights_by_tokens( +pub async fn list_media_files_by_tokens( mysql_pool: &MySqlPool, - weight_tokens: &[ModelWeightToken], + media_file_tokens: &[MediaFileToken], can_see_deleted: bool -) -> AnyhowResult> { +) -> AnyhowResult> { let mut connection = mysql_pool.acquire().await?; - let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + let raw_media_files: Vec = get_raw_media_files_by_tokens(&mut connection, media_file_tokens, can_see_deleted).await?; - Ok(map_to_weights(raw_weights)) + Ok(map_to_media_files(raw_media_files)) } -async fn get_raw_weights_by_tokens( +async fn get_raw_media_files_by_tokens( connection: &mut MySqlConnection, - weight_tokens: &[ModelWeightToken], + media_file_tokens: &[MediaFileToken], can_see_deleted: bool -) -> AnyhowResult> { +) -> AnyhowResult> { let connection = connection.acquire().await?; let mut query_builder : QueryBuilder = if can_see_deleted { QueryBuilder::new(r#" SELECT - mw.token, - mw.title, - mw.weights_type, - mw.weights_category, - mw.maybe_thumbnail_token, - users.token as creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - users.email_gravatar_hash as creator_email_gravatar_hash, - mw.private_bucket_hash, - mw.maybe_private_bucket_prefix, - mw.maybe_private_bucket_extension, - mw.cached_user_ratings_negative_count, - mw.cached_user_ratings_positive_count, - mw.cached_user_ratings_total_count, - mw.maybe_cached_user_ratings_ratio, - mw.cached_user_ratings_last_updated_at, - mw.created_at, - mw.updated_at, - mw.user_deleted_at, - mw.mod_deleted_at - FROM model_weights as mw - JOIN users - ON users.token = mw.creator_user_token + m.token, + + m.origin_category, + m.origin_product_category, + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_email_gravatar_hash, + + m.media_type, + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + + FROM media_files as m + LEFT OUTER JOIN users + ON users.token = m.maybe_creator_user_token WHERE - mw.creator_set_visibility = "public" - AND mw.token IN ( + m.creator_set_visibility = "public" + AND m.token IN ( "#, ) } else { QueryBuilder::new(r#" SELECT - mw.token, - mw.title, - mw.weights_type, - mw.weights_category, - mw.maybe_thumbnail_token, - users.token as creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - users.email_gravatar_hash as creator_email_gravatar_hash, - mw.private_bucket_hash, - mw.maybe_private_bucket_prefix, - mw.maybe_private_bucket_extension, - mw.cached_user_ratings_negative_count, - mw.cached_user_ratings_positive_count, - mw.cached_user_ratings_total_count, - mw.maybe_cached_user_ratings_ratio, - mw.cached_user_ratings_last_updated_at, - mw.created_at, - mw.updated_at, - mw.user_deleted_at, - mw.mod_deleted_at - FROM model_weights as mw - JOIN users - ON users.token = mw.creator_user_token + m.token, + + m.origin_category, + m.origin_product_category, + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_email_gravatar_hash, + + m.media_type, + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + + FROM media_files as m + LEFT OUTER JOIN users + ON users.token = m.maybe_creator_user_token WHERE - mw.creator_set_visibility = "public" - AND mw.user_deleted_at IS NULL - AND mw.mod_deleted_at IS NULL - AND mw.token IN ( + m.creator_set_visibility = "public" + AND m.user_deleted_at IS NULL + AND m.mod_deleted_at IS NULL + AND m.token IN ( "# ) }; - query_builder.push(token_predicate(weight_tokens)); + query_builder.push(token_predicate(media_file_tokens)); query_builder.push(")"); - let query = query_builder.build_query_as::(); + let query = query_builder.build_query_as::(); let results = query.fetch_all(connection).await?; @@ -146,7 +135,7 @@ async fn get_raw_weights_by_tokens( /// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. /// Issue: https://github.com/launchbadge/sqlx/issues/875 -fn token_predicate(tokens: &[ModelWeightToken]) -> String { +fn token_predicate(tokens: &[MediaFileToken]) -> String { tokens.iter() .map(|ty| ty.as_str()) .map(|ty| format!("\"{}\"", ty)) @@ -154,73 +143,57 @@ fn token_predicate(tokens: &[ModelWeightToken]) -> String { .join(", ") } -fn map_to_weights(dataset:Vec) -> Vec { - let weights: Vec = dataset +fn map_to_media_files(dataset:Vec) -> Vec { + let media_files: Vec = dataset .into_iter() - .map(|dataset: RawWeightJoinUser| { - WeightsByTokensRecord { - token: dataset.token, - title: dataset.title, - weights_type: dataset.weights_type, - weights_category: dataset.weights_category, - maybe_thumbnail_token: dataset.maybe_thumbnail_token, + .map(|media_file: RawMediaFileJoinUser| { + MediaFilesByTokensRecord { + token: media_file.token, - creator_user_token: dataset.creator_user_token, - creator_username:dataset.creator_username, - creator_display_name:dataset.creator_display_name, - creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + origin_category: media_file.origin_category, + origin_product_category: media_file.origin_product_category, + maybe_origin_model_type: media_file.maybe_origin_model_type, + maybe_origin_model_token: media_file.maybe_origin_model_token, - private_bucket_hash: dataset.private_bucket_hash, - maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, - maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + maybe_creator_user_token: media_file.maybe_creator_user_token, + maybe_creator_username: media_file.maybe_creator_username, + maybe_creator_display_name: media_file.maybe_creator_display_name, + maybe_creator_email_gravatar_hash: media_file.maybe_creator_email_gravatar_hash, - cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, - cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, - cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + media_type: media_file.media_type, + public_bucket_directory_hash: media_file.public_bucket_directory_hash, + maybe_public_bucket_prefix: media_file.maybe_public_bucket_prefix, + maybe_public_bucket_extension: media_file.maybe_public_bucket_extension, - maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, - cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, - - created_at: dataset.created_at, - updated_at: dataset.updated_at, - user_deleted_at: dataset.user_deleted_at, - mod_deleted_at: dataset.mod_deleted_at, + created_at: media_file.created_at, + updated_at: media_file.updated_at, } }).collect(); - weights + media_files } #[derive(Serialize)] - pub struct RawWeightJoinUser { - pub token: ModelWeightToken, + pub struct RawMediaFileJoinUser { + pub token: MediaFileToken, - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, - - pub title: String, - - pub maybe_thumbnail_token: Option, + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, - pub creator_user_token: UserToken, - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_email_gravatar_hash: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, - pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, - pub maybe_private_bucket_extension: Option, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, - pub created_at: DateTime, pub updated_at: DateTime, - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, } // NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or @@ -235,30 +208,25 @@ fn map_to_weights(dataset:Vec) -> Vec // This results in the automatic mapping not being able to be found by name (for macro derive), and // in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the // full "as" clause). -impl FromRow<'_, MySqlRow> for RawWeightJoinUser { +impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { fn from_row(row: &MySqlRow) -> Result { Ok(Self { - token: ModelWeightToken::new(row.try_get("token")?), - weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, - weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, - title: row.try_get("title")?, - maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, - creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), - creator_username: row.try_get("creator_username")?, - creator_display_name: row.try_get("creator_display_name")?, - creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, - private_bucket_hash: row.try_get("private_bucket_hash")?, - maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, - maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, - cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, - cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, - cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, - maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, - cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + maybe_creator_user_token: row.try_get::, _>("maybe_creator_user_token")? + .and_then(|token| Some(UserToken::new_from_str(&token))), + maybe_creator_username: row.try_get("maybe_creator_username")?, + maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, + maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, - user_deleted_at: row.try_get("user_deleted_at")?, - mod_deleted_at: row.try_get("mod_deleted_at")?, }) } } From a88d888c18cb4f51e4d5e2bff9ee3b25b44281b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 16:35:07 -0500 Subject: [PATCH 135/136] featured media weights endpoint --- .../media_files/list_media_files_by_tokens.rs | 12 +- .../media_files/list_featured_media_files.rs | 170 ++++++++++++++++++ .../http_server/endpoints/media_files/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 5 + 4 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs index a6e06e49f8..d55cf8be87 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -15,12 +15,13 @@ use tokens::tokens::users::UserToken; pub struct MediaFilesByTokensRecord { pub token: MediaFileToken, + pub media_type: MediaFileType, + pub origin_category: MediaFileOriginCategory, pub origin_product_category: MediaFileOriginProductCategory, pub maybe_origin_model_type: Option, pub maybe_origin_model_token: Option, - pub media_type: MediaFileType, pub public_bucket_directory_hash: String, pub maybe_public_bucket_prefix: Option, pub maybe_public_bucket_extension: Option, @@ -59,6 +60,7 @@ async fn get_raw_media_files_by_tokens( QueryBuilder::new(r#" SELECT m.token, + m.media_type, m.origin_category, m.origin_product_category, @@ -70,7 +72,6 @@ async fn get_raw_media_files_by_tokens( users.display_name as maybe_creator_display_name, users.email_gravatar_hash as maybe_creator_email_gravatar_hash, - m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, @@ -91,6 +92,7 @@ async fn get_raw_media_files_by_tokens( QueryBuilder::new(r#" SELECT m.token, + m.media_type, m.origin_category, m.origin_product_category, @@ -102,7 +104,6 @@ async fn get_raw_media_files_by_tokens( users.display_name as maybe_creator_display_name, users.email_gravatar_hash as maybe_creator_email_gravatar_hash, - m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, @@ -177,6 +178,8 @@ fn map_to_media_files(dataset:Vec) -> Vec, @@ -187,7 +190,6 @@ fn map_to_media_files(dataset:Vec) -> Vec, pub maybe_creator_email_gravatar_hash: Option, - pub media_type: MediaFileType, pub public_bucket_directory_hash: String, pub maybe_public_bucket_prefix: Option, pub maybe_public_bucket_extension: Option, @@ -212,6 +214,7 @@ impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { fn from_row(row: &MySqlRow) -> Result { Ok(Self { token: MediaFileToken::new(row.try_get("token")?), + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, @@ -221,7 +224,6 @@ impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { maybe_creator_username: row.try_get("maybe_creator_username")?, maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?, - media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs new file mode 100644 index 0000000000..55cbfaef7b --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{DateTime, Utc}; +use log::{debug, error, warn}; +use r2d2_redis::redis::Commands; +use utoipa::ToSchema; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use mysql_queries::queries::media_files::list_media_files_by_tokens::list_media_files_by_tokens; +use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, ToSchema)] +pub struct ListFeaturedMediaFilesSuccessResponse { + pub success: bool, + pub media_files: Vec, +} + +#[derive(Serialize, ToSchema)] +pub struct MediaFile { + pub token: MediaFileToken, + + /// Type of media will dictate which fields are populated and what + /// the frontend should display (eg. video player vs audio player). + pub media_type: MediaFileType, + + /// URL to the media file + pub public_bucket_path: String, + + // TODO: Provenance data - hide some of the models we use, especially zero shot. + //pub origin_category: MediaFileOriginCategory, + //pub origin_product_category: MediaFileOriginProductCategory, + //pub maybe_origin_model_type: Option, + //pub maybe_origin_model_token: Option, + + pub maybe_creator: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// The key we store featured media file tokens under +const REDIS_KEY : &str = "featured_media_files_list"; + +#[derive(Debug, ToSchema)] +pub enum ListFeaturedMediaFilesError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListFeaturedMediaFilesError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListFeaturedMediaFilesError { + fn status_code(&self) -> StatusCode { + match *self { + ListFeaturedMediaFilesError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListFeaturedMediaFilesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/media_files/list_featured", + responses( + (status = 200, description = "List Featured Media Files", body = ListFeaturedMediaFilesSuccessResponse), + (status = 401, description = "Not authorized", body = ListFeaturedMediaFilesError), + (status = 500, description = "Server error", body = ListFeaturedMediaFilesError), + ), +)] +pub async fn list_featured_media_files_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let mut redis = server_state.redis_pool.get() + .map_err(|err| { + error!("Could not obtain redis: {err}"); + ListFeaturedMediaFilesError::ServerError + })?; + + let token_list : Option = redis.get(REDIS_KEY) + .map_err(|err| { + error!("Could not get redis result: {err}"); + ListFeaturedMediaFilesError::ServerError + })?; + + let media_file_tokens = token_list + .unwrap_or_else(|| "".to_string()) + .split(",") + .into_iter() + .map(|item| item.trim()) + .filter(|item| !item.is_empty()) + .map(|item| MediaFileToken::new_from_str(item)) + .collect::>(); + + debug!("Weight tokens from Redis: {:?}", media_file_tokens); + + let mut media_files = Vec::new(); + + if !media_file_tokens.is_empty() { + let query_results = + list_media_files_by_tokens(&server_state.mysql_pool, &media_file_tokens, false).await; + + media_files = match query_results { + Ok(media_files) => media_files, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListFeaturedMediaFilesError::ServerError); + } + }; + } + + let response = ListFeaturedMediaFilesSuccessResponse { + success: true, + media_files: media_files.into_iter() + .map(|m| { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix.as_deref(), + m.maybe_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string(); + + MediaFile { + token: m.token, + media_type: m.media_type, + public_bucket_path, + //origin_category: m.origin_category, + //origin_product_category: m.origin_product_category, + //maybe_origin_model_type: m.maybe_origin_model_type, + //maybe_origin_model_token: m.maybe_origin_model_token, + maybe_creator: UserDetailsLight::from_optional_db_fields_owned( + m.maybe_creator_user_token, + m.maybe_creator_username, + m.maybe_creator_display_name, + m.maybe_creator_email_gravatar_hash + ), + created_at: m.created_at, + updated_at: m.updated_at, + } + }).collect::>(), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListFeaturedMediaFilesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index ab3552ac20..2893bf89ce 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,5 +1,6 @@ pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; +pub mod list_featured_media_files; pub mod list_media_files; pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 5585ee1370..7e9569dad2 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -45,6 +45,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_featured_media_files::list_featured_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; @@ -1024,6 +1025,10 @@ fn add_media_file_routes (app: App) -> App .route(web::get().to(list_media_files_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/list_featured") + .route(web::get().to(list_featured_media_files_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/list/user/{username}") .route(web::get().to(list_media_files_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 60d419921617164e499e8efa1402216ecdb4e339 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 16:58:37 -0500 Subject: [PATCH 136/136] fix logout The reason this is so labyrinthine is because of the users and billing components. They bind dynamic behaviors that are defined in storyteller-web (one of them being clearing the redis session cache). Actually, the users component _didn't_, because it was defined in the billing component. So I hoisted it out into a new component library. In the fullness of time I imagine all of this will be consolidated back under storyteller-web so we don't have to worry about binding actions at a distance. Anyway, this is tested and works locally. --- Cargo.lock | 13 +++++++++ Cargo.toml | 1 + crates/service/plugins/billing/Cargo.toml | 1 + ...pe_create_checkout_session_json_handler.rs | 2 +- ...reate_checkout_session_redirect_handler.rs | 2 +- ...te_customer_portal_session_json_handler.rs | 2 +- ...ustomer_portal_session_redirect_handler.rs | 2 +- .../plugins/billing/src/stripe/traits/mod.rs | 1 - .../plugins/user_traits_component/Cargo.toml | 27 +++++++++++++++++++ .../plugins/user_traits_component/src/lib.rs | 21 +++++++++++++++ .../traits/internal_session_cache_purge.rs | 0 .../user_traits_component/src/traits/mod.rs | 1 + crates/service/plugins/users/Cargo.toml | 1 + .../users/src/endpoints/logout_handler.rs | 5 ++++ crates/service/web/storyteller_web/Cargo.toml | 1 + .../internal_session_cache_purge_impl.rs | 2 +- .../service/web/storyteller_web/src/main.rs | 16 +++++------ 17 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 crates/service/plugins/user_traits_component/Cargo.toml create mode 100644 crates/service/plugins/user_traits_component/src/lib.rs rename crates/service/plugins/{billing/src/stripe => user_traits_component/src}/traits/internal_session_cache_purge.rs (100%) create mode 100644 crates/service/plugins/user_traits_component/src/traits/mod.rs diff --git a/Cargo.lock b/Cargo.lock index af9894f874..8d6d76535f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -813,6 +813,7 @@ dependencies = [ "tokio", "toml 0.5.11", "url_config", + "user_traits_component", ] [[package]] @@ -5954,6 +5955,7 @@ dependencies = [ "twitch_oauth2", "url_config", "user_input_common", + "user_traits_component", "users_component", "utoipa", "utoipa-swagger-ui", @@ -7066,6 +7068,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "user_traits_component" +version = "0.0.1" +dependencies = [ + "actix-web", + "errors", + "mockall", + "my-workspace-hack", +] + [[package]] name = "users_component" version = "0.0.1" @@ -7109,6 +7121,7 @@ dependencies = [ "tokens", "toml 0.5.11", "user_input_common", + "user_traits_component", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1ea5d8fefc..411f131e53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ members = [ # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.) "crates/service/plugins/billing", "crates/service/plugins/inference_workloads", + "crates/service/plugins/user_traits_component", "crates/service/plugins/users", # Vendored code diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 869eee7790..56a8176786 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -20,6 +20,7 @@ http_server_common = { path = "../../../lib/deprecated/http_server_common" } mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } url_config = { path = "../../../lib/url_config" } +user_traits_component = { path = "../user_traits_component" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs index bdc4de4af8..c81a63b39d 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs @@ -3,12 +3,12 @@ use actix_web::web::Json; use reusable_types::server_environment::ServerEnvironment; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs index 3f6ccc221d..657761495a 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs @@ -4,12 +4,12 @@ use actix_web::web::Query; use reusable_types::server_environment::ServerEnvironment; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs index 3e41b3b2b5..bb21b81621 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs @@ -2,12 +2,12 @@ use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::web::Json; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs index c4e69a3288..4aafd6e37c 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs @@ -3,12 +3,12 @@ use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::web::Query; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/traits/mod.rs b/crates/service/plugins/billing/src/stripe/traits/mod.rs index 2cf3b026fa..44d38b0eac 100644 --- a/crates/service/plugins/billing/src/stripe/traits/mod.rs +++ b/crates/service/plugins/billing/src/stripe/traits/mod.rs @@ -1,4 +1,3 @@ pub mod internal_product_to_stripe_lookup; -pub mod internal_session_cache_purge; pub mod internal_subscription_product_lookup; pub mod internal_user_lookup; \ No newline at end of file diff --git a/crates/service/plugins/user_traits_component/Cargo.toml b/crates/service/plugins/user_traits_component/Cargo.toml new file mode 100644 index 0000000000..6521233d9d --- /dev/null +++ b/crates/service/plugins/user_traits_component/Cargo.toml @@ -0,0 +1,27 @@ +[package] +publish = false +name = "user_traits_component" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[lib] +name = "user_traits_component" +path = "src/lib.rs" + +[dependencies] + +# Internal +errors = { path = "../../../lib/errors" } + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + +# Web +actix-web = "4.4.0" + +[dev-dependencies] +mockall = "0.11.3" diff --git a/crates/service/plugins/user_traits_component/src/lib.rs b/crates/service/plugins/user_traits_component/src/lib.rs new file mode 100644 index 0000000000..a490ebd661 --- /dev/null +++ b/crates/service/plugins/user_traits_component/src/lib.rs @@ -0,0 +1,21 @@ +//! user_traits_component +//! +//! Expose some traits for other plugins / components. +//! This will probably go away once we start consolidating the plugins into the main service. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod traits; diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/user_traits_component/src/traits/internal_session_cache_purge.rs similarity index 100% rename from crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs rename to crates/service/plugins/user_traits_component/src/traits/internal_session_cache_purge.rs diff --git a/crates/service/plugins/user_traits_component/src/traits/mod.rs b/crates/service/plugins/user_traits_component/src/traits/mod.rs new file mode 100644 index 0000000000..01df49e0ee --- /dev/null +++ b/crates/service/plugins/user_traits_component/src/traits/mod.rs @@ -0,0 +1 @@ +pub mod internal_session_cache_purge; diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 86db221476..7d2a8f616e 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -30,6 +30,7 @@ reusable_types = { path = "../../../_types/reusable_types" } server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } user_input_common = { path = "../../../lib/user_input_common" } +user_traits_component = { path = "../user_traits_component" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/src/endpoints/logout_handler.rs b/crates/service/plugins/users/src/endpoints/logout_handler.rs index 806b525d48..4c51375357 100644 --- a/crates/service/plugins/users/src/endpoints/logout_handler.rs +++ b/crates/service/plugins/users/src/endpoints/logout_handler.rs @@ -13,6 +13,7 @@ use sqlx::MySqlPool; use http_server_common::response::response_error_helpers::to_simple_json_error; use mysql_queries::queries::users::user_sessions::delete_user_session::delete_user_session; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::cookies::session::session_cookie_manager::SessionCookieManager; @@ -53,8 +54,12 @@ pub async fn logout_handler( http_request: HttpRequest, session_cookie_manager: web::Data, mysql_pool: web::Data, + internal_session_cache_purge: web::Data, ) -> Result { + // Best effort to delete Redis session cache + internal_session_cache_purge.best_effort_purge_session_cache(&http_request); + let mut delete_cookie = match http_request.cookie("session") { Some(cookie) => { match session_cookie_manager.decode_session_token(&cookie) { diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index bc0e4b61bc..6767b33240 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -52,6 +52,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # Internal Components billing_component = { path = "../../plugins/billing" } +user_traits_component = { path = "../../plugins/user_traits_component" } users_component = { path = "../../plugins/users" } # Workspace hack (faster compile times) diff --git a/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs b/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs index c296ed5296..2dcf6eb523 100644 --- a/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs +++ b/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs @@ -1,8 +1,8 @@ use actix_web::HttpRequest; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use redis_caching::redis_ttl_cache::RedisTtlCache; use redis_common::redis_cache_keys::RedisCacheKeys; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use users_component::utils::session_checker::SessionChecker; pub struct InternalSessionCachePurgeImpl { diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 57f09a82c8..8b815cd6f3 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -20,15 +20,13 @@ use std::sync::Arc; use std::time::Duration; -use actix_cors::Cors; -use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::{App, HttpServer, middleware, web}; use actix_web::middleware::{DefaultHeaders, Logger}; use anyhow::anyhow; use elasticsearch::Elasticsearch; use elasticsearch::http::transport::Transport; use futures::Future; -use limitation::Limiter; -use log::{error, info}; +use log::info; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; @@ -48,7 +46,6 @@ use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exa use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; @@ -56,10 +53,9 @@ use cloud_storage::bucket_client::BucketClient; use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; use email_sender::smtp_email_sender::SmtpEmailSender; use errors::AnyhowResult; -use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use http_server_common::cors::build_cors_config; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; @@ -67,6 +63,7 @@ use redis_caching::redis_ttl_cache::RedisTtlCache; use reusable_types::server_environment::ServerEnvironment; use twitch_common::twitch_secrets::TwitchSecrets; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; @@ -76,12 +73,11 @@ use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurge use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; -use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::configs::static_api_tokens::StaticApiTokenSet; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; -use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; use crate::routes::add_routes; -use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::poll_ip_banlist_thread::poll_ip_bans;