From 5ed397af4e0ee5d9306c659db86473fc8653a341 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 22 Nov 2023 23:05:25 -0500 Subject: [PATCH] WIP get and update gotta write unit tests, WIP do not review. --- .../model_weights/weights_category.rs | 2 +- .../by_table/model_weights/weights_types.rs | 2 +- .../src/queries/model_weights/get_weight.rs | 237 +++++++++++++++++- .../model_weights/list_weights_by_user.rs | 74 ++++-- .../src/queries/model_weights/mod.rs | 2 + .../queries/model_weights/update_weight.rs | 41 +++ .../voice_designer/voices/update_voice.rs | 13 +- 7 files changed, 329 insertions(+), 42 deletions(-) diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index 7792f63fce..3e61c830a6 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -6,7 +6,7 @@ use strum::EnumCount; use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,Debug)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd,sqlx::Type, Deserialize, Serialize,Debug)] pub enum WeightsCategory { #[serde(rename = "image_generation")] ImageGeneration, diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 7f09c5a539..e50b65d560 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -6,7 +6,7 @@ use strum::EnumCount; use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,Debug)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, sqlx::Type, Deserialize, Serialize,Debug)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs index 7a77bb8d6c..b7c15176f6 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs @@ -1,20 +1,235 @@ use container_common::anyhow_result::AnyhowResult; -use sqlx::MySqlPool; -use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory }; -use log::warn; + +use sqlx::{ MySqlPool, MySql }; +use sqlx::pool::PoolConnection; +use anyhow::anyhow; +use log::{ info, warn, error }; +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; use enums::common::visibility::Visibility; use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; +use chrono::{ DateTime, Utc }; +pub struct RetrivedModelWeight { + pub token: ModelWeightToken, + pub title: String, + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + pub maybe_thumbnail_token: String, + pub description_markdown: String, + pub description_rendered_html: String, + pub creator_user_token: UserToken, + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + pub maybe_last_update_user_token: Option, + pub original_filename: Option, + pub original_download_url: Option, + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + pub cached_user_ratings_negative_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_total_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + pub version: i32, + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} pub async fn get_weight_by_token( weight_token: &ModelWeightToken, can_see_deleted: bool, - mysql_pool: &MySqlPool, -) -> AnyhowResult> { + mysql_pool: &MySqlPool +) -> AnyhowResult> { let mut connection = mysql_pool.acquire().await?; - get_voice_by_token_with_connection( - weight_token, - can_see_deleted, - &mut connection - ).await -} \ No newline at end of file + get_weights_by_token_with_connection(weight_token, can_see_deleted, &mut connection).await +} + +pub async fn get_weights_by_token_with_connection( + voice_token: &ModelWeightToken, + can_see_deleted: bool, + mysql_connection: &mut PoolConnection +) -> AnyhowResult> { + let maybe_result = if can_see_deleted { + select_include_deleted(voice_token, mysql_connection).await + } else { + select_without_deleted(voice_token, mysql_connection).await + }; + + let record: RawWeight = match maybe_result { + Ok(record) => record, + Err(sqlx::Error::RowNotFound) => { + return Ok(None); + } + Err(err) => { + error!("Error fetching weights by token: {:?}", err); + return Err(anyhow!("Error fetching weights by token: {:?}", err)); + } + }; + + Ok( + Some(RetrivedModelWeight { + token: record.token, + title: record.title, + weights_type: record.weights_type, + weights_category: record.weights_category, + maybe_thumbnail_token: record.maybe_thumbnail_token, + description_markdown: record.description_markdown, + description_rendered_html: record.description_rendered_html, + creator_user_token: record.creator_user_token, + creator_ip_address: record.creator_ip_address, + creator_set_visibility: record.creator_set_visibility, + maybe_last_update_user_token: record.maybe_last_update_user_token, + original_download_url: record.original_download_url, + original_filename: record.original_filename, + file_size_bytes: record.file_size_bytes, + file_checksum_sha2: record.file_checksum_sha2, + private_bucket_hash: record.private_bucket_hash, + maybe_private_bucket_prefix: record.maybe_private_bucket_prefix, + maybe_private_bucket_extension: record.maybe_private_bucket_extension, + cached_user_ratings_negative_count: record.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: record.cached_user_ratings_positive_count, + cached_user_ratings_total_count: record.cached_user_ratings_total_count, + maybe_cached_user_ratings_ratio: record.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: record.cached_user_ratings_last_updated_at, + version: record.version, + created_at: record.created_at, + updated_at: record.updated_at, + user_deleted_at: record.user_deleted_at, + mod_deleted_at: record.mod_deleted_at, + }) + ) +} + +async fn select_include_deleted( + weight_token: &ModelWeightToken, + mysql_connection: &mut PoolConnection +) -> Result { + sqlx + ::query_as!( + RawWeight, + r#" + SELECT + wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`, + wt.title, + wt.weights_type as 'weights_type: enums::by_table::model_weights::weights_types::WeightsType', + wt.weights_category as 'weights_category: enums::by_table::model_weights::weights_category::WeightsCategory', + wt.maybe_thumbnail_token, + wt.description_markdown, + wt.description_rendered_html, + wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`, + wt.creator_ip_address, + wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + wt.maybe_last_update_user_token as `creator_user_token: tokens::tokens::users::UserToken', + wt.original_download_url, + wt.original_filename, + wt.file_size_bytes, + wt.file_checksum_sha2, + wt.private_bucket_hash, + wt.maybe_private_bucket_prefix, + wt.maybe_private_bucket_extension, + wt.cached_user_ratings_negative_count, + wt.cached_user_ratings_positive_count, + wt.cached_user_ratings_total_count, + wt.maybe_cached_user_ratings_ratio, + wt.cached_user_ratings_last_updated_at, + wt.version, + wt.created_at, + wt.updated_at, + wt.user_deleted_at, + wt.mod_deleted_at + FROM model_weights as wt + WHERE + wt.token = ? + "#, + weight_token.as_str() + ) + .fetch_one(mysql_connection).await +} + +async fn select_without_deleted( + weight_token: &ModelWeightToken, + mysql_connection: &mut PoolConnection +) -> Result { + sqlx + ::query_as!( + RawWeight, + r#" + SELECT + wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`, + wt.title, + wt.weights_type as 'weights_type: enums::by_table::model_weights::weights_types::WeightsType', + wt.weights_category as 'weights_category: enums::by_table::model_weights::weights_category::WeightsCategory', + wt.maybe_thumbnail_token, + wt.description_markdown, + wt.description_rendered_html, + wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`, + wt.creator_ip_address, + wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + wt.maybe_last_update_user_token as 'creator_user_token: tokens::tokens::users::UserToken', + wt.original_download_url, + wt.original_filename, + wt.file_size_bytes, + wt.file_checksum_sha2, + wt.private_bucket_hash, + wt.maybe_private_bucket_prefix, + wt.maybe_private_bucket_extension, + wt.cached_user_ratings_negative_count, + wt.cached_user_ratings_positive_count, + wt.cached_user_ratings_total_count, + wt.maybe_cached_user_ratings_ratio, + wt.cached_user_ratings_last_updated_at, + wt.version, + wt.created_at, + wt.updated_at, + wt.user_deleted_at, + wt.mod_deleted_at + FROM model_weights as wt + WHERE + wt.token = ? + AND wt.user_deleted_at IS NULL + AND wt.mod_deleted_at IS NULL + "#, + weight_token.as_str() + ) + .fetch_one(mysql_connection).await +} + +#[derive(Serialize)] +pub struct RawWeight { + pub token: ModelWeightToken, + pub title: String, + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + pub maybe_thumbnail_token: String, + pub description_markdown: String, + pub description_rendered_html: String, + pub creator_user_token: UserToken, + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + pub maybe_last_update_user_token: Option, + pub original_download_url: Option, + pub original_filename: Option, + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + pub cached_user_ratings_negative_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_total_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + pub version: i32, + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs index 7726596de2..976fa1feb2 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs @@ -8,36 +8,47 @@ use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::model_weights::ModelWeightToken; -// FIXME: This is the old style of query scoping and shouldn't be copied. +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; #[derive(Serialize)] pub struct WeightsRecord { - pub weights_token: ZsVoiceDatasetToken, + pub weights_token: ModelWeightToken, pub title: String, pub creator_user_token: UserToken, - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, - pub creator_set_visibility: Visibility, + pub creator_username: String, + pub creator_ip_address: String, - pub ietf_language_tag: String, - pub ietf_primary_language_subtag: String, + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub cached_user_ratings_negative_count: i32, + pub cached_user_ratings_positive_count: i32, + pub cached_user_ratings_total_count: i32, + + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_total_count: i32, pub created_at: DateTime, pub updated_at: DateTime, } -pub async fn list_datasets_by_username( +pub async fn list_weights_by_username( mysql_pool: &MySqlPool, username: &str, can_see_deleted: bool, ) -> AnyhowResult> { let mut connection = mysql_pool.acquire().await?; - list_datasets_by_username_with_connection(&mut connection, username, can_see_deleted).await + list_weights_by_username_with_connection(&mut connection, username, can_see_deleted).await } -pub async fn list_datasets_by_username_with_connection( +pub async fn list_weights_by_username_with_connection( mysql_connection: &mut PoolConnection, creator_username: &str, can_see_deleted: bool, @@ -47,7 +58,7 @@ pub async fn list_datasets_by_username_with_connection( list_datasets_by_creator_username(mysql_connection, creator_username, can_see_deleted) .await; - let datasets : Vec = match datasets { + let datasets : Vec = match datasets { Ok(datasets) => datasets, Err(err) => { match err { @@ -55,8 +66,8 @@ pub async fn list_datasets_by_username_with_connection( return Ok(Vec::new()); }, _ => { - warn!("dataset list query error: {:?}", err); - return Err(anyhow!("dataset list query error")); + warn!("weights dataset list query error: {:?}", err); + return Err(anyhow!("weights dataset list query error")); } } } @@ -65,21 +76,26 @@ pub async fn list_datasets_by_username_with_connection( Ok(datasets.into_iter() .map(|dataset| { WeightsRecord { - dataset_token: dataset.token, + weights_token: ModelWeightToken(dataset.token), title: dataset.title, - ietf_language_tag: dataset.ietf_language_tag, - ietf_primary_language_subtag: dataset.ietf_primary_language_subtag, creator_user_token: dataset.creator_user_token, creator_username: dataset.creator_username, - creator_display_name: dataset.creator_display_name, - creator_email_gravatar_hash: dataset.creator_email_gravatar_hash, - creator_set_visibility: dataset.creator_set_visibility, + creator_ip_address: dataset.creator_ip_address, + weights_type: dataset.weights_type, + weights_category: dataset.weights_category, + description_markdown: dataset.description_markdown, + description_rendered_html: dataset.description_rendered_html, + cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, created_at: dataset.created_at, updated_at: dataset.updated_at, } }) .filter(|dataset| { - dataset.creator_username == creator_username || dataset.creator_set_visibility == Visibility::Public || can_see_deleted + dataset.weights_token == creator_username || dataset.creator_set_visibility == Visibility::Public || can_see_deleted }) .collect::>()) } @@ -89,13 +105,13 @@ async fn list_datasets_by_creator_username( mysql_connection: &mut PoolConnection, creator_username: &str, can_see_deleted: bool, -) -> AnyhowResult> { +) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. let maybe_datasets = if !can_see_deleted { info!("listing datasets for user;"); sqlx::query_as!( - InternalRawWeightsRecordForList, + WeightsRecord, r#" SELECT zd.token as `token: tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken`, @@ -123,7 +139,7 @@ async fn list_datasets_by_creator_username( } else { info!("listing datasets for user"); sqlx::query_as!( - InternalRawWeightsRecordForList + WeightsRecord , r#" SELECT @@ -151,3 +167,13 @@ async fn list_datasets_by_creator_username( Ok(maybe_datasets) } + +#[cfg(test)] +mod tests { + + #[test] + fn test() { + assert_eq!("one", "two"); // This fails + } + +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs index d55c3bf35a..bd32b6e8bf 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs @@ -1 +1,3 @@ pub mod create_weight; +//pub mod delete_weight; +pub mod get_weight; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index ab36d88f11..05b728979b 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -1,10 +1,15 @@ use anyhow::anyhow; use sqlx::MySqlPool; +use log::info; +use sqlx::mysql::MySqlPoolOptions; + use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::model_weights::ModelWeightToken; +use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; + pub struct UpdateWeightArgs<'a> { pub weight_token: &ModelWeightToken, pub title: Option<&'a str>, @@ -48,3 +53,39 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::MySqlPoolOptions; + use easyenv; + use tokio; + + #[tokio::test] + async fn test_update_weights() -> AnyhowResult<()> { + let db_connection_string = easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_CONNECTION_STRING + ); + + let pool = MySqlPoolOptions::new() + .max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?) + .connect(&db_connection_string) + .await?; + + let args = UpdateWeightArgs { + weight_token: &ModelWeightToken::new("test_token"), + title: Some("Test Title"), + description_markdown: "Test Description Markdown", + description_rendered_html: "Test Description HTML", + creator_set_visibility: &Visibility::Public, + mysql_pool: &pool, + }; + + let result = update_weights(args).await; + + assert!(result.is_ok()); + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voices/update_voice.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voices/update_voice.rs index d4b1385bdb..738a916761 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voices/update_voice.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voices/update_voice.rs @@ -3,10 +3,10 @@ use sqlx::MySqlPool; use enums::common::visibility::Visibility; use errors::AnyhowResult; -use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::zs_voices::ZsVoiceToken; pub struct UpdateVoiceArgs<'a> { - pub voice_token: &'a ModelWeightToken, + pub voice_token: &'a ZsVoiceToken, pub voice_title: Option<&'a str>, pub creator_set_visibility: &'a Visibility, pub maybe_mod_user_token: Option<&'a str>, @@ -18,7 +18,8 @@ pub struct UpdateVoiceArgs<'a> { pub async fn update_voice(args: UpdateVoiceArgs<'_>) -> AnyhowResult<()>{ let mut transaction = args.mysql_pool.begin().await?; - let query_result = sqlx::query!( + let query_result = sqlx + ::query!( r#" UPDATE zs_voices SET @@ -39,11 +40,13 @@ pub async fn update_voice(args: UpdateVoiceArgs<'_>) -> AnyhowResult<()>{ args.voice_token.as_str(), ).execute(args.mysql_pool).await; - // TODO(Kasisnu): This should probably rollback transaction.commit().await?; match query_result { Ok(_) => Ok(()), - Err(err) => Err(anyhow!("zs voice update error: {:?}", err)), + Err(err) => + { + Err(anyhow!("zs voice update error: {:?}", err)) + }, } }