WIP get and update gotta write unit tests, WIP do not review.

This commit is contained in:
Michael Chung committed 2023-11-22 23:05:25 -05:00
1 parent ccbc89fd3e
commit 5ed397af4e
7 files changed
+329 -42

No files matched your search

@@ -6,7 +6,7 @@ use strum::EnumCount;
use strum::EnumIter;
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,Debug)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd,sqlx::Type, Deserialize, Serialize,Debug)]
pub enum WeightsCategory {
#[serde(rename = "image_generation")]
ImageGeneration,
@@ -6,7 +6,7 @@ use strum::EnumCount;
use strum::EnumIter;
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,Debug)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, sqlx::Type, Deserialize, Serialize,Debug)]
pub enum WeightsType {
#[serde(rename = "hifigan_tt2")]
HifiganTacotron2,
@@ -1,20 +1,235 @@
use container_common::anyhow_result::AnyhowResult;
use sqlx::MySqlPool;
use enums::by_table::model_weights::{ weights_types::WeightsType, weights_category::WeightsCategory };
use log::warn;
use sqlx::{ MySqlPool, MySql };
use sqlx::pool::PoolConnection;
use anyhow::anyhow;
use log::{ info, warn, error };
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
use enums::common::visibility::Visibility;
use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken };
use chrono::{ DateTime, Utc };
pub struct RetrivedModelWeight {
pub token: ModelWeightToken,
pub title: String,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub maybe_thumbnail_token: String,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_filename: Option<String>,
pub original_download_url: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
}
pub async fn get_weight_by_token(
weight_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_pool: &MySqlPool,
) -> AnyhowResult<Option<ZsVoice>> {
mysql_pool: &MySqlPool
) -> AnyhowResult<Option<RetrivedModelWeight>> {
let mut connection = mysql_pool.acquire().await?;
get_voice_by_token_with_connection(
weight_token,
can_see_deleted,
&mut connection
).await
}
get_weights_by_token_with_connection(weight_token, can_see_deleted, &mut connection).await
}
pub async fn get_weights_by_token_with_connection(
voice_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_connection: &mut PoolConnection<MySql>
) -> AnyhowResult<Option<RetrivedModelWeight>> {
let maybe_result = if can_see_deleted {
select_include_deleted(voice_token, mysql_connection).await
} else {
select_without_deleted(voice_token, mysql_connection).await
};
let record: RawWeight = match maybe_result {
Ok(record) => record,
Err(sqlx::Error::RowNotFound) => {
return Ok(None);
}
Err(err) => {
error!("Error fetching weights by token: {:?}", err);
return Err(anyhow!("Error fetching weights by token: {:?}", err));
}
};
Ok(
Some(RetrivedModelWeight {
token: record.token,
title: record.title,
weights_type: record.weights_type,
weights_category: record.weights_category,
maybe_thumbnail_token: record.maybe_thumbnail_token,
description_markdown: record.description_markdown,
description_rendered_html: record.description_rendered_html,
creator_user_token: record.creator_user_token,
creator_ip_address: record.creator_ip_address,
creator_set_visibility: record.creator_set_visibility,
maybe_last_update_user_token: record.maybe_last_update_user_token,
original_download_url: record.original_download_url,
original_filename: record.original_filename,
file_size_bytes: record.file_size_bytes,
file_checksum_sha2: record.file_checksum_sha2,
private_bucket_hash: record.private_bucket_hash,
maybe_private_bucket_prefix: record.maybe_private_bucket_prefix,
maybe_private_bucket_extension: record.maybe_private_bucket_extension,
cached_user_ratings_negative_count: record.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: record.cached_user_ratings_positive_count,
cached_user_ratings_total_count: record.cached_user_ratings_total_count,
maybe_cached_user_ratings_ratio: record.maybe_cached_user_ratings_ratio,
cached_user_ratings_last_updated_at: record.cached_user_ratings_last_updated_at,
version: record.version,
created_at: record.created_at,
updated_at: record.updated_at,
user_deleted_at: record.user_deleted_at,
mod_deleted_at: record.mod_deleted_at,
})
)
}
async fn select_include_deleted(
weight_token: &ModelWeightToken,
mysql_connection: &mut PoolConnection<MySql>
) -> Result<RawWeight, sqlx::Error> {
sqlx
::query_as!(
RawWeight,
r#"
SELECT
wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
wt.title,
wt.weights_type as 'weights_type: enums::by_table::model_weights::weights_types::WeightsType',
wt.weights_category as 'weights_category: enums::by_table::model_weights::weights_category::WeightsCategory',
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.maybe_last_update_user_token as `creator_user_token: tokens::tokens::users::UserToken',
wt.original_download_url,
wt.original_filename,
wt.file_size_bytes,
wt.file_checksum_sha2,
wt.private_bucket_hash,
wt.maybe_private_bucket_prefix,
wt.maybe_private_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
wt.maybe_cached_user_ratings_ratio,
wt.cached_user_ratings_last_updated_at,
wt.version,
wt.created_at,
wt.updated_at,
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
WHERE
wt.token = ?
"#,
weight_token.as_str()
)
.fetch_one(mysql_connection).await
}
async fn select_without_deleted(
weight_token: &ModelWeightToken,
mysql_connection: &mut PoolConnection<MySql>
) -> Result<RawWeight, sqlx::Error> {
sqlx
::query_as!(
RawWeight,
r#"
SELECT
wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
wt.title,
wt.weights_type as 'weights_type: enums::by_table::model_weights::weights_types::WeightsType',
wt.weights_category as 'weights_category: enums::by_table::model_weights::weights_category::WeightsCategory',
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.maybe_last_update_user_token as 'creator_user_token: tokens::tokens::users::UserToken',
wt.original_download_url,
wt.original_filename,
wt.file_size_bytes,
wt.file_checksum_sha2,
wt.private_bucket_hash,
wt.maybe_private_bucket_prefix,
wt.maybe_private_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
wt.maybe_cached_user_ratings_ratio,
wt.cached_user_ratings_last_updated_at,
wt.version,
wt.created_at,
wt.updated_at,
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
WHERE
wt.token = ?
AND wt.user_deleted_at IS NULL
AND wt.mod_deleted_at IS NULL
"#,
weight_token.as_str()
)
.fetch_one(mysql_connection).await
}
#[derive(Serialize)]
pub struct RawWeight {
pub token: ModelWeightToken,
pub title: String,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub maybe_thumbnail_token: String,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
}
@@ -8,36 +8,47 @@ use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
// FIXME: This is the old style of query scoping and shouldn't be copied.
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
#[derive(Serialize)]
pub struct WeightsRecord {
pub weights_token: ZsVoiceDatasetToken,
pub weights_token: ModelWeightToken,
pub title: String,
pub creator_user_token: UserToken,
pub creator_username: String,
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
pub creator_set_visibility: Visibility,
pub creator_username: String,
pub creator_ip_address: String,
pub ietf_language_tag: String,
pub ietf_primary_language_subtag: String,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub description_markdown: String,
pub description_rendered_html: String,
pub cached_user_ratings_negative_count: i32,
pub cached_user_ratings_positive_count: i32,
pub cached_user_ratings_total_count: i32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_total_count: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
pub async fn list_datasets_by_username(
pub async fn list_weights_by_username(
mysql_pool: &MySqlPool,
username: &str,
can_see_deleted: bool,
) -> AnyhowResult<Vec<WeightsRecord>> {
let mut connection = mysql_pool.acquire().await?;
list_datasets_by_username_with_connection(&mut connection, username, can_see_deleted).await
list_weights_by_username_with_connection(&mut connection, username, can_see_deleted).await
}
pub async fn list_datasets_by_username_with_connection(
pub async fn list_weights_by_username_with_connection(
mysql_connection: &mut PoolConnection<MySql>,
creator_username: &str,
can_see_deleted: bool,
@@ -47,7 +58,7 @@ pub async fn list_datasets_by_username_with_connection(
list_datasets_by_creator_username(mysql_connection, creator_username, can_see_deleted)
.await;
let datasets : Vec<InternalRawWeightsRecordForList> = match datasets {
let datasets : Vec<WeightsRecord> = match datasets {
Ok(datasets) => datasets,
Err(err) => {
match err {
@@ -55,8 +66,8 @@ pub async fn list_datasets_by_username_with_connection(
return Ok(Vec::new());
},
_ => {
warn!("dataset list query error: {:?}", err);
return Err(anyhow!("dataset list query error"));
warn!("weights dataset list query error: {:?}", err);
return Err(anyhow!("weights dataset list query error"));
}
}
}
@@ -65,21 +76,26 @@ pub async fn list_datasets_by_username_with_connection(
Ok(datasets.into_iter()
.map(|dataset| {
WeightsRecord {
dataset_token: dataset.token,
weights_token: ModelWeightToken(dataset.token),
title: dataset.title,
ietf_language_tag: dataset.ietf_language_tag,
ietf_primary_language_subtag: dataset.ietf_primary_language_subtag,
creator_user_token: dataset.creator_user_token,
creator_username: dataset.creator_username,
creator_display_name: dataset.creator_display_name,
creator_email_gravatar_hash: dataset.creator_email_gravatar_hash,
creator_set_visibility: dataset.creator_set_visibility,
creator_ip_address: dataset.creator_ip_address,
weights_type: dataset.weights_type,
weights_category: dataset.weights_category,
description_markdown: dataset.description_markdown,
description_rendered_html: dataset.description_rendered_html,
cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count,
cached_user_ratings_total_count: dataset.cached_user_ratings_total_count,
maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio,
cached_user_ratings_total_count: dataset.cached_user_ratings_total_count,
created_at: dataset.created_at,
updated_at: dataset.updated_at,
}
})
.filter(|dataset| {
dataset.creator_username == creator_username || dataset.creator_set_visibility == Visibility::Public || can_see_deleted
dataset.weights_token == creator_username || dataset.creator_set_visibility == Visibility::Public || can_see_deleted
})
.collect::<Vec<WeightsRecord>>())
}
@@ -89,13 +105,13 @@ async fn list_datasets_by_creator_username(
mysql_connection: &mut PoolConnection<MySql>,
creator_username: &str,
can_see_deleted: bool,
) -> AnyhowResult<Vec<InternalRawWeightsRecordForList>> {
) -> AnyhowResult<Vec<WeightsRecord>> {
// TODO: There has to be a better way.
// Sqlx doesn't like anything except string literals.
let maybe_datasets = if !can_see_deleted {
info!("listing datasets for user;");
sqlx::query_as!(
InternalRawWeightsRecordForList,
WeightsRecord,
r#"
SELECT
zd.token as `token: tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken`,
@@ -123,7 +139,7 @@ async fn list_datasets_by_creator_username(
} else {
info!("listing datasets for user");
sqlx::query_as!(
InternalRawWeightsRecordForList
WeightsRecord
,
r#"
SELECT
@@ -151,3 +167,13 @@ async fn list_datasets_by_creator_username(
Ok(maybe_datasets)
}
#[cfg(test)]
mod tests {
#[test]
fn test() {
assert_eq!("one", "two"); // This fails
}
}
@@ -1 +1,3 @@
pub mod create_weight;
//pub mod delete_weight;
pub mod get_weight;
@@ -1,10 +1,15 @@
use anyhow::anyhow;
use sqlx::MySqlPool;
use log::info;
use sqlx::mysql::MySqlPoolOptions;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
pub struct UpdateWeightArgs<'a> {
pub weight_token: &ModelWeightToken,
pub title: Option<&'a str>,
@@ -48,3 +53,39 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use sqlx::MySqlPoolOptions;
use easyenv;
use tokio;
#[tokio::test]
async fn test_update_weights() -> AnyhowResult<()> {
let db_connection_string = easyenv::get_env_string_or_default(
"MYSQL_URL",
DEFAULT_MYSQL_CONNECTION_STRING
);
let pool = MySqlPoolOptions::new()
.max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?)
.connect(&db_connection_string)
.await?;
let args = UpdateWeightArgs {
weight_token: &ModelWeightToken::new("test_token"),
title: Some("Test Title"),
description_markdown: "Test Description Markdown",
description_rendered_html: "Test Description HTML",
creator_set_visibility: &Visibility::Public,
mysql_pool: &pool,
};
let result = update_weights(args).await;
assert!(result.is_ok());
Ok(())
}
}
@@ -3,10 +3,10 @@ use sqlx::MySqlPool;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::zs_voices::ZsVoiceToken;
pub struct UpdateVoiceArgs<'a> {
pub voice_token: &'a ModelWeightToken,
pub voice_token: &'a ZsVoiceToken,
pub voice_title: Option<&'a str>,
pub creator_set_visibility: &'a Visibility,
pub maybe_mod_user_token: Option<&'a str>,
@@ -18,7 +18,8 @@ pub struct UpdateVoiceArgs<'a> {
pub async fn update_voice(args: UpdateVoiceArgs<'_>) -> AnyhowResult<()>{
let mut transaction = args.mysql_pool.begin().await?;
let query_result = sqlx::query!(
let query_result = sqlx
::query!(
r#"
UPDATE zs_voices
SET
@@ -39,11 +40,13 @@ pub async fn update_voice(args: UpdateVoiceArgs<'_>) -> AnyhowResult<()>{
args.voice_token.as_str(),
).execute(args.mysql_pool).await;
// TODO(Kasisnu): This should probably rollback
transaction.commit().await?;
match query_result {
Ok(_) => Ok(()),
Err(err) => Err(anyhow!("zs voice update error: {:?}", err)),
Err(err) =>
{
Err(anyhow!("zs voice update error: {:?}", err))
},
}
}