diff --git a/.env b/.env new file mode 100644 index 0000000000..a39f1b1d8d --- /dev/null +++ b/.env @@ -0,0 +1,79 @@ +SERVER_ENVIRONMENT=development + +DATABASE_URL=mysql://storyteller:password@localhost/storyteller +#DATABASE_URL=mysql://root:root@localhost/storyteller +#DATABASE_URL=mysql://root@localhost/storyteller + +# Diesel-cli reads this for the migration directory to use. +# We've renamed it so the migrations are sorted near the top of the project directory tree +# (since they kind of serve as documentation) +MIGRATION_DIRECTORY=_sql/migrations/ + +# SQLX normally requires a live database to reflect upon at build time. +# This setting turns that on and requires a `sql-data.json` cache file instead. +# See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme +#SQLX_OFFLINE=true + +# NB(bt, 2023-03-20): +# These are used in storyteller-web, image-generation-service, and websocket-gateway. +# I'm moving them to env configs that live under storyteller-web/.env. +# This will break the other services for now. +# # We can't serve HTTPS locally without self-signed certificate nonsense. +# COOKIE_SECURE=false +# COOKIE_DOMAIN=.jungle.horse +# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse + +# Bucket config (non-secret) +REGION_NAME=us-east1 + +TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models + +W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public + +PRIVATE_BUCKET_NAME=dev-vocodes-private-uploads +PUBLIC_BUCKET_NAME=dev-vocodes-public + +# Bucket roots +AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio +W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates +W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output + +W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip +W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py +W2L_MODEL_FILENAME=wav2lip_gan.pth + +#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth +W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth +W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 + +W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py + +TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts +TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py + +TACOTRON_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts +TACOTRON_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" +TACOTRON_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_tacotron.py" + +TALKNET_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts +TALKNET_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_talknet.py" + +HIFIGAN_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts +HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" +HIFIGAN_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_hifigan.py" + +TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 + +#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID +#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! +TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt + +TTS_HIFIGAN_VOCODER_MODEL_FILENAME=hifigan_universal_g_02500000.pt +TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt + +# Development key. Permissible to check in. +NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL + +#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. diff --git a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json similarity index 76% rename from .sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json rename to .sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json index 0115c0bc84..047b14d7ff 100644 --- a/.sqlx/query-f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d.json +++ b/.sqlx/query-02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -120,5 +120,5 @@ true ] }, - "hash": "f1ef63978793e7da5bd5e1a486254dd436db8021c8f049efa41f43cf7ce75f1d" + "hash": "02136088e16742de2154cd9e84eeebbdb76045ad26aeb1c517a54c296bf31087" } diff --git a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json b/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json deleted file mode 100644 index fb10d9d281..0000000000 --- a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - }, - "hash": "030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1" -} diff --git a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json similarity index 76% rename from .sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json rename to .sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json index c4b6c41fdb..48546a186c 100644 --- a/.sqlx/query-99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16.json +++ b/.sqlx/query-1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n f.created_at,\n f.updated_at,\n f.deleted_at\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\nWHERE\n f.entity_type = ?\n AND f.entity_token = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -98,5 +98,5 @@ true ] }, - "hash": "99cff649bf75f5b05330e7db8536ecc832affbae7c9710e158099079609e5e16" + "hash": "1c1ef2195e81362d0eb023ede236c6fa65ede80d9efcaa4996694ce32c12cc0c" } diff --git a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json b/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json deleted file mode 100644 index b6a7dbec85..0000000000 --- a/.sqlx/query-4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\n UPDATE model_weights\n SET\n title = ?,\n description_markdown = ?,\n maybe_thumbnail_token = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n version = version + 1\n WHERE token = ?\n LIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - }, - "hash": "4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4" -} diff --git a/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json new file mode 100644 index 0000000000..c829a96d4d --- /dev/null +++ b/.sqlx/query-576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nUPDATE user_bookmarks\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "576e5a72e385d44a1e53dc473dd0b5a04e33d7c6b9c38bef96f275b064a12386" +} diff --git a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json b/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json deleted file mode 100644 index fefdbaf494..0000000000 --- a/.sqlx/query-6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nUPDATE favorites\nSET\ndeleted_at = CURRENT_TIMESTAMP,\nversion = version + 1\nWHERE\ntoken = ?\nAND user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "6a9fc0a3d983eb1d2bdf7d4c80904d73b9f4a649098fc0ba78cbc20cfde12437" -} diff --git a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json b/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json deleted file mode 100644 index 01e7177d85..0000000000 --- a/.sqlx/query-72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO favorites\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "72eaa9124662acf0ed354b868fcdfe33c94b61046c30940f55cdc73789f94041" -} diff --git a/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json new file mode 100644 index 0000000000..cc5ca52c8c --- /dev/null +++ b/.sqlx/query-81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\n UPDATE model_weights\n SET\n title = COALESCE(?, title),\n description_markdown = COALESCE(?, description_markdown),\n maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token),\n description_rendered_html = COALESCE(?, description_rendered_html),\n creator_set_visibility = COALESCE(?, creator_set_visibility),\n version = version + 1\n WHERE token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a" +} diff --git a/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json new file mode 100644 index 0000000000..6a0e91bef8 --- /dev/null +++ b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n maybe_origin_filename = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + }, + "hash": "8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588" +} diff --git a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json similarity index 70% rename from .sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json rename to .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json index 2c4133a738..33334d1769 100644 --- a/.sqlx/query-4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038.json +++ b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "4214d8e86b81eb0138a34702ce12960ee007ca602a7bd64b68924515f8fbb038" + "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" } diff --git a/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json new file mode 100644 index 0000000000..d2dc52c79e --- /dev/null +++ b/.sqlx/query-e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO user_bookmarks\nSET\n token = ?,\n user_token = ?,\n entity_type = ?,\n entity_token = ?\n\nON DUPLICATE KEY UPDATE\n deleted_at = NULL,\n version = version + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + }, + "hash": "e8f3bb68896d316498b52fdaaf4c352624b79bd4e097f57db5ad59f53fd66995" +} diff --git a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json similarity index 70% rename from .sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json rename to .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json index 4ebd7fa022..55d497beee 100644 --- a/.sqlx/query-c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4.json +++ b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json @@ -1,11 +1,11 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::favorites::FavoriteToken`,\n f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n favorites AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "token: tokens::tokens::favorites::FavoriteToken", + "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", "type_info": { "type": "VarString", "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -15,7 +15,7 @@ }, { "ordinal": 1, - "name": "entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType", + "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", "type_info": { "type": "VarString", "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", @@ -197,5 +197,5 @@ true ] }, - "hash": "c33f8df5a9ed1991366d6078460d9e936a5470d1794ea319d6cf19b0f5338fd4" + "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" } diff --git a/Cargo.lock b/Cargo.lock index 4193767e2c..9f21b922eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -39,7 +39,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -78,7 +78,7 @@ dependencies = [ "log", "mime", "mime_guess", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", ] @@ -110,12 +110,12 @@ dependencies = [ "language-tags", "local-channel", "mime", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "rand 0.8.5", "sha1 0.10.6", "smallvec 1.11.2", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", "zstd 0.12.4", @@ -127,7 +127,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -153,7 +153,7 @@ dependencies = [ "serde_json", "serde_plain", "tempfile", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -164,8 +164,8 @@ checksum = "0a0a77f836d869f700e5b47ac7c3c8b9c8bc82e4aec861954c6198abee3ebd4d" dependencies = [ "darling 0.20.3", "parse-size", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -190,7 +190,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "actix-macros", "futures-core", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -206,7 +206,7 @@ dependencies = [ "futures-util", "mio 0.8.9", "socket2 0.5.5", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -268,7 +268,7 @@ dependencies = [ "smallvec 1.11.2", "socket2 0.5.5", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -285,7 +285,7 @@ dependencies = [ "bytestring", "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -296,8 +296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -307,8 +307,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -416,8 +416,8 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9afe97decc070be1864d96ab118383abed1c58fc09696663666049431110c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -465,7 +465,7 @@ dependencies = [ "serde_json", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -600,7 +600,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -624,7 +624,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -633,8 +633,8 @@ version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -681,7 +681,7 @@ dependencies = [ "native-tls", "serde", "serde_json", - "url 2.5.0", + "url", ] [[package]] @@ -715,7 +715,7 @@ dependencies = [ "serde", "thiserror", "time 0.3.30", - "url 2.5.0", + "url", ] [[package]] @@ -810,8 +810,8 @@ dependencies = [ "serde_derive", "serde_json", "sqlx", - "tokio 1.34.0", - "toml", + "tokio", + "toml 0.5.11", "url_config", ] @@ -1142,8 +1142,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1162,7 +1162,7 @@ dependencies = [ "log", "my-workspace-hack", "rust-s3", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1218,7 +1218,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tokio-util", ] @@ -1252,7 +1252,7 @@ dependencies = [ "anyhow", "easyenv", "my-workspace-hack", - "url 2.5.0", + "url", ] [[package]] @@ -1278,7 +1278,7 @@ dependencies = [ "rand 0.8.5", "serde", "tempfile", - "toml", + "toml 0.5.11", "uuid 0.8.2", ] @@ -1294,7 +1294,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "time 0.3.30", "version_check", ] @@ -1386,17 +1386,6 @@ dependencies = [ "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1404,25 +1393,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.15", + "crossbeam-epoch", "crossbeam-utils 0.8.16", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -1436,17 +1410,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1522,7 +1485,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1564,8 +1527,8 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1578,8 +1541,8 @@ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 1.0.109", ] @@ -1592,8 +1555,8 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "strsim 0.10.0", "syn 2.0.39", ] @@ -1605,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1616,7 +1579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core 0.14.4", - "quote 1.0.33", + "quote", "syn 1.0.109", ] @@ -1627,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", - "quote 1.0.33", + "quote", "syn 2.0.39", ] @@ -1730,8 +1693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling 0.14.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -1752,8 +1715,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1791,7 +1754,7 @@ dependencies = [ "sqlx", "storyteller_root", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1812,7 +1775,7 @@ dependencies = [ "mysql_queries", "sqlx", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -1825,8 +1788,8 @@ dependencies = [ "byteorder", "diesel_derives", "mysqlclient-sys", - "percent-encoding 2.3.1", - "url 2.5.0", + "percent-encoding", + "url", ] [[package]] @@ -1836,8 +1799,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" dependencies = [ "diesel_table_macro_syntax", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1912,8 +1875,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -1976,7 +1939,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", - "tokio 1.34.0", + "tokio", "zip", ] @@ -2022,7 +1985,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2067,13 +2030,13 @@ dependencies = [ "bytes 1.5.0", "dyn-clone", "lazy_static", - "percent-encoding 2.3.1", + "percent-encoding", "reqwest", "rustc_version 0.2.3", "serde", "serde_json", "serde_with", - "url 2.5.0", + "url", "void", ] @@ -2099,7 +2062,7 @@ dependencies = [ "storyteller_root", "strum", "tokens", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -2174,7 +2137,7 @@ dependencies = [ "tempdir", "testing", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -2214,7 +2177,8 @@ dependencies = [ "sqlx-core", "sqlx-mysql", "strum", - "toml", + "toml 0.5.11", + "utoipa", ] [[package]] @@ -2383,7 +2347,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", ] [[package]] @@ -2511,8 +2475,8 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -2638,7 +2602,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.34.0", + "tokio", "tokio-util", "tracing", ] @@ -2779,8 +2743,8 @@ dependencies = [ "log", "mac", "markup5ever", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -2830,7 +2794,7 @@ dependencies = [ "serde_json", "serde_qs 0.8.5", "serde_urlencoded", - "url 2.5.0", + "url", ] [[package]] @@ -2893,7 +2857,7 @@ dependencies = [ "itoa 1.0.9", "pin-project-lite", "socket2 0.4.10", - "tokio 1.34.0", + "tokio", "tower-service", "tracing", "want", @@ -2908,7 +2872,7 @@ dependencies = [ "bytes 1.5.0", "hyper", "native-tls", - "tokio 1.34.0", + "tokio", "tokio-native-tls", ] @@ -2949,17 +2913,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -3067,7 +3020,6 @@ dependencies = [ "mimetypes", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "once_cell", "r2d2_redis", "rand 0.8.5", @@ -3081,7 +3033,7 @@ dependencies = [ "tempdir", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -3129,15 +3081,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.10.5" @@ -3283,15 +3226,15 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna 0.5.0", + "idna", "mime", "native-tls", "nom", "once_cell", "quoted_printable", "socket2 0.5.5", - "tokio 1.34.0", - "url 2.5.0", + "tokio", + "url", ] [[package]] @@ -3352,9 +3295,10 @@ version = "0.1.1" dependencies = [ "chrono", "futures 0.1.31", + "my-workspace-hack", "redis 0.13.0", "time 0.1.45", - "tokio 0.1.22", + "tokio", "version-sync", ] @@ -3491,20 +3435,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "maybe-async" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3557,15 +3495,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.7.1" @@ -3719,8 +3648,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3756,11 +3685,11 @@ dependencies = [ "generic-array", "getrandom 0.1.16", "getrandom 0.2.11", - "hashbrown 0.12.3", "hashbrown 0.14.2", "heck", "hmac 0.12.1", "hyper", + "indexmap", "lazy_static", "libc", "log", @@ -3798,16 +3727,17 @@ dependencies = [ "syn 2.0.39", "time 0.3.30", "time-macros", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", "twitch_oauth2", "unicode-bidi", "unicode-normalization", - "url 2.5.0", + "url", "void", "winapi 0.3.9", "windows-sys 0.48.0", + "zip", ] [[package]] @@ -3834,8 +3764,8 @@ dependencies = [ "serial_test", "sqlx", "tokens", - "tokio 1.34.0", - "toml", + "tokio", + "toml 0.5.11", ] [[package]] @@ -3889,25 +3819,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" -[[package]] -name = "newrelic-telemetry" -version = "0.1.0" -dependencies = [ - "anyhow", - "env_logger", - "flate2", - "futures 0.3.29", - "hyper", - "hyper-tls", - "log", - "my-workspace-hack", - "rand 0.8.5", - "serde", - "serde_json", - "tokio 1.34.0", - "uuid 0.8.2", -] - [[package]] name = "nix" version = "0.26.4" @@ -4107,8 +4018,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4258,12 +4169,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -4327,8 +4232,8 @@ checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4365,8 +4270,8 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -4477,8 +4382,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", "version_check", ] @@ -4489,20 +4394,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "version_check", ] -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid", -] - [[package]] name = "proc-macro2" version = "1.0.69" @@ -4521,17 +4417,6 @@ dependencies = [ "cc", ] -[[package]] -name = "pulldown-cmark" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1b74cc784b038a9921fd1a48310cc2e238101aa8ae0b94201e2d85121dd68b5" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4573,22 +4458,13 @@ dependencies = [ "memchr", ] -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.69", + "proc-macro2", ] [[package]] @@ -4733,7 +4609,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-deque 0.8.3", + "crossbeam-deque", "crossbeam-utils 0.8.16", ] @@ -4757,7 +4633,7 @@ dependencies = [ "dtoa 0.4.8", "futures 0.1.31", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", "tokio-codec", "tokio-executor", @@ -4765,7 +4641,7 @@ dependencies = [ "tokio-sync", "tokio-tcp", "tokio-uds", - "url 2.5.0", + "url", ] [[package]] @@ -4778,9 +4654,9 @@ dependencies = [ "combine 4.6.6", "dtoa 0.4.8", "itoa 0.4.8", - "percent-encoding 2.3.1", + "percent-encoding", "sha1 0.6.1", - "url 2.5.0", + "url", ] [[package]] @@ -4794,13 +4670,13 @@ dependencies = [ "combine 4.6.6", "futures-util", "itoa 1.0.9", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "ryu", "sha1 0.6.1", - "tokio 1.34.0", + "tokio", "tokio-util", - "url 2.5.0", + "url", ] [[package]] @@ -4915,17 +4791,17 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "serde", "serde_json", "serde_urlencoded", "system-configuration", - "tokio 1.34.0", + "tokio", "tokio-native-tls", "tokio-util", "tower-service", - "url 2.5.0", + "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", @@ -4942,7 +4818,7 @@ dependencies = [ "serde", "serde_derive", "sqlx", - "toml", + "toml 0.5.11", ] [[package]] @@ -5023,8 +4899,8 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rust-embed-utils", "shellexpand", "syn 2.0.39", @@ -5071,7 +4947,7 @@ dependencies = [ "maybe-async", "md5", "minidom", - "percent-encoding 2.3.1", + "percent-encoding", "quick-xml 0.26.0", "reqwest", "serde", @@ -5079,9 +4955,9 @@ dependencies = [ "sha2 0.10.8", "thiserror", "time 0.3.30", - "tokio 1.34.0", + "tokio", "tokio-stream", - "url 2.5.0", + "url", ] [[package]] @@ -5251,7 +5127,7 @@ dependencies = [ "clap 2.34.0", "errors", "my-workspace-hack", - "tokio 1.34.0", + "tokio", "web_scrapers", ] @@ -5340,7 +5216,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0", + "semver-parser", ] [[package]] @@ -5355,12 +5231,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "semver-parser" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46e1121e8180c12ff69a742aabc4f310542b6ccb69f1691689ac17fdf8618aa" - [[package]] name = "serde" version = "1.0.193" @@ -5376,8 +5246,8 @@ version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5417,7 +5287,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5428,7 +5298,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cac3f1e2ca2fe333923a1ae72caca910b98ed0630bb35ef6f8c8517d6e81afa" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", "serde", "thiserror", ] @@ -5439,11 +5309,20 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5473,8 +5352,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5498,8 +5377,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -5652,8 +5531,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -5767,7 +5646,7 @@ dependencies = [ "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue 0.3.8", + "crossbeam-queue", "dotenvy", "either", "event-listener", @@ -5784,7 +5663,7 @@ dependencies = [ "native-tls", "once_cell", "paste", - "percent-encoding 2.3.1", + "percent-encoding", "rustls 0.21.9", "rustls-pemfile", "serde", @@ -5793,10 +5672,10 @@ dependencies = [ "smallvec 1.11.2", "sqlformat", "thiserror", - "tokio 1.34.0", + "tokio", "tokio-stream", "tracing", - "url 2.5.0", + "url", "webpki-roots 0.25.3", ] @@ -5806,8 +5685,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "sqlx-core", "sqlx-macros-core", "syn 1.0.109", @@ -5825,8 +5704,8 @@ dependencies = [ "heck", "hex", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "serde", "serde_json", "sha2 0.10.8", @@ -5836,8 +5715,8 @@ dependencies = [ "sqlx-sqlite", "syn 1.0.109", "tempfile", - "tokio 1.34.0", - "url 2.5.0", + "tokio", + "url", ] [[package]] @@ -5869,7 +5748,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "rand 0.8.5", "rsa", "serde", @@ -5939,11 +5818,11 @@ dependencies = [ "futures-util", "libsqlite3-sys", "log", - "percent-encoding 2.3.1", + "percent-encoding", "serde", "sqlx-core", "tracing", - "url 2.5.0", + "url", "urlencoding", ] @@ -6047,7 +5926,7 @@ dependencies = [ "native-tls", "once_cell", "pin-project-lite", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_caching", @@ -6067,8 +5946,8 @@ dependencies = [ "symphonia", "time 0.3.30", "tokens", - "tokio 1.34.0", - "toml", + "tokio", + "toml 0.5.11", "tts_common", "twitch_common", "twitch_oauth2", @@ -6111,8 +5990,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", ] [[package]] @@ -6154,8 +6033,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "rustversion", "syn 2.0.39", ] @@ -6351,25 +6230,14 @@ dependencies = [ "symphonia-metadata", ] -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid", -] - [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6379,8 +6247,8 @@ version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "unicode-ident", ] @@ -6486,8 +6354,8 @@ version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6580,34 +6448,10 @@ dependencies = [ "sqlx", "sqlx-core", "strum", - "toml", + "toml 0.5.11", "utoipa", ] -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.34.0" @@ -6638,16 +6482,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - [[package]] name = "tokio-executor" version = "0.1.10" @@ -6658,17 +6492,6 @@ dependencies = [ "futures 0.1.31", ] -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - [[package]] name = "tokio-io" version = "0.1.13" @@ -6686,8 +6509,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6698,7 +6521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6727,7 +6550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "webpki", ] @@ -6739,7 +6562,7 @@ checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6766,35 +6589,6 @@ dependencies = [ "tokio-reactor", ] -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue 0.2.3", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - [[package]] name = "tokio-tungstenite" version = "0.15.0" @@ -6805,28 +6599,13 @@ dependencies = [ "log", "pin-project", "rustls 0.19.1", - "tokio 1.34.0", + "tokio", "tokio-rustls", "tungstenite", "webpki", "webpki-roots 0.21.1", ] -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - [[package]] name = "tokio-uds" version = "0.2.7" @@ -6855,7 +6634,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.34.0", + "tokio", "tracing", ] @@ -6868,6 +6647,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -6892,8 +6705,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] @@ -6940,7 +6753,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -6966,7 +6779,6 @@ dependencies = [ "memory_caching", "my-workspace-hack", "mysql_queries", - "newrelic-telemetry", "r2d2_redis", "serde", "serde_derive", @@ -6974,7 +6786,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", "tts_common", ] @@ -7012,7 +6824,7 @@ dependencies = [ "rustls-native-certs", "sha-1", "thiserror", - "url 2.5.0", + "url", "utf-8", "webpki", ] @@ -7042,7 +6854,7 @@ dependencies = [ "my-workspace-hack", "mysql_queries", "native-tls", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "r2d2_redis", "rand 0.8.5", "redis_common", @@ -7055,9 +6867,9 @@ dependencies = [ "sqlx", "time 0.3.30", "tokens", - "tokio 1.34.0", + "tokio", "tokio-tungstenite", - "toml", + "toml 0.5.11", "tts_common", "twitch_api2", "twitch_common", @@ -7087,7 +6899,7 @@ dependencies = [ "thiserror", "twitch_oauth2", "typed-builder", - "url 2.5.0", + "url", "version_check", ] @@ -7123,7 +6935,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "url 2.5.0", + "url", ] [[package]] @@ -7132,8 +6944,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -7185,12 +6997,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -7218,17 +7024,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - [[package]] name = "url" version = "2.5.0" @@ -7236,8 +7031,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", - "percent-encoding 2.3.1", + "idna", + "percent-encoding", "serde", ] @@ -7264,7 +7059,7 @@ version = "0.0.1" dependencies = [ "my-workspace-hack", "once_cell", - "pulldown-cmark 0.9.3", + "pulldown-cmark", "regex", "unicode-segmentation", ] @@ -7310,7 +7105,7 @@ dependencies = [ "strum_macros", "time 0.3.30", "tokens", - "toml", + "toml 0.5.11", "user_input_common", ] @@ -7345,8 +7140,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "regex", "syn 2.0.39", ] @@ -7396,18 +7191,17 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version-sync" -version = "0.8.1" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "844f3d3a2467f15cb999f5af7775f6e108ac546d4f42365832ed4c755404f806" +checksum = "835169da0173ea373ddf5987632aac1f918967fbbe58195e304342282efa6089" dependencies = [ - "itertools 0.8.2", - "proc-macro2 0.4.30", - "pulldown-cmark 0.4.1", + "proc-macro2", + "pulldown-cmark", "regex", - "semver-parser 0.9.0", - "syn 0.15.44", - "toml", - "url 1.7.2", + "semver 1.0.20", + "syn 2.0.39", + "toml 0.7.8", + "url", ] [[package]] @@ -7450,7 +7244,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7479,7 +7273,7 @@ dependencies = [ "sqlx", "subprocess", "tempdir", - "tokio 1.34.0", + "tokio", ] [[package]] @@ -7544,8 +7338,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-shared", ] @@ -7568,7 +7362,7 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ - "quote 1.0.33", + "quote", "wasm-bindgen-macro-support", ] @@ -7578,8 +7372,8 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7708,8 +7502,8 @@ dependencies = [ "sha2 0.9.9", "sqlx", "time 0.3.30", - "tokio 1.34.0", - "toml", + "tokio", + "toml 0.5.11", "twitch_api2", "twitch_common", "twitch_oauth2", @@ -7912,6 +7706,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -7947,8 +7750,8 @@ version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", + "proc-macro2", + "quote", "syn 2.0.39", ] diff --git a/Cargo.toml b/Cargo.toml index 7649065a0d..1ea5d8fefc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,8 @@ members = [ # Vendored code "crates/vendor/limitation", - "crates/vendor/newrelic-telemetry-sdk-rust", + # NB(bt,2023-12-12): Newrelic keeps breaking with upgrades. Kill for now. + #"crates/vendor/newrelic-telemetry-sdk-rust", # cargo-hakari workspace hack (compilation speedup hack) "crates/my-workspace-hack", diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 8520aa609e..9fd7b4ee20 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -344,6 +344,36 @@ where token IN ( 'vcm_zmxv2k1q9jd8' ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'takedown request' +where token IN ( + 'vcm_3mzcjkf78x7m', + 'vcm_7k2t1wnyqxsc', + 'vcm_w2edpe6q2nya', + 'vcm_vz9cg294xa29', + 'vcm_0z1ys58589k7', + 'vcm_4x8a30h1r26z', + 'vcm_0vf9prxsww60', + 'vcm_yxbadkvqhg16', + 'vcm_p0ezrc6wvzak', + 'vcm_6r5pbtexyvtp', + 'vcm_pb0ewrf7ych5', + 'vcm_7frb2xk0cxk3', + 'vcm_x129c9s2af06', + 'vcm_e0snndfd59b6', + 'vcm_nb8zb0djqwqw', + 'vcm_2erz5hbh8tpf', + 'vcm_e9wn2ccqe3bk', + 'vcm_f8jbxdbjgcjb', + 'vcm_x02h7z3jqxb0', + 'vcm_xy97xcgnk9y6', + 'vcm_a47wtfbsjrfc', + 'vcm_kygenhbg5mgj', + 'vcm_s8gndv19yt7p' +); + diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql new file mode 100644 index 0000000000..8b6d041461 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE user_bookmarks; \ No newline at end of file diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql new file mode 100644 index 0000000000..54ad3a50b0 --- /dev/null +++ b/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql @@ -0,0 +1,46 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Favorites that can be attached to just about anything. +-- This is a polymorphic table and can apply to a wide variety of entities. +CREATE TABLE user_bookmarks +( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- User making the favorite + user_token VARCHAR(32) NOT NULL, + + -- The type of entity the favorite applies to + entity_type VARCHAR(32) NOT NULL, + + -- The token of the entity the favorite applies to + entity_token VARCHAR(32) NOT NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY index_entity_type_entity_token (entity_type, entity_token), + KEY index_user_token_entity_token (user_token, entity_type), + KEY index_user_token (user_token), + KEY index_created_at (created_at), + KEY index_user_deleted_at (deleted_at) + +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml index 80145886da..866ad39853 100644 --- a/crates/_types/buckets/Cargo.toml +++ b/crates/_types/buckets/Cargo.toml @@ -17,6 +17,8 @@ path = "src/lib.rs" # Internal crockford = { path = "../../lib/crockford" } errors = { path = "../../lib/errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/_types/buckets/src/private/mod.rs b/crates/_types/buckets/src/private/mod.rs index 27089e4df1..d3090254af 100644 --- a/crates/_types/buckets/src/private/mod.rs +++ b/crates/_types/buckets/src/private/mod.rs @@ -1 +1,2 @@ -pub mod private_path; \ No newline at end of file +pub mod private_path; +pub mod zs_voices; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/directory.rs b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/zs_voices/directory.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_directory.rs index 1cfc740769..9d4ecee304 100644 --- a/crates/_types/buckets/src/public/zs_voices/directory.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_directory.rs @@ -1,6 +1,6 @@ use crockford::crockford_entropy_lower; -use crate::private::private_path::PrivatePath; +use crate::private::private_path::PrivatePath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. @@ -79,7 +79,7 @@ impl ZeroShotVoiceEmbeddingBucketDirectory { #[cfg(test)] mod tests { - use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; + use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/zs_voices/file.rs b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs similarity index 88% rename from crates/_types/buckets/src/public/zs_voices/file.rs rename to crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs index 6428936c7c..47948912ec 100644 --- a/crates/_types/buckets/src/public/zs_voices/file.rs +++ b/crates/_types/buckets/src/private/zs_voices/bucket_file_path.rs @@ -3,11 +3,10 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::private::private_path::PrivatePath; -use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; +use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory}; // TODO: Generate these from a macro. - /// The original user upload file. /// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] @@ -74,11 +73,9 @@ impl ZeroShotVoiceEmbeddingBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; - mod with_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; #[test] pub fn generate_new_entropy() { @@ -97,8 +94,8 @@ mod tests { } mod without_prefix_and_extension { - use crate::public::zs_voices::directory::ModelCategory; - use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; + use crate::private::zs_voices::bucket_directory::ModelCategory; + use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath}; use super::*; @@ -135,7 +132,7 @@ mod tests { #[test] pub fn get_object_hash() { let hash = "abcdefghijk"; - let file = MediaFileBucketPath::from_object_hash(hash, None, None); + let file = ZeroShotVoiceEmbeddingBucketPath::from_object_hash(hash, ModelCategory::Tts, ModelType::VallEx, 1); assert_eq!(file.get_object_hash(), hash); } } diff --git a/crates/_types/buckets/src/private/zs_voices/mod.rs b/crates/_types/buckets/src/private/zs_voices/mod.rs new file mode 100644 index 0000000000..29c183a5c6 --- /dev/null +++ b/crates/_types/buckets/src/private/zs_voices/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_files/directory.rs b/crates/_types/buckets/src/public/media_files/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/directory.rs rename to crates/_types/buckets/src/public/media_files/bucket_directory.rs index e0083da128..855332bcf3 100644 --- a/crates/_types/buckets/src/public/media_files/directory.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_directory.rs @@ -4,7 +4,6 @@ use crate::public::public_path::PublicPath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; // TODO: Generate these from a macro. - // TODO: Use a central path registry for quick reference const MEDIA_FILE_DIRECTORY: &str = "/media"; @@ -47,7 +46,7 @@ impl MediaFileBucketDirectory { #[cfg(test)] mod tests { - use crate::public::media_files::directory::MediaFileBucketDirectory; + use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_files/original_file.rs b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs similarity index 96% rename from crates/_types/buckets/src/public/media_files/original_file.rs rename to crates/_types/buckets/src/public/media_files/bucket_file_path.rs index efd0fd9458..e503819440 100644 --- a/crates/_types/buckets/src/public/media_files/original_file.rs +++ b/crates/_types/buckets/src/public/media_files/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_files::directory::MediaFileBucketDirectory; +use crate::public::media_files::bucket_directory::MediaFileBucketDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "file.bin"; @@ -79,7 +79,7 @@ impl MediaFileBucketPath { mod tests { use std::path::PathBuf; - use crate::public::media_files::original_file::MediaFileBucketPath; + use crate::public::media_files::bucket_file_path::MediaFileBucketPath; mod with_prefix_and_extension { use super::*; diff --git a/crates/_types/buckets/src/public/media_files/mod.rs b/crates/_types/buckets/src/public/media_files/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_files/mod.rs +++ b/crates/_types/buckets/src/public/media_files/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_uploads/directory.rs b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs similarity index 96% rename from crates/_types/buckets/src/public/media_uploads/directory.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_directory.rs index aa4ffc0470..77ff4ca662 100644 --- a/crates/_types/buckets/src/public/media_uploads/directory.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_directory.rs @@ -47,7 +47,7 @@ impl MediaUploadDirectory { #[cfg(test)] mod tests { - use crate::public::media_uploads::directory::MediaUploadDirectory; + use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/original_file.rs b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/media_uploads/original_file.rs rename to crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs index 7f0a40a6b7..6867ff6e5a 100644 --- a/crates/_types/buckets/src/public/media_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/media_uploads/bucket_file_path.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::media_uploads::bucket_directory::MediaUploadDirectory; use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; @@ -61,7 +61,7 @@ impl MediaUploadOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + use crate::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/media_uploads/mod.rs b/crates/_types/buckets/src/public/media_uploads/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/media_uploads/mod.rs +++ b/crates/_types/buckets/src/public/media_uploads/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index d85de56720..fcceb68d9b 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,4 +2,4 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod zs_voices; \ No newline at end of file +pub mod weight_uploads; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs similarity index 95% rename from crates/_types/buckets/src/public/voice_conversion_results/directory.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs index c853cf2d2f..4f726b2c14 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_directory.rs @@ -40,7 +40,7 @@ impl VoiceConversionResultDirectory { #[cfg(test)] mod tests { - use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; + use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; #[test] pub fn get_directory_path_str() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs similarity index 94% rename from crates/_types/buckets/src/public/voice_conversion_results/original_file.rs rename to crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs index 9e256e89d2..9163363e49 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/bucket_file_path.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use crockford::crockford_entropy_lower; use crate::public::public_path::PublicPath; -use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; +use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory; const ORIGINAL_FILE_PREFIX : &str = "fakeyou_"; const ORIGINAL_FILE_SUFFIX : &str = ".wav"; @@ -65,7 +65,7 @@ impl VoiceConversionResultOriginalFilePath { mod tests { use std::path::PathBuf; - use crate::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; + use crate::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; #[test] pub fn generate_new_entropy() { diff --git a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs index 42ede393cf..29c183a5c6 100644 --- a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs +++ b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs @@ -1,2 +1,2 @@ -pub mod directory; -pub mod original_file; \ No newline at end of file +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs new file mode 100644 index 0000000000..1f3e016fb1 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_directory.rs @@ -0,0 +1,77 @@ +use crockford::crockford_entropy_lower; +// TODO Thoughts NOTE: +// I am wondering if it be easier if we just had a single struct and impl to handle downloads from public and private buckets. +// essentially, the only difference is the prefix and file postfix and the path of where the file goes and the path will be constructed from there. +// then you can tie in the downloader as a single object that can be used for both public and private buckets. + +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +const WEIGHT_UPLOAD_DIRECTORY : &str = "/weight_upload"; + +/// Directory for user media uploads. +/// Each uploaded file gets its own directory so that we can store the original +/// file alongside re-processed derivatives. +#[derive(Clone)] +pub struct WeightUploadDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for WeightUploadDirectory {} + +impl WeightUploadDirectory { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", WEIGHT_UPLOAD_DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } +} + +#[cfg(test)] +mod tests { + use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; + + #[test] + pub fn generate_new_entropy() { + let directory = WeightUploadDirectory::generate_new(); + assert_eq!(directory.get_object_hash().len(), 32); + } + + #[test] + pub fn get_directory_path_str() { + let directory = WeightUploadDirectory::from_object_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = WeightUploadDirectory::from_object_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/weight_upload/f/o/foo"); + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = WeightUploadDirectory::from_object_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } +} diff --git a/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs new file mode 100644 index 0000000000..69ca626486 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/bucket_file_path.rs @@ -0,0 +1,100 @@ +use std::path::PathBuf; + +use crockford::crockford_entropy_lower; + +use crate::public::public_path::PublicPath; +use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory; + +const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin"; + +#[derive(Clone)] +pub struct WeightUploadOriginalFilePath { + directory: WeightUploadDirectory, + full_object_path: String, +} + +impl PublicPath for WeightUploadOriginalFilePath {} + +impl WeightUploadOriginalFilePath { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let directory = WeightUploadDirectory::from_object_hash(hash); + let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); + Self { + directory, + full_object_path, + } + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } + + pub fn to_full_object_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_object_path) + } + + pub fn get_directory(&self) -> &WeightUploadDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename() -> &'static str { + ORIGINAL_FILE_BASENAME + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::public::weight_uploads::bucket_file_path::WeightUploadOriginalFilePath; + + #[test] + pub fn generate_new_entropy() { + let file = WeightUploadOriginalFilePath::generate_new(); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin"); + } + + #[test] + pub fn to_full_object_pathbuf() { + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weight_upload/a/b/c/d/e/abcdefghijk/original_upload.bin")); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = WeightUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/weight_upload/f/o/foo/original_upload.bin"); + } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = WeightUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = WeightUploadOriginalFilePath::from_object_hash(hash); + assert_eq!(file.get_object_hash(), hash); + } +} diff --git a/crates/_types/buckets/src/public/weight_uploads/mod.rs b/crates/_types/buckets/src/public/weight_uploads/mod.rs new file mode 100644 index 0000000000..29c183a5c6 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_uploads/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/zs_voices/mod.rs b/crates/_types/buckets/src/public/zs_voices/mod.rs deleted file mode 100644 index a5c0d4bfab..0000000000 --- a/crates/_types/buckets/src/public/zs_voices/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod directory; -pub mod file; \ No newline at end of file diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 6182365ff6..255ed1a634 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -13,6 +13,11 @@ name = "enums" path = "src/lib.rs" [dependencies] +# Internal + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -20,7 +25,8 @@ serde_derive = "1.0.125" sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] } sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs b/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs deleted file mode 100644 index 5b96603ff1..0000000000 --- a/crates/_types/enums/src/by_table/favorites/favorite_entity_type.rs +++ /dev/null @@ -1,128 +0,0 @@ -use serde::Deserialize; -use serde::Serialize; -#[cfg(test)] -use strum::EnumCount; -#[cfg(test)] -use strum::EnumIter; - -/// Used in the `favorites` table in a `VARCHAR(32)` field named `entity_type`. -#[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] -pub enum FavoriteEntityType { - /// User - #[serde(rename = "user")] - User, - - /// TTS model (architecture does not matter) - #[serde(rename = "tts_model")] - TtsModel, - - /// TTS result (architecture does not matter) - #[serde(rename = "tts_result")] - TtsResult, - - /// W2L template - #[serde(rename = "w2l_template")] - W2lTemplate, - - /// W2L result - #[serde(rename = "w2l_result")] - W2lResult, - - /// MediaFile - #[serde(rename = "media_file")] - MediaFile, - - /// VoiceConversionModel - #[serde(rename = "voice_conversion_model")] - VoiceConversionModel, - - /// ZsVoice - #[serde(rename = "zs_voice")] - ZsVoice, -} - -// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! -impl_enum_display_and_debug_using_to_str!(FavoriteEntityType); -impl_mysql_enum_coders!(FavoriteEntityType); - -/// NB: Legacy API for older code. -impl FavoriteEntityType { - pub fn to_str(&self) -> &'static str { - match self { - Self::User => "user", - Self::TtsModel => "tts_model", - Self::TtsResult => "tts_result", - Self::W2lTemplate => "w2l_template", - Self::W2lResult => "w2l_result", - Self::MediaFile => "media_file", - Self::VoiceConversionModel => "voice_conversion_model", - Self::ZsVoice => "zs_voice", - } - } - - pub fn from_str(value: &str) -> Result { - match value { - "user" => Ok(Self::User), - "tts_model" => Ok(Self::TtsModel), - "tts_result" => Ok(Self::TtsResult), - "w2l_template" => Ok(Self::W2lTemplate), - "w2l_result" => Ok(Self::W2lResult), - "media_file" => Ok(Self::MediaFile), - "voice_conversion_model" => Ok(Self::VoiceConversionModel), - "zs_voice" => Ok(Self::ZsVoice), - _ => Err(format!("invalid value: {:?}", value)), - } - } -} - -#[cfg(test)] -mod tests { - use crate::by_table::favorites::favorite_entity_type::FavoriteEntityType; - use crate::test_helpers::assert_serialization; - - mod serde { - use super::*; - - #[test] - fn test_serialization() { - assert_serialization(FavoriteEntityType::User, "user"); - assert_serialization(FavoriteEntityType::TtsModel, "tts_model"); - assert_serialization(FavoriteEntityType::TtsResult, "tts_result"); - assert_serialization(FavoriteEntityType::W2lTemplate, "w2l_template"); - assert_serialization(FavoriteEntityType::W2lResult, "w2l_result"); - assert_serialization(FavoriteEntityType::MediaFile, "media_file"); - assert_serialization(FavoriteEntityType::VoiceConversionModel, "voice_conversion_model"); - assert_serialization(FavoriteEntityType::ZsVoice, "zs_voice"); - } - } - - mod impl_methods { - use super::*; - - #[test] - fn test_to_str() { - assert_eq!(FavoriteEntityType::User.to_str(), "user"); - assert_eq!(FavoriteEntityType::TtsModel.to_str(), "tts_model"); - assert_eq!(FavoriteEntityType::TtsResult.to_str(), "tts_result"); - assert_eq!(FavoriteEntityType::W2lTemplate.to_str(), "w2l_template"); - assert_eq!(FavoriteEntityType::W2lResult.to_str(), "w2l_result"); - assert_eq!(FavoriteEntityType::MediaFile.to_str(), "media_file"); - assert_eq!(FavoriteEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); - assert_eq!(FavoriteEntityType::ZsVoice.to_str(), "zs_voice"); - } - - #[test] - fn test_from_str() { - assert_eq!(FavoriteEntityType::from_str("user").unwrap(), FavoriteEntityType::User); - assert_eq!(FavoriteEntityType::from_str("tts_model").unwrap(), FavoriteEntityType::TtsModel); - assert_eq!(FavoriteEntityType::from_str("tts_result").unwrap(), FavoriteEntityType::TtsResult); - assert_eq!(FavoriteEntityType::from_str("w2l_template").unwrap(), FavoriteEntityType::W2lTemplate); - assert_eq!(FavoriteEntityType::from_str("w2l_result").unwrap(), FavoriteEntityType::W2lResult); - assert_eq!(FavoriteEntityType::from_str("media_file").unwrap(), FavoriteEntityType::MediaFile); - assert_eq!(FavoriteEntityType::from_str("voice_conversion_model").unwrap(), FavoriteEntityType::VoiceConversionModel); - assert_eq!(FavoriteEntityType::from_str("zs_voice").unwrap(), FavoriteEntityType::ZsVoice); - assert!(FavoriteEntityType::from_str("foo").is_err()); - } - } -} diff --git a/crates/_types/enums/src/by_table/favorites/mod.rs b/crates/_types/enums/src/by_table/favorites/mod.rs deleted file mode 100644 index 18ac93ed9c..0000000000 --- a/crates/_types/enums/src/by_table/favorites/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod favorite_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs index 91f06ed283..42fd07ce41 100644 --- a/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs +++ b/crates/_types/enums/src/by_table/media_uploads/media_upload_type.rs @@ -20,6 +20,9 @@ pub enum MediaUploadType { /// Video files: mp4, etc. Video, + + // Binary files: safetensors ... weights, etc. + Binary, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -33,6 +36,7 @@ impl MediaUploadType { Self::Audio => "audio", Self::Image => "image", Self::Video => "video", + Self::Binary => "binary", } } @@ -41,6 +45,7 @@ impl MediaUploadType { "audio" => Ok(Self::Audio), "image" => Ok(Self::Image), "video" => Ok(Self::Video), + "binary" => Ok(Self::Binary), _ => Err(format!("invalid value: {:?}", value)), } } @@ -56,6 +61,7 @@ mod tests { assert_serialization(MediaUploadType::Audio, "audio"); assert_serialization(MediaUploadType::Image, "image"); assert_serialization(MediaUploadType::Video, "video"); + assert_serialization(MediaUploadType::Binary, "binary"); } #[test] @@ -63,6 +69,7 @@ mod tests { assert_eq!(MediaUploadType::Audio.to_str(), "audio"); assert_eq!(MediaUploadType::Image.to_str(), "image"); assert_eq!(MediaUploadType::Video.to_str(), "video"); + assert_eq!(MediaUploadType::Binary.to_str(), "binary"); } #[test] @@ -70,6 +77,7 @@ mod tests { assert_eq!(MediaUploadType::from_str("audio").unwrap(), MediaUploadType::Audio); assert_eq!(MediaUploadType::from_str("image").unwrap(), MediaUploadType::Image); assert_eq!(MediaUploadType::from_str("video").unwrap(), MediaUploadType::Video); + assert_eq!(MediaUploadType::from_str("binary").unwrap(), MediaUploadType::Binary); assert!(MediaUploadType::from_str("foo").is_err()); } } diff --git a/crates/_types/enums/src/by_table/mod.rs b/crates/_types/enums/src/by_table/mod.rs index 91f8cb15da..97876b2094 100644 --- a/crates/_types/enums/src/by_table/mod.rs +++ b/crates/_types/enums/src/by_table/mod.rs @@ -10,6 +10,7 @@ pub mod media_uploads; pub mod model_categories; pub mod trending_model_analytics; pub mod tts_models; +pub mod user_bookmarks; pub mod user_ratings; pub mod voice_conversion_models; pub mod voice_conversion_results; @@ -19,4 +20,3 @@ pub mod model_weights; pub mod tts_render_tasks; pub mod web_rendition_targets; pub mod web_scraping_targets; -pub mod favorites; diff --git a/crates/_types/enums/src/by_table/model_weights/mod.rs b/crates/_types/enums/src/by_table/model_weights/mod.rs index 5217cf739e..0755131487 100644 --- a/crates/_types/enums/src/by_table/model_weights/mod.rs +++ b/crates/_types/enums/src/by_table/model_weights/mod.rs @@ -1,2 +1,3 @@ pub mod weights_types; pub mod weights_category; +pub mod weights_file_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index c2bca7f24d..f6c4e6ed3f 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -5,8 +5,10 @@ use strum::EnumCount; #[cfg(test)] use strum::EnumIter; + +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { #[serde(rename = "image_generation")] ImageGeneration, diff --git a/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs new file mode 100644 index 0000000000..a7adeb72cd --- /dev/null +++ b/crates/_types/enums/src/by_table/model_weights/weights_file_type.rs @@ -0,0 +1,68 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; +// TODO we will need to scan the checkpoints for malicious code. We can't just trust the file extension. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +pub enum WeightsFileType { + #[serde(rename = "checkpoint")] + Checkpoint, + #[serde(rename = "safetensors")] + SafeTensors, +} + +impl WeightsFileType { + pub fn to_str(&self) -> &'static str { + match self { + Self::Checkpoint => "checkpoint", + Self::SafeTensors => "safetensors", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "checkpoint" => Ok(Self::Checkpoint), + "safetensors" => Ok(Self::SafeTensors), + _ => Err(format!("invalid value: {:?}", value)), + } + } + + pub fn all_variants() -> BTreeSet { + BTreeSet::from([ + Self::Checkpoint, + Self::SafeTensors, + ]) + } +} + +impl_enum_display_and_debug_using_to_str!(WeightsFileType); +impl_mysql_enum_coders!(WeightsFileType); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(WeightsFileType::Checkpoint.to_str(), "checkpoint"); + assert_eq!(WeightsFileType::SafeTensors.to_str(), "safetensors"); + } + + #[test] + fn test_from_str() { + assert_eq!(WeightsFileType::from_str("checkpoint").unwrap(), WeightsFileType::Checkpoint); + assert_eq!(WeightsFileType::from_str("safetensors").unwrap(), WeightsFileType::SafeTensors); + assert!(WeightsFileType::from_str("invalid").is_err()); + } + + #[test] + fn test_all_variants() { + let variants = WeightsFileType::all_variants(); + assert_eq!(variants.len(), 2); + assert!(variants.contains(&WeightsFileType::Checkpoint)); + assert!(variants.contains(&WeightsFileType::SafeTensors)); + } +} \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index e7ff3c3c97..1b6bbcb132 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -4,10 +4,10 @@ use std::collections::BTreeSet; use strum::EnumCount; #[cfg(test)] use strum::EnumIter; - +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsType { #[serde(rename = "hifigan_tt2")] HifiganTacotron2, @@ -23,6 +23,8 @@ pub enum WeightsType { Tacotron2, #[serde(rename = "loRA")] LoRA, + #[serde(rename = "vall_e")] + VallE, } @@ -36,6 +38,7 @@ impl WeightsType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tt2", Self::LoRA => "loRA", + Self::VallE => "vall_e", } } @@ -48,6 +51,7 @@ impl WeightsType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tt2" => Ok(Self::Tacotron2), "loRA" => Ok(Self::LoRA), + "vall_e" => Ok(Self::VallE), _ => Err(format!("invalid value: {:?}", value)), } } @@ -61,6 +65,7 @@ impl WeightsType { Self::SoVitsSvc, Self::Tacotron2, Self::LoRA, + Self::VallE ]) } } @@ -82,6 +87,7 @@ mod tests { assert_eq!(WeightsType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(WeightsType::Tacotron2.to_str(), "tt2"); assert_eq!(WeightsType::LoRA.to_str(), "loRA"); + assert_eq!(WeightsType::VallE.to_str(), "vall_e"); } #[test] @@ -93,13 +99,14 @@ mod tests { assert_eq!(WeightsType::from_str("so_vits_svc").unwrap(), WeightsType::SoVitsSvc); assert_eq!(WeightsType::from_str("tt2").unwrap(), WeightsType::Tacotron2); assert_eq!(WeightsType::from_str("loRA").unwrap(), WeightsType::LoRA); + assert_eq!(WeightsType::from_str("vall_e").unwrap(), WeightsType::VallE); assert!(WeightsType::from_str("invalid").is_err()); } #[test] fn test_all_variants() { let variants = WeightsType::all_variants(); - assert_eq!(variants.len(), 7); + assert_eq!(variants.len(), 8); assert!(variants.contains(&WeightsType::HifiganTacotron2)); assert!(variants.contains(&WeightsType::RvcV2)); assert!(variants.contains(&WeightsType::StableDiffusion15)); @@ -107,5 +114,6 @@ mod tests { assert!(variants.contains(&WeightsType::SoVitsSvc)); assert!(variants.contains(&WeightsType::Tacotron2)); assert!(variants.contains(&WeightsType::LoRA)); + assert!(variants.contains(&WeightsType::VallE)); } } diff --git a/crates/_types/enums/src/by_table/user_bookmarks/mod.rs b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs new file mode 100644 index 0000000000..c3c539aad3 --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/mod.rs @@ -0,0 +1 @@ +pub mod user_bookmark_entity_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs new file mode 100644 index 0000000000..69bf47cf5e --- /dev/null +++ b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs @@ -0,0 +1,128 @@ +use serde::Deserialize; +use serde::Serialize; +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the `user_bookmarks` table in a `VARCHAR(32)` field named `entity_type`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum UserBookmarkEntityType { + /// User + #[serde(rename = "user")] + User, + + /// TTS model (architecture does not matter) + #[serde(rename = "tts_model")] + TtsModel, + + /// TTS result (architecture does not matter) + #[serde(rename = "tts_result")] + TtsResult, + + /// W2L template + #[serde(rename = "w2l_template")] + W2lTemplate, + + /// W2L result + #[serde(rename = "w2l_result")] + W2lResult, + + /// MediaFile + #[serde(rename = "media_file")] + MediaFile, + + /// VoiceConversionModel + #[serde(rename = "voice_conversion_model")] + VoiceConversionModel, + + /// ZsVoice + #[serde(rename = "zs_voice")] + ZsVoice, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(UserBookmarkEntityType); +impl_mysql_enum_coders!(UserBookmarkEntityType); + +/// NB: Legacy API for older code. +impl UserBookmarkEntityType { + pub fn to_str(&self) -> &'static str { + match self { + Self::User => "user", + Self::TtsModel => "tts_model", + Self::TtsResult => "tts_result", + Self::W2lTemplate => "w2l_template", + Self::W2lResult => "w2l_result", + Self::MediaFile => "media_file", + Self::VoiceConversionModel => "voice_conversion_model", + Self::ZsVoice => "zs_voice", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "user" => Ok(Self::User), + "tts_model" => Ok(Self::TtsModel), + "tts_result" => Ok(Self::TtsResult), + "w2l_template" => Ok(Self::W2lTemplate), + "w2l_result" => Ok(Self::W2lResult), + "media_file" => Ok(Self::MediaFile), + "voice_conversion_model" => Ok(Self::VoiceConversionModel), + "zs_voice" => Ok(Self::ZsVoice), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(UserBookmarkEntityType::User, "user"); + assert_serialization(UserBookmarkEntityType::TtsModel, "tts_model"); + assert_serialization(UserBookmarkEntityType::TtsResult, "tts_result"); + assert_serialization(UserBookmarkEntityType::W2lTemplate, "w2l_template"); + assert_serialization(UserBookmarkEntityType::W2lResult, "w2l_result"); + assert_serialization(UserBookmarkEntityType::MediaFile, "media_file"); + assert_serialization(UserBookmarkEntityType::VoiceConversionModel, "voice_conversion_model"); + assert_serialization(UserBookmarkEntityType::ZsVoice, "zs_voice"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(UserBookmarkEntityType::User.to_str(), "user"); + assert_eq!(UserBookmarkEntityType::TtsModel.to_str(), "tts_model"); + assert_eq!(UserBookmarkEntityType::TtsResult.to_str(), "tts_result"); + assert_eq!(UserBookmarkEntityType::W2lTemplate.to_str(), "w2l_template"); + assert_eq!(UserBookmarkEntityType::W2lResult.to_str(), "w2l_result"); + assert_eq!(UserBookmarkEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserBookmarkEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); + assert_eq!(UserBookmarkEntityType::ZsVoice.to_str(), "zs_voice"); + } + + #[test] + fn test_from_str() { + assert_eq!(UserBookmarkEntityType::from_str("user").unwrap(), UserBookmarkEntityType::User); + assert_eq!(UserBookmarkEntityType::from_str("tts_model").unwrap(), UserBookmarkEntityType::TtsModel); + assert_eq!(UserBookmarkEntityType::from_str("tts_result").unwrap(), UserBookmarkEntityType::TtsResult); + assert_eq!(UserBookmarkEntityType::from_str("w2l_template").unwrap(), UserBookmarkEntityType::W2lTemplate); + assert_eq!(UserBookmarkEntityType::from_str("w2l_result").unwrap(), UserBookmarkEntityType::W2lResult); + assert_eq!(UserBookmarkEntityType::from_str("media_file").unwrap(), UserBookmarkEntityType::MediaFile); + assert_eq!(UserBookmarkEntityType::from_str("voice_conversion_model").unwrap(), UserBookmarkEntityType::VoiceConversionModel); + assert_eq!(UserBookmarkEntityType::from_str("zs_voice").unwrap(), UserBookmarkEntityType::ZsVoice); + assert!(UserBookmarkEntityType::from_str("foo").is_err()); + } + } +} diff --git a/crates/_types/enums/src/common/visibility.rs b/crates/_types/enums/src/common/visibility.rs index f7240ca332..7e9c0341f3 100644 --- a/crates/_types/enums/src/common/visibility.rs +++ b/crates/_types/enums/src/common/visibility.rs @@ -15,9 +15,10 @@ use strum::EnumIter; /// See also: https://docs.rs/sqlx/0.4.0-beta.1/sqlx/trait.Type.html /// /// *DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY!* -/// + +use utoipa::ToSchema; #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type)] +#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type,ToSchema)] #[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum Visibility { diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 3fee6d8cfa..3ef6dd4af7 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal # (none) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } @@ -26,7 +29,6 @@ serde_derive = "1.0.125" # MySql Database sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # Test serialization diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c3c9ea1c73..581f8c65de 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -13,6 +13,13 @@ name = "tokens" path = "src/lib.rs" [dependencies] + +# Internal +# (none) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" @@ -22,9 +29,8 @@ rand = "0.8.5" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } -utoipa = { version = "4", features = ["actix_extras"] } +utoipa = { version = "4", features = ["chrono","actix_extras"] } [dev-dependencies] diff --git a/crates/_types/tokens/src/prefixes.rs b/crates/_types/tokens/src/prefixes.rs index 6e24a9802b..382492bae4 100644 --- a/crates/_types/tokens/src/prefixes.rs +++ b/crates/_types/tokens/src/prefixes.rs @@ -27,7 +27,6 @@ pub(crate) enum TokenPrefix { Comment, DownloadJob, EmailSenderJob, - Favorite, InferenceJob, MediaFile, MediaUpload, @@ -36,6 +35,7 @@ pub(crate) enum TokenPrefix { PasswordReset, TtsRenderTask, // NB: aichatbot / sqlite User, + UserBookmark, UserSession, VoiceConversionModel, VoiceConversionResult, @@ -94,7 +94,6 @@ impl PrefixGenerator for TokenPrefix { Self::DownloadJob => "jdown_", // NB: Previously "JGUP:" Self::EmailSenderJob => "email_job_", Self::InferenceJob => "jinf_", - Self::Favorite => "fav_", Self::MediaFile => "m_", Self::MediaUpload => "mu_", Self::ModelWeight => "weight_", @@ -102,6 +101,7 @@ impl PrefixGenerator for TokenPrefix { Self::PasswordReset => "pw_reset_", Self::TtsRenderTask => "tts_task_", Self::User => "user_", // NB: Previously "U:" + Self::UserBookmark => "ub_", Self::UserSession => "session_", Self::VoiceConversionModel => "vcm_", Self::VoiceConversionResult => "vcr_", diff --git a/crates/_types/tokens/src/tokens/mod.rs b/crates/_types/tokens/src/tokens/mod.rs index e75f732b55..b7c4eed592 100644 --- a/crates/_types/tokens/src/tokens/mod.rs +++ b/crates/_types/tokens/src/tokens/mod.rs @@ -34,4 +34,4 @@ pub mod w2l_templates; pub mod zs_voice_dataset_samples; pub mod zs_voice_datasets; pub mod zs_voices; -pub mod favorites; +pub mod user_bookmarks; diff --git a/crates/_types/tokens/src/tokens/model_weights.rs b/crates/_types/tokens/src/tokens/model_weights.rs index ba29c1f8fb..a63f2d9028 100644 --- a/crates/_types/tokens/src/tokens/model_weights.rs +++ b/crates/_types/tokens/src/tokens/model_weights.rs @@ -5,8 +5,9 @@ use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for the "model_weights" table. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct ModelWeightToken(pub String); diff --git a/crates/_types/tokens/src/tokens/favorites.rs b/crates/_types/tokens/src/tokens/user_bookmarks.rs similarity index 59% rename from crates/_types/tokens/src/tokens/favorites.rs rename to crates/_types/tokens/src/tokens/user_bookmarks.rs index 78ecb7e0b9..ac91dc079d 100644 --- a/crates/_types/tokens/src/tokens/favorites.rs +++ b/crates/_types/tokens/src/tokens/user_bookmarks.rs @@ -8,7 +8,7 @@ use crate::prefixes::TokenPrefix; /// The primary key for Audit Logs. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] -pub struct FavoriteToken(pub String); +pub struct UserBookmarkToken(pub String); -impl_string_token!(FavoriteToken); -impl_crockford_generator!(FavoriteToken, 32usize, TokenPrefix::Favorite, CrockfordLower); +impl_string_token!(UserBookmarkToken); +impl_crockford_generator!(UserBookmarkToken, 32usize, TokenPrefix::UserBookmark, CrockfordLower); diff --git a/crates/_types/tokens/src/tokens/users.rs b/crates/_types/tokens/src/tokens/users.rs index 82438cc81c..094ebd4a31 100644 --- a/crates/_types/tokens/src/tokens/users.rs +++ b/crates/_types/tokens/src/tokens/users.rs @@ -4,9 +4,9 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; - +use utoipa::ToSchema; /// The primary key for users. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize, Default)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize, Default,ToSchema)] #[sqlx(transparent)] pub struct UserToken(pub String); diff --git a/crates/_types/tokens/src/tokens/weight_uploads.rs b/crates/_types/tokens/src/tokens/weight_uploads.rs new file mode 100644 index 0000000000..cd7c046fc8 --- /dev/null +++ b/crates/_types/tokens/src/tokens/weight_uploads.rs @@ -0,0 +1,14 @@ +use std::fmt::Debug; + +use serde::Deserialize; +use serde::Serialize; + +use crate::prefixes::TokenPrefix; + +/// The primary key for user weight uploads +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[sqlx(transparent)] +pub struct WeightUploadToken(pub String); + +impl_string_token!(WeightUploadToken); +impl_crockford_generator!(WeightUploadToken, 32usize, TokenPrefix::WeightUpload, CrockfordLower); \ No newline at end of file diff --git a/crates/cli/database_backup/Cargo.toml b/crates/cli/database_backup/Cargo.toml index 691890dfae..02bd043b23 100644 --- a/crates/cli/database_backup/Cargo.toml +++ b/crates/cli/database_backup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/database_cleanup/Cargo.toml b/crates/cli/database_cleanup/Cargo.toml index 1600fd11f6..16485809b8 100644 --- a/crates/cli/database_cleanup/Cargo.toml +++ b/crates/cli/database_cleanup/Cargo.toml @@ -18,6 +18,9 @@ path = "src/main.rs" container_common = { path = "../../lib/deprecated/container_common" } easyenv = { path = "../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" # TODO/NB: Commenting out clap4 as it prevents windows builds @@ -28,4 +31,3 @@ log = "0.4.14" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index c07d0431ac..a771241cf5 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -28,18 +28,20 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 7dfaf13a32..558478772e 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -8,6 +8,7 @@ use errors::AnyhowResult; use crate::bucket_clients::get_bucket_clients; use crate::cli_args::parse_cli_args; use crate::seeding::model_weights::seed_weights; +use crate::seeding::seed_media_files::seed_media_files; use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; use crate::seeding::voice_conversion::seed_voice_conversion; @@ -52,11 +53,11 @@ pub async fn main() -> AnyhowResult<()> { } seed_user_accounts(&pool).await?; + seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?; seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?; seed_voice_conversion(&pool).await?; seed_weights(&pool).await?; - seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index cb462d17c9..c1100805b6 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -1,5 +1,6 @@ +pub mod model_weights; +pub mod seed_media_files; pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; -pub mod zero_shot_tts; -pub mod model_weights; \ No newline at end of file +pub mod zero_shot_tts; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index de9bcd90a9..8860d323c7 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -1,20 +1,355 @@ +use std::vec; + use log::info; +use sqlx::{MySql, Pool}; -use tokens::tokens::users::UserToken; -use sqlx::{ MySql, Pool }; use enums::by_table::model_weights::{ - weights_types::WeightsType, weights_category::WeightsCategory, + weights_types::WeightsType, }; -use errors::{ anyhow, AnyhowResult }; +use enums::common::visibility::Visibility; +use errors::{anyhow, AnyhowResult}; +use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs}; +use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; use crate::seeding::users::HANASHI_USERNAME; -use mysql_queries::queries::model_weights::create_weight::{ create_weight, CreateModelWeightsArgs }; -use tokens::tokens::model_weights::ModelWeightToken; -use enums::common::visibility::Visibility; -use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username; +pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserToken) -> AnyhowResult<()> { + let sd1_5_markdown_description = r#" +# Dragonfruit AI Models Update and Workflow +## Last Update for Dragonfruit Models +- **Note:** This might be the last update for a while, focusing next on a realistic merge (WIP). +- **Future Plans:** Possible update to Z.5D v2 after completing the realistic merge. + +## New Models Announcement +- **Upcoming Models:** Separate from the Dragonfruit branch, so stay tuned! + +## DragonfruitGT v1 (Z v2) +- **Focus:** More fantasy, richer color range, improved landscapes, more masculine men. +- **Alternative:** Use DragonfruitOG v2.0 for thicker lines, simpler colors. + +## GT v1.0 (Z v2.0) Features +- Improved backgrounds +- Deeper contrast and color +- Bolder lines and more detail + +### Settings for GT +- Steps: 30-50 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +## Workflow for Images +1. **BerryMix VAE:** Ensure correct color. +2. **Adetailer Extension:** Improve faces, bodies, and hands. Also available on Civitai and Huggingface. +3. **Hires Fix:** Denoising strength 0.3 - 0.35, using upscaler "x_NMKD-Superscale-SP_178000_G". +4. **Img2img Transfer:** Use denoising strength 0.3-0.35 with the same prompt and settings. Adjust height after hires. +5. **Second Img2img Transfer:** Low denoising strength 0.1-3.0. Use SD upscale script with "RealESRGAN_x4plus_anime_6B" sampler for size increase. + +## DragonfruitZ.5D v1.0 +- **Style:** 2.5D model, merging DragonfruitZ with realistic elements. +- **Features:** Skin color variety, masculine men, landscapes, fantasy, NSFW content, detailed faces. + +### Settings for Z.5D +- Steps: 30 +- Clip skip: 2 +- Upscaler: Euler a +- Extensions: Adetailer, VAE: berrysmix + +## DragonfruitOG v2.0 +- **Refinement:** Of the original Dragonfruit model. +- **Style:** Bold, clean lines and bold colors with decreased detail for simplicity. +- **Improvements:** Teeth, skin color variety, landscapes, fantasy elements, NSFW content. + +## DragonfruitZ v1.0 +- **Alternative Version:** More fantasy and color range, better landscapes, more masculine men. +- **Suggestion:** Use Dragonfruit (Unisex) v2 for thicker lines and simpler colors. + +## Dragonfruit OG v1.0 +- **Focus:** More unisex, suitable for creating male characters. +- **Personal Note:** First merge, aimed at 2D, flat, colorful, thick lined styles. + +### Personal Settings +- Steps: 20-30 +- Clip skip: 2 +- Upscaler: Euler a / DPM++ 2M SDE Karras +- Extensions: Adetailer, VAE: berrysmix + +*Thank you for your continued support and feedback!* +"#; + + let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + // create a loop that loops from 1 to 100 + for i in 1..=100 { + // create a new weight + + let weights_category = match i { + 1..=20 => WeightsCategory::VoiceConversion, + 21..=40 => WeightsCategory::TextToSpeech, + 41..=60 => WeightsCategory::ImageGeneration, + 61..=80 => WeightsCategory::ImageGeneration, + 81..=100 => WeightsCategory::ImageGeneration, + _ => WeightsCategory::ImageGeneration + }; + + let weights_types = match i { + 1..=20 => WeightsType::RvcV2, + 21..=40 => WeightsType::HifiganTacotron2, + 41..=60 => WeightsType::StableDiffusion15, + 61..=80 => WeightsType::StableDiffusionXL, + 81..=100 => WeightsType::LoRA, + _ => WeightsType::LoRA + }; + + let mut model_weight_token; + let mut title; + let mut description; + let mut description_rendered_html; + let mut original_filename; + let mut original_download_url; + let mut thumbnail_token; + + let mut private_bucket_hash:String = "".to_string(); + let mut private_bucket_prefix; + let mut private_bucket_extension; + + let mut cached_user_ratings_total_count; + let mut cached_user_ratings_positive_count; + let mut cached_user_ratings_negative_count; + let mut cached_user_ratings_ratio; + let mut version; + + match i { + 1..=20 => { + // voice conversion model + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding RVCv2 model {}", i); + }, + 21..=40 =>{ + // TTS + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + let tts_markdown_description = r#" + # HifiGAN Tacotron2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Goku MOS 4 + "#; + + description_rendered_html = tts_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("tt2"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding TT2 model {}", i); + }, + 41..=60 => { + // SD 1.5 + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("dragonfruitGTv1: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = sd1_5_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; //sd1_5_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sd15"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SD15 model {}", i); + }, + 61..=80 => { + // SD XL + let sdxl_markdown_description = r#" + # Special Edition Release Announcement + + ## What's New? + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + + ## Tips for Use + 1. **Simple Prompts:** Avoid adding words like "detailed, realistic..." or specifying a specific artist. Simply describe what you want to see. + 2. **CFG Scale:** A range between 3 and 8.5 is effective (recommended: 7). + 3. **Minimum Steps:** Ensure at least 36 steps for optimal results. + 4. **Compatibility:** + - Works excellently with Hires fix. + - Great results with unaestheticXLv31 embedding, although sometimes better without it. + 5. **Sampling Method:** Either DPM++ 2M SDE Karras or DPM++ 2M Karras. + + Have fun! 😊 + "#; + let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("SDXL_Niji_Special Edition: {}", i); + description = format!("Description Number {}:", i); + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("sdxl"); + + description_rendered_html = sdxl_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding SDXL model {}", i); + }, + 81..=100 => { + // LoRA + let lora_markdown_description = r#" + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - LoRA Gawr Gura + "#; + + let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg"; + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura LoRA: {}", i); + + description = format!("Description Number {}:", i); + + description_rendered_html = lora_markdown_description; + + original_filename = format!("filename{}.txt", i); + original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; // lora_image_token; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("loRA"); + + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + println!("Seeding LoRA model {}", i); + }, + _ => { + // We went out of range so have more gwar gura + let rvc_markdown_description = r#" + # RVC v2 + - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned. + - Gawr Gura MOS 4 + "#; + + model_weight_token = ModelWeightToken(format!("{}", i)); + title = format!("Gawr Gura: {}", i); + description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); + description_rendered_html = rvc_markdown_description; + + original_filename = format!("gura_gwar{}.safetensors", i); + original_download_url = format!("www.google.ca"); + + // NOTE TO SELF this will be a join for later on for now they can use this. + thumbnail_token = ""; + + // private_bucket_hash = format!("bucket_hash{}", i); + private_bucket_prefix = format!("_fake"); + private_bucket_extension = format!("rvcV2"); + cached_user_ratings_total_count = i; + cached_user_ratings_positive_count = i; + cached_user_ratings_negative_count = i; + cached_user_ratings_ratio = i as u32 / 100; + version = i as i32; + + println!("Out of range"); + } + } + + let args = CreateModelWeightsArgs { + token: &model_weight_token, // replace with actual ModelWeightToken + weights_type: weights_types, // replace with actual WeightsType + weights_category, // replace with actual WeightsCategory + title, + maybe_thumbnail_token: Some(thumbnail_token.to_string()), + description_markdown: description, + description_rendered_html: description_rendered_html.to_string(), + creator_user_token: Some(&user_token), // replace with actual UserToken + creator_ip_address: "192.168.1.1", + creator_set_visibility: Visibility::Public, + maybe_last_update_user_token: Some(user_token.to_string()), + original_download_url: Some(original_download_url), + original_filename: Some(original_filename), + file_size_bytes: 1024, + file_checksum_sha2: "checksum1".to_string(), + private_bucket_hash: "bucket_hash1".to_string(), + maybe_private_bucket_prefix: Some(private_bucket_prefix), + maybe_private_bucket_extension: Some(private_bucket_extension), + cached_user_ratings_total_count, + cached_user_ratings_positive_count, + cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32), + version, + mysql_pool: &mysql_pool, // replace with actual MySqlPool + }; + + create_weight(args).await; + } + Ok(()) +} pub async fn seed_weights_for_user_token( mysql_pool: &Pool, user_token: UserToken @@ -165,16 +500,9 @@ pub async fn seed_weights_for_user_token( } Ok(()) } -pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { - info!("Seeding weights..."); - let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { - None => { - return Err(anyhow!("could not find user hanashi")); - } - Some(token) => token, - }; +pub async fn original_seed_weights(mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { let model_weight_token1 = ModelWeightToken("1".to_string()); let creator_token1 = UserToken("creatorToken1".to_string()); @@ -327,9 +655,22 @@ pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { create_weight(model_weights_arg).await?; } - // adds this to HANASHI - get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await?; - seed_weights_for_user_token(mysql_pool, user_token).await?; + Ok(()) + +} +pub async fn seed_weights(mysql_pool: &Pool) -> AnyhowResult<()> { + info!("Seeding weights..."); + + let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { + None => { + return Err(anyhow!("could not find user hanashi")); + } + Some(token) => token, + }; + + //original_seed_weights(mysql_pool,user_token).await?; + //seed_weights_for_user_token(mysql_pool, user_token).await?; + seed_weights_for_paging(mysql_pool,user_token).await?; Ok(()) } diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs new file mode 100644 index 0000000000..1e3f2c3c80 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -0,0 +1,224 @@ +use std::path::Path; + +use log::{info, warn}; +use sqlx::{MySql, Pool}; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use filesys::file_read_bytes::file_read_bytes; +use filesys::file_size::file_size; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs}; +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { + info!("Seeding media files..."); + + // NB: This is idempotent and will only install the media files once. + // Delete the records if you want to re-seed. + let media_files = [ + // Audio + (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), + (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), + (HANASHI_USER_TOKEN, "m_sq2vhrrmmbr079afk21543aa2jg53g", MediaFileType::Audio, "media/audio/voice-samples/brandon2.wav"), + (HANASHI_USER_TOKEN, "m_s61a8g3jnddr3xdg3gctsjxxk9jwja", MediaFileType::Audio, "media/audio/voice-samples/elon.wav"), + (HANASHI_USER_TOKEN, "m_e59fh75g76pvebx4qe4r4aymd93q30", MediaFileType::Audio, "media/audio/voice-samples/goku.wav"), + (HANASHI_USER_TOKEN, "m_ttq0ym8ht4v0z87zqz27fhadp0743c", MediaFileType::Audio, "media/audio/voice-samples/hillary.wav"), + (HANASHI_USER_TOKEN, "m_300bv6f141hv553fjf287p5gb76k37", MediaFileType::Audio, "media/audio/voice-samples/mario_impersonator.wav"), + (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), + (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), + (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/image/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/image/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/image/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/image/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/image/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/image/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/image/face-animator-inputs/zelda.jpg"), + // Video + (HANASHI_USER_TOKEN, "m_gedarcfdkxx3zv8zz2wmwsnc1r8jbp", MediaFileType::Video, "media/video/storyteller-studio-renders/girl-rotating-room.mp4"), + (HANASHI_USER_TOKEN, "m_3s74yyd4jwrjdp17yx1694n29jv5gr", MediaFileType::Video, "media/video/storyteller-studio-renders/hanashi-run.mp4"), + (HANASHI_USER_TOKEN, "m_p6d64gm1wawz25bmdahfded3366b9f", MediaFileType::Video, "media/video/storyteller-studio-renders/mocap-kiss.mp4"), + (HANASHI_USER_TOKEN, "m_szp2c77z6je965k44vc22xn9432a88", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-girl.mp4"), + (HANASHI_USER_TOKEN, "m_mdwspz81zymw9hvmb1d3jm830t25xe", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-mocap-m.mp4"), + (HANASHI_USER_TOKEN, "m_prcs6xkm8phs5y4a2dfwemwjy6n6z3", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-girl.mp4"), + (HANASHI_USER_TOKEN, "m_5j889dzw08vn9pbrscx1sncff43ekp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-jww.mp4"), + (HANASHI_USER_TOKEN, "m_fwm2w6wbssez274ayz6x8ta0h1a9qp", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pascal.mp4"), + (HANASHI_USER_TOKEN, "m_yqqkrt2srep6a35xg78114jn9epwg1", MediaFileType::Video, "media/video/storyteller-studio-renders/rotating-run-pop.mp4"), + (HANASHI_USER_TOKEN, "m_m4r043kez14pm6vaamsecpz8fafh7g", MediaFileType::Video, "media/video/video-style-transfer/man-explaining.mp4"), + (HANASHI_USER_TOKEN, "m_81gq3hygqvrahg6ncxbtcpjap4f9nv", MediaFileType::Video, "media/video/video-style-transfer/woman-gradient-background.mp4"), + (HANASHI_USER_TOKEN, "m_dp9ykj3yasjx69g2fx2bpd26wwjz0f", MediaFileType::Video, "media/video/video-style-transfer/woman-red-hair.mp4"), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + for (user_token, media_file_token, media_file_type, subdirectory_path) in media_files { + let user_token = UserToken::new_from_str(user_token); + let media_file_token = MediaFileToken::new_from_str(media_file_token); + + let mut media_file_path = seed_tool_data_root.clone(); + media_file_path.push(subdirectory_path); + + let result = seed_model( + &mysql_pool, + &media_file_token, + &user_token, + media_file_type, + &media_file_path, + maybe_bucket_clients, + ).await; + + match result { + Ok(_) => info!("Seeded {:?}", media_file_path), + Err(err) => warn!(r#" + Could not seed media file {} , {:?} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the media file on a previous invocation!) + "#, media_file_token, subdirectory_path, err), + } + } + + Ok(()) +} + +async fn seed_model( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, + user_token: &UserToken, + media_file_type: MediaFileType, + media_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients>, +) -> AnyhowResult<()> { + info!("Seeding Media file {:?} ...", media_file_path); + + let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut maybe_bucket_prefix = None; + let mut maybe_bucket_extension = None; + + if media_file_exists(mysql_pool, media_file_token).await? { + info!("Media file already seeded: {:?}", media_file_token); + return Ok(()) + } + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(media_file_path, bucket_clients).await?; + bucket_hash = bucket_details.bucket_hash; + maybe_bucket_prefix = bucket_details.maybe_bucket_prefix; + maybe_bucket_extension = bucket_details.maybe_bucket_extension; + } + + let maybe_mime_type = get_mimetype_for_file(media_file_path)?; + let file_size_bytes = file_size(media_file_path)?; + let sha256_checksum = sha256_hash_file(media_file_path)?; + + let maybe_filename = media_file_path + .file_name() + .map(|name| name.to_str()) + .flatten(); + + insert_media_file_from_cli_tool(InsertArgs { + pool: &mysql_pool, + maybe_use_apriori_media_token: Some(media_file_token), + media_file_type, + maybe_mime_type, + file_size_bytes, + sha256_checksum: &sha256_checksum, + maybe_origin_filename: maybe_filename, + maybe_creator_user_token: Some(user_token), + creator_set_visibility: Visibility::Public, + public_bucket_directory_hash: &bucket_hash, + maybe_public_bucket_prefix: maybe_bucket_prefix.as_deref(), + maybe_public_bucket_extension: maybe_bucket_extension.as_deref(), + }).await?; + + Ok(()) +} + +async fn media_file_exists( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, +) -> AnyhowResult { + + const CAN_SEE_DELETED : bool = true; + + let maybe_file = get_media_file( + &media_file_token, + CAN_SEE_DELETED, + mysql_pool + ).await?; + + Ok(maybe_file.is_some()) +} + +struct BucketDetails { + bucket_hash: String, + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +async fn seed_file_to_bucket( + media_file_path: &Path, + bucket_clients: &BucketClients, +) -> AnyhowResult { + + info!("Uploading media file {:?} ...", media_file_path); + + let maybe_bucket_prefix = Some("fakeyou_"); + + let maybe_bucket_extension = media_file_path + .extension() + .map(|extension| extension.to_str()) + .flatten(); + + let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension); + + let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf()); + + info!("Reading media file: {:?}", media_file_path); + + let bytes = file_read_bytes(media_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + info!("Uploading media file to bucket path: {:?}", bucket_path); + + let _r = bucket_clients.public.upload_file_with_content_type( + &bucket_path, + bytes.as_ref(), + mimetype + ).await?; + + Ok(BucketDetails { + bucket_hash: bucket_location.get_object_hash().to_string(), + maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()), + maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()), + }) +} + +// Temporary "test" to generate random tokens. +// #[cfg(test)] +// mod tests { +// use tokens::tokens::media_files::MediaFileToken; +// +// #[test] +// fn test() { +// // Print some randomly sampled tokens +// for _i in 0..50 { +// let token = MediaFileToken::generate(); +// println!("Token: {}", token); +// } +// // Deliberately fail the test to get the output +// assert_eq!(1, 2); +// } +// } diff --git a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs index d01c359d13..196c36b6ef 100644 --- a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs @@ -1,7 +1,7 @@ use log::info; use sqlx::{MySql, Pool}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 5831756152..47d58a89c0 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -25,16 +25,18 @@ mimetypes = { path = "../../lib/files/mimetypes" } mysql_queries = { path = "../../lib/mysql_queries" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated log = "0.4.14" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 7a72bacf56..3928071c5d 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -3,7 +3,7 @@ use std::time::Duration; use log::info; use sqlx::mysql::MySqlPoolOptions; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; use enums::by_table::media_files::media_file_type::MediaFileType; @@ -119,10 +119,12 @@ pub async fn main() -> AnyhowResult<()> { let (media_file_token, _id) = insert_media_file_from_cli_tool(InsertArgs { pool: &pool, + maybe_use_apriori_media_token: None, media_file_type, maybe_mime_type, file_size_bytes: 0, // TODO sha256_checksum: "", // TODO + maybe_origin_filename: None, maybe_creator_user_token: None, creator_set_visibility, public_bucket_directory_hash: public_upload_path.get_object_hash(), diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index f6ca014556..7361ad74b6 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../lib/mysql_queries" } storyteller_root = { path = "../../lib/storyteller_root" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.14" clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated @@ -31,7 +34,7 @@ strum = { version = "0.25", features = ["derive"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } @@ -41,7 +44,6 @@ elasticsearch = "8.5.0-alpha.1" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/kubectl-pod-cleanup/Cargo.toml b/crates/cli/kubectl-pod-cleanup/Cargo.toml index a0bb79e162..73d842a561 100644 --- a/crates/cli/kubectl-pod-cleanup/Cargo.toml +++ b/crates/cli/kubectl-pod-cleanup/Cargo.toml @@ -18,12 +18,14 @@ config = { path = "../../lib/deprecated/config" } easyenv = { path = "../../lib/easyenv" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile yytimes) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External log = "0.4.20" once_cell = "1.18.0" rand = "0.8.5" regex = "1.10.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # Async Runtime # NB: Not using Tokio here since everything is blocking so far. diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml index f7af132bb6..eb08a399a3 100644 --- a/crates/cli/lru-delete-models/Cargo.toml +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -17,10 +17,12 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } logging = { path = "../../lib/logging" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = { version = "4.4.7", features = ["derive"] } walkdir = "2.4.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index 99753086ef..36284d1f33 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -18,7 +18,9 @@ path = "src/main.rs" errors = { path = "../../lib/errors" } web_scrapers = { path = "../../lib/web_scrapers" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated -tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/actix_helpers/Cargo.toml b/crates/lib/actix_helpers/Cargo.toml index 89c0cbf5d9..4f57ff1785 100644 --- a/crates/lib/actix_helpers/Cargo.toml +++ b/crates/lib/actix_helpers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # Actix actix-http = "3.4.0" actix-rt = "2.9.0" @@ -34,7 +37,6 @@ futures-util = "0.3.29" # Serialization serde = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index ee5435b17a..2cd5d20198 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal buckets = { path = "../../../_types/buckets" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" @@ -27,5 +30,4 @@ rust-s3 = { version = "0.33.0" } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/lib/api_clients/google_drive_common/Cargo.toml b/crates/lib/api_clients/google_drive_common/Cargo.toml index d15ae569df..75e1d58c0e 100644 --- a/crates/lib/api_clients/google_drive_common/Cargo.toml +++ b/crates/lib/api_clients/google_drive_common/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } subprocess_common = { path = "../../subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" log = "0.4.14" tempdir = "0.3.7" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/twitch_common/Cargo.toml b/crates/lib/api_clients/twitch_common/Cargo.toml index b377c34ee2..9d1fb2eaa8 100644 --- a/crates/lib/api_clients/twitch_common/Cargo.toml +++ b/crates/lib/api_clients/twitch_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../../deprecated/container_common" } easyenv = { path = "../../easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" once_cell = "1.18.0" @@ -31,4 +34,3 @@ twitch_oauth2 = "0.6.0-rc.2" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/bootstrap/Cargo.toml b/crates/lib/bootstrap/Cargo.toml index 3baecb1333..e3cfd8376e 100644 --- a/crates/lib/bootstrap/Cargo.toml +++ b/crates/lib/bootstrap/Cargo.toml @@ -19,11 +19,13 @@ easyenv = { path = "../easyenv" } errors = { path = "../errors" } server_environment = { path = "../server_environment" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/caching/memory_caching/Cargo.toml b/crates/lib/caching/memory_caching/Cargo.toml index 06a8da7dfc..22b6ee7736 100644 --- a/crates/lib/caching/memory_caching/Cargo.toml +++ b/crates/lib/caching/memory_caching/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" # In-memory caching lru_time_cache = "0.11.11" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/caching/redis_caching/Cargo.toml b/crates/lib/caching/redis_caching/Cargo.toml index 9f31f5bdbd..a36bc100f1 100644 --- a/crates/lib/caching/redis_caching/Cargo.toml +++ b/crates/lib/caching/redis_caching/Cargo.toml @@ -16,6 +16,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Redis Database r2d2_redis = "0.14.0" @@ -23,7 +26,6 @@ r2d2_redis = "0.14.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/collections/Cargo.toml b/crates/lib/collections/Cargo.toml index f74421ec5a..080bd36abb 100644 --- a/crates/lib/collections/Cargo.toml +++ b/crates/lib/collections/Cargo.toml @@ -16,6 +16,8 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/concurrency/Cargo.toml b/crates/lib/concurrency/Cargo.toml index c9511ffd58..4d9e02d580 100644 --- a/crates/lib/concurrency/Cargo.toml +++ b/crates/lib/concurrency/Cargo.toml @@ -14,6 +14,9 @@ path = "src/lib.rs" [dependencies] # Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/cookies/Cargo.toml b/crates/lib/cookies/Cargo.toml index 8abda155b4..6bd0b45ec9 100644 --- a/crates/lib/cookies/Cargo.toml +++ b/crates/lib/cookies/Cargo.toml @@ -17,11 +17,13 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External hmac = "0.10.0" # jwt signing jwt = "0.13.0" sha2 = "0.9.5" # hmac signing -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/crockford/Cargo.toml b/crates/lib/crockford/Cargo.toml index 45fa6a9f04..526e91fb99 100644 --- a/crates/lib/crockford/Cargo.toml +++ b/crates/lib/crockford/Cargo.toml @@ -15,8 +15,14 @@ name = "crockford" path = "src/lib.rs" [dependencies] -rand = "0.8.5" +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +rand = "0.8.5" + [dev-dependencies] # None yet diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 7b69245835..6bb08d5c56 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,9 +13,15 @@ name = "datetimes" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External chrono = { version = "0.4.22", features = ["serde"] } once_cell = "1.18.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # (none) \ No newline at end of file diff --git a/crates/lib/deprecated/config/Cargo.toml b/crates/lib/deprecated/config/Cargo.toml index 6142b3221e..80e5232bb5 100644 --- a/crates/lib/deprecated/config/Cargo.toml +++ b/crates/lib/deprecated/config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" # Internal easyenv = { path = "../../../lib/easyenv" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" url = "2.4.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/config/src/bad_urls.rs b/crates/lib/deprecated/config/src/bad_urls.rs index 13be46019b..280ed1d3bb 100644 --- a/crates/lib/deprecated/config/src/bad_urls.rs +++ b/crates/lib/deprecated/config/src/bad_urls.rs @@ -3,11 +3,16 @@ use crate::is_bad_download_url::is_bad_download_url; /// Reports if the URL is inappropriate for downloading as a TTS model, vocoder, etc. +/// DEPRECATE: pub fn is_bad_tts_model_download_url(url: &str) -> anyhow::Result { // TODO: Just use this. is_bad_download_url(url) } +pub fn is_bad_model_weights_download_url(url: &str) -> anyhow::Result { + is_bad_download_url(url) +} + #[cfg(test)] mod tests { use crate::bad_urls::is_bad_tts_model_download_url; diff --git a/crates/lib/deprecated/container_common/Cargo.toml b/crates/lib/deprecated/container_common/Cargo.toml index 7d0e9cb9dd..32fbc9f9f2 100644 --- a/crates/lib/deprecated/container_common/Cargo.toml +++ b/crates/lib/deprecated/container_common/Cargo.toml @@ -21,10 +21,12 @@ rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } log = "0.4.14" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # Serialization serde = { version = "1.0.125", features = ["derive"] } toml = "0.5.8" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 4b7259c477..0a4e4e2159 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" errors = { path = "../../errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -39,7 +42,6 @@ language-tags = "0.3.2" # Parsing single tags richly serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] actix-rt = "2.9.0" diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index 8bfc0d1044..b3e5c1bfee 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -16,11 +16,13 @@ path = "src/lib.rs" # Internal storyteller_root = { path = "../storyteller_root" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.14" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs index 1ae2cc18ca..30055f1774 100644 --- a/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs +++ b/crates/lib/elasticsearch_schema/src/searches/search_tts_models.rs @@ -27,7 +27,7 @@ pub async fn search_tts_models( let mut response_json = search_response.json::().await?; - let mut hits = response_json.get_mut("hits") + let hits = response_json.get_mut("hits") .map(|hits| hits.take()); let hits = hits.map(|mut hits| { diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index feb580c8c2..020ced75b9 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -10,6 +10,12 @@ path = "src/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Internal errors = { path = "../errors" } -lettre = "0.11.2" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External +lettre = "0.11.2" + diff --git a/crates/lib/errors/Cargo.toml b/crates/lib/errors/Cargo.toml index 6268214f7d..ffa8549b87 100644 --- a/crates/lib/errors/Cargo.toml +++ b/crates/lib/errors/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Consider the dependencies here wisely. # Ideally we have almost nothing as a dependency of this crate. anyhow = "1.0.75" + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] diff --git a/crates/lib/files/archives/Cargo.toml b/crates/lib/files/archives/Cargo.toml index d260c780c6..4e02301f23 100644 --- a/crates/lib/files/archives/Cargo.toml +++ b/crates/lib/files/archives/Cargo.toml @@ -18,9 +18,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External zip = "0.6.6" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index bbd7dbf27f..19aae75a68 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -16,11 +16,14 @@ path = "src/lib.rs" [dependencies] # Internal errors = { path = "../../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } -log = "0.4.20" # External # KEEP THIS TO A MINIMUM OR NIL. +log = "0.4.20" + [dev-dependencies] # None yet diff --git a/crates/lib/files/hashing/Cargo.toml b/crates/lib/files/hashing/Cargo.toml index adb00e6168..62d47fa10b 100644 --- a/crates/lib/files/hashing/Cargo.toml +++ b/crates/lib/files/hashing/Cargo.toml @@ -17,12 +17,14 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes data-encoding = "2.3.2" md-5 = "0.9.1" # DO NOT USE FOR SERIOUS CRYPTO (we use this for gravatar) ring = "0.17.0-alpha.10" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml index 3327f009b6..ddfe507191 100644 --- a/crates/lib/files/images/Cargo.toml +++ b/crates/lib/files/images/Cargo.toml @@ -19,9 +19,11 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External image = "0.24.1" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/files/media/Cargo.toml b/crates/lib/files/media/Cargo.toml index 47d371d0b8..02df59e277 100644 --- a/crates/lib/files/media/Cargo.toml +++ b/crates/lib/files/media/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal errors = { path = "../../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External log = "0.4.14" @@ -26,7 +29,5 @@ log = "0.4.14" # Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/ symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", rev = "e1a70095c4409478c1ded499fb8dd4afa540fb90", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [dev-dependencies] # None yet diff --git a/crates/lib/files/mimetypes/Cargo.toml b/crates/lib/files/mimetypes/Cargo.toml index 6fe34f2034..0e37ad2df4 100644 --- a/crates/lib/files/mimetypes/Cargo.toml +++ b/crates/lib/files/mimetypes/Cargo.toml @@ -14,9 +14,12 @@ name = "mimetypes" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External infer = "0.11" # Mimetype detection -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + [dev-dependencies] # None yet diff --git a/crates/lib/idempotency/Cargo.toml b/crates/lib/idempotency/Cargo.toml index 578404b0fb..1c927b73ab 100644 --- a/crates/lib/idempotency/Cargo.toml +++ b/crates/lib/idempotency/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # errors = { path = "../../lib/errors" } # (None yet) +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External uuid = { version = "0.8.2", features = ["v4"] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index 1e6767ea01..98ae1d11d1 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -22,9 +22,11 @@ container_common = { path = "../deprecated/container_common" } errors = { path = "../errors" } redis_common = { path = "../redis_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index bf708a7b84..779659367c 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -14,10 +14,16 @@ name = "logging" path = "src/lib.rs" [dependencies] +# Internal +# (None yet) + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External dotenv = "0.15.0" env_logger = "0.10.1" log = "0.4.20" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 4b09061ac0..228c472322 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -23,6 +23,9 @@ errors = { path = "../errors" } reusable_types = { path = "../../_types/reusable_types" } tokens = { path = "../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -40,9 +43,8 @@ serde_json = "1.0.108" toml = "0.5.8" serial_test = "2.0.0" -tokio = { version = "1.6.0", features = ["full"] } +tokio = { version = "1.34.0", features = ["full"] } rand = "0.8" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs index 55669b4422..8420295eb5 100644 --- a/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/email_sender_jobs/list_available_email_sender_jobs.rs @@ -167,7 +167,7 @@ FROM email_sender_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableEmailSenderJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs b/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs deleted file mode 100644 index 3a0007253f..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/favorite_entity_token.rs +++ /dev/null @@ -1,48 +0,0 @@ -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use tokens::tokens::media_files::MediaFileToken; -use tokens::tokens::tts_models::TtsModelToken; -use tokens::tokens::tts_results::TtsResultToken; -use tokens::tokens::users::UserToken; -use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; -use tokens::tokens::w2l_results::W2lResultToken; -use tokens::tokens::w2l_templates::W2lTemplateToken; -use tokens::tokens::zs_voices::ZsVoiceToken; - -pub enum FavoriteEntityToken { - User(UserToken), - TtsModel(TtsModelToken), - TtsResult(TtsResultToken), - W2lTemplate(W2lTemplateToken), - W2lResult(W2lResultToken), - MediaFile(MediaFileToken), - VoiceConversionModel(VoiceConversionModelToken), - ZsVoice(ZsVoiceToken), -} - -impl FavoriteEntityToken { - pub fn from_entity_type_and_token(entity_type: FavoriteEntityType, token: &str) -> Self { - match entity_type { - FavoriteEntityType::User => Self::User(UserToken::new_from_str(token)), - FavoriteEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), - FavoriteEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), - FavoriteEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), - FavoriteEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), - FavoriteEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), - FavoriteEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), - FavoriteEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), - } - } - - pub fn get_composite_keys(&self) -> (FavoriteEntityType, &str) { - match self { - FavoriteEntityToken::User(user_token) => (FavoriteEntityType::User, user_token.as_str()), - FavoriteEntityToken::TtsModel(tts_model_token) => (FavoriteEntityType::TtsModel, tts_model_token.as_str()), - FavoriteEntityToken::TtsResult(tts_result_token) => (FavoriteEntityType::TtsResult, tts_result_token.as_str()), - FavoriteEntityToken::W2lTemplate(w2l_template_token) => (FavoriteEntityType::W2lTemplate, w2l_template_token.as_str()), - FavoriteEntityToken::W2lResult(w2l_result_token) => (FavoriteEntityType::W2lResult, w2l_result_token.as_str()), - FavoriteEntityToken::MediaFile(media_file_token) => (FavoriteEntityType::MediaFile, media_file_token.as_str()), - FavoriteEntityToken::VoiceConversionModel(voice_conversion_model_token) => (FavoriteEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), - FavoriteEntityToken::ZsVoice(zs_voice_token) => (FavoriteEntityType::ZsVoice, zs_voice_token.as_str()), - } - } -} diff --git a/crates/lib/mysql_queries/src/queries/favorites/mod.rs b/crates/lib/mysql_queries/src/queries/favorites/mod.rs deleted file mode 100644 index 8cc8a3c7c5..0000000000 --- a/crates/lib/mysql_queries/src/queries/favorites/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod create_favorite; -pub mod delete_favorite; -pub mod favorite_entity_token; -pub mod get_favorite; -pub mod list_favorites_for_entity; -pub mod list_user_favorites; -pub mod list_user_favorites_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 4cb3b5fa7e..1f72d607cb 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -336,7 +336,7 @@ FROM generic_inference_jobs"#.to_string(); LIMIT ? "#); - let mut query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) + let query = sqlx::query_as::<_, AvailableInferenceJobRawInternal>(&query) .bind(args.is_debug_worker) .bind(args.num_records); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs index a1df90a95a..5a3ea2fb6d 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs @@ -20,7 +20,7 @@ pub async fn mark_generic_inference_job_failure( // statuses: "attempt_failed", "complete_failure", "dead" let mut next_status = "attempt_failed"; - let mut maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { + let maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { let mut reason = reason.trim().to_string(); reason.truncate(512); // Max length of column is 512 reason diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 247eb2b9c0..489ddd8391 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; + use chrono::{DateTime, Utc}; use log::warn; use sqlx::{MySql, MySqlPool}; diff --git a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs index ec94000555..8dc215e82f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs @@ -13,7 +13,7 @@ use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct MediaFile { pub token: MediaFileToken, diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index 43a543db7d..e829ba53fd 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -14,12 +14,18 @@ use tokens::tokens::users::UserToken; pub struct InsertArgs<'a> { pub pool: &'a MySqlPool, + /// If supplied, use this media token rather than generating a new one. + /// This is good to support idempotency in development and testing. + pub maybe_use_apriori_media_token: Option<&'a MediaFileToken>, + pub media_file_type: MediaFileType, pub maybe_mime_type: Option<&'a str>, pub file_size_bytes: u64, pub sha256_checksum: &'a str, - pub maybe_creator_user_token: Option, + pub maybe_origin_filename: Option<&'a str>, + + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_set_visibility: Visibility, pub public_bucket_directory_hash: &'a str, @@ -33,7 +39,9 @@ pub async fn insert_media_file_from_cli_tool( args: InsertArgs<'_> ) -> AnyhowResult<(MediaFileToken, u64)> { - let media_file_token = MediaFileToken::generate(); + let media_file_token = args.maybe_use_apriori_media_token + .map(|token| token.clone()) + .unwrap_or_else(|| MediaFileToken::generate()); let mut transaction = args.pool.begin().await?; @@ -58,6 +66,8 @@ SET file_size_bytes = ?, checksum_sha2 = ?, + maybe_origin_filename = ?, + public_bucket_directory_hash = ?, maybe_public_bucket_prefix = ?, maybe_public_bucket_extension = ?, @@ -77,6 +87,8 @@ SET args.file_size_bytes, args.sha256_checksum, + args.maybe_origin_filename, + args.public_bucket_directory_hash, args.maybe_public_bucket_prefix, args.maybe_public_bucket_extension, diff --git a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs index 10ea2024c4..8faa739b0f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/update_media_file.rs @@ -14,7 +14,7 @@ pub struct UpdateMediaFileArgs<'a> { pub async fn update_media_file(args: UpdateMediaFileArgs<'_>) -> AnyhowResult<()>{ // TODO: enforce checks for idempotency token - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; let query_result = sqlx::query!( r#" UPDATE media_files diff --git a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs index 043a3b81f3..667e3751f1 100644 --- a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; diff --git a/crates/lib/mysql_queries/src/queries/mod.rs b/crates/lib/mysql_queries/src/queries/mod.rs index 1acd2d7a89..39499425b4 100644 --- a/crates/lib/mysql_queries/src/queries/mod.rs +++ b/crates/lib/mysql_queries/src/queries/mod.rs @@ -3,7 +3,6 @@ pub mod audit_logs; pub mod billing; pub mod comments; pub mod email_sender_jobs; -pub mod favorites; pub mod generic_download; pub mod generic_inference; pub mod generic_synthetic_ids; @@ -18,6 +17,7 @@ pub mod trending_model_analytics; pub mod tts; pub mod twitch; pub mod users; +pub mod user_bookmarks; pub mod vocoder; pub mod voice_clone_requests; pub mod voice_conversion; diff --git a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs index 136751050d..845b7e60f1 100644 --- a/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_categories/list_categories_query_builder.rs @@ -184,7 +184,7 @@ impl ListCategoriesQueryBuilder { query = query.bind(model_type); } - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) @@ -239,7 +239,7 @@ LEFT OUTER JOIN users let mut query = "".to_string(); - if let Some(username) = self.scope_creator_user_token.as_deref() { + if let Some(_username) = self.scope_creator_user_token.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.creator_user_token = ?"); first_predicate_added = true; @@ -248,7 +248,7 @@ LEFT OUTER JOIN users } } - if let Some(model_type) = self.scope_model_type.as_deref() { + if let Some(_model_type) = self.scope_model_type.as_deref() { if !first_predicate_added { query.push_str(" WHERE model_categories.model_type = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs index d21f8a789e..4827edd907 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/create_weight.rs @@ -7,6 +7,7 @@ use enums::by_table::model_weights::{ use log::warn; use enums::common::visibility::Visibility; use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + pub struct CreateModelWeightsArgs<'a> { pub token: &'a ModelWeightToken, pub weights_type: WeightsType, diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs index 71c0e9f43a..158e529649 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_user.rs @@ -71,11 +71,7 @@ pub async fn list_weights_by_creator_username( let raw_weights: Vec = get_raw_weights_by_creator_username(&mut connection, creator_username, can_see_deleted).await?; let weights_records: Vec = map_to_weights(raw_weights).await; - let filtered_weights: Vec = weights_records.into_iter().filter(|weight| { - weight.creator_username == creator_username && weight.creator_set_visibility == Visibility::Public && can_see_deleted - }).collect(); - - Ok(filtered_weights) + Ok(weights_records) } async fn get_raw_weights_by_creator_username( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs index 69ccf3617a..d1d18302b9 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_query_builder.rs @@ -133,13 +133,13 @@ impl ListWeightsQueryBuilder { self } - pub fn weights_type(mut self, weights_type: WeightsType) -> Self { - self.weights_type = Some(weights_type); + pub fn weights_type(mut self, weights_type: Option) -> Self { + self.weights_type = weights_type; self } - pub fn weights_category(mut self, weights_category: WeightsCategory) -> Self { - self.weights_category = Some(weights_category); + pub fn weights_category(mut self, weights_category: Option) -> Self { + self.weights_category = weights_category; self } @@ -505,7 +505,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_type() { - let query_builder = ListWeightsQueryBuilder::new().weights_type(WeightsType::RvcV2); + let query_builder = ListWeightsQueryBuilder::new().weights_type(Some(WeightsType::RvcV2)); assert_eq!( &query_builder.build_predicates(), " WHERE model_weights.weights_type = ? \ @@ -520,7 +520,7 @@ mod tests { #[test] fn predicates_scoped_to_weights_category() { let query_builder = ListWeightsQueryBuilder::new().weights_category( - WeightsCategory::VoiceConversion + Some(WeightsCategory::VoiceConversion) ); println!("Query HERE! {}", &query_builder.build_predicates()); assert_eq!( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs index dca6d0cab6..3239e4173e 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/tests.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/tests.rs @@ -1,32 +1,32 @@ #[cfg(test)] mod tests { - use std::result; - use anyhow::Ok; - use enums::by_table::model_weights::{weights_types, weights_category}; - use sqlx::{MySqlPool, Acquire}; use rand::Rng; - - use tokio; use serial_test::serial; - use container_common::anyhow_result::AnyhowResult; + use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; - use config::shared_constants::{ DEFAULT_MYSQL_CONNECTION_STRING }; + use tokio; + + use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; + use container_common::anyhow_result::AnyhowResult; // common tests use enums::by_table::model_weights::{ - weights_types::WeightsType, weights_category::WeightsCategory, + weights_types::WeightsType, }; use enums::common::visibility::Visibility; - use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken }; + use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken}; - use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; use crate::queries::model_weights::create_weight::create_weight; + use crate::queries::model_weights::create_weight::CreateModelWeightsArgs; + use crate::queries::model_weights::delete_weights::{ + delete_weights_as_mod, + delete_weights_as_user, + undelete_weights_as_mod, + undelete_weights_as_user, + }; use crate::queries::model_weights::get_weight::get_weight_by_token; - - use crate::queries::model_weights::delete_weights::{ delete_weights_as_user, delete_weights_as_mod, undelete_weights_as_mod , undelete_weights_as_user}; use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username; - use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username; @@ -45,23 +45,19 @@ mod tests { pub async fn delete_all_weights_for_table(mysql_pool: &MySqlPool) -> AnyhowResult<()> { // write a query that deletes all weights - let _r = sqlx::query("DELETE FROM model_weights") - .execute(mysql_pool) - .await?; + let _r = sqlx::query("DELETE FROM model_weights").execute(mysql_pool).await?; Ok(()) } #[tokio::test] #[serial] async fn test_create_weights() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); let random_number: u32 = rng.gen(); let model_weight_token1 = ModelWeightToken(random_number.to_string()); - let creator_token1 = UserToken("creatorToken!1".to_string()); let args = CreateModelWeightsArgs { @@ -131,7 +127,6 @@ mod tests { #[tokio::test] #[serial] async fn test_delete_and_undelete_weights_user() -> AnyhowResult<()> { - let pool = setup().await; // create a random token for the model weight let mut rng = rand::thread_rng(); @@ -170,7 +165,7 @@ mod tests { create_weight(args).await?; delete_weights_as_user(&model_weight_token1, &pool).await?; - + let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -261,7 +256,6 @@ mod tests { } } - undelete_weights_as_mod(&model_weight_token1, &pool).await?; let result = get_weight_by_token(&model_weight_token1, true, &pool).await?; let result = result.unwrap(); @@ -285,35 +279,54 @@ mod tests { #[serial] async fn list_all_weights_by_user() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); let can_see_deleted = true; - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,5,&creator_username).await?; + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 5, + &creator_username + ).await?; - let weights_by_username = list_weights_by_creator_username(&pool, &creator_username, can_see_deleted).await?; + let weights_by_username = list_weights_by_creator_username( + &pool, + &creator_username, + can_see_deleted + ).await?; + for weight in weights_by_username.iter() { + // print weight + println!("weight: {:?}", weight.creator_username); + } assert_eq!(weights_by_username.len(), 5); Ok(()) } - async fn seed_weights_with_category_and_type(weights_type:WeightsType,weights_category:WeightsCategory,number_of_items:u32,creator_username:&str) -> AnyhowResult<()> { - + async fn seed_weights_with_category_and_type( + weights_type: WeightsType, + weights_category: WeightsCategory, + number_of_items: u32, + creator_username: &str + ) -> AnyhowResult<()> { let pool = setup().await; - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..number_of_items { - let mut rng = rand::thread_rng(); let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -342,7 +355,6 @@ mod tests { } Ok(()) - } // Tests paging for the list weights query builder @@ -352,40 +364,45 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,20,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 20, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); - + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + let result = qb.perform_query_for_page(&pool).await?; - + // let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 10); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .offset(Some(10)) - .limit(10); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .offset(Some(10)) + .limit(10); let result = qb.perform_query_for_page(&pool).await?; @@ -394,27 +411,31 @@ mod tests { Ok(()) } - #[tokio::test] #[serial] async fn list_weights_query_build_test_asc_desc_cursor_reverse() -> AnyhowResult<()> { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true) - .cursor_is_reversed(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .cursor_is_reversed(false); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -424,19 +445,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false) - .cursor_is_reversed(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false) + .cursor_is_reversed(true); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -452,18 +472,23 @@ mod tests { let pool = setup().await; let creator_username = "hanashi".to_string(); - seed_weights_with_category_and_type(WeightsType::LoRA,WeightsCategory::ImageGeneration,10,&creator_username).await?; - + seed_weights_with_category_and_type( + WeightsType::LoRA, + WeightsCategory::ImageGeneration, + 10, + &creator_username + ).await?; + let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(true); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true); let result = qb.perform_query_for_page(&pool).await?; - + // write code that will loop through the result and check if the ids are in ascending order let mut previous_id = 0; for weight in result.weights.iter() { @@ -473,19 +498,18 @@ mod tests { } let qb = ListWeightsQueryBuilder::new() - .offset(Some(0)) - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .limit(10) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false) - .sort_ascending(false); + .offset(Some(0)) + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(false); let result = qb.perform_query_for_page(&pool).await?; // write code that will loop through the result and check if the ids are in descending order - let mut previous_id - = 10000; + let mut previous_id = 10000; for weight in result.weights.iter() { let current_id = weight.weight_id; assert!(current_id < previous_id); @@ -499,11 +523,14 @@ mod tests { #[serial] async fn list_weights_query_build_test() -> AnyhowResult<()> { let pool = setup().await; - + let creator_username = "hanashi".to_string(); - let creator_token = get_user_token_by_username(&creator_username, &pool).await?.unwrap_or_else( - || panic!("Could not find user with username {}", creator_username) + let creator_token = get_user_token_by_username( + &creator_username, + &pool + ).await?.unwrap_or_else(|| + panic!("Could not find user with username {}", creator_username) ); for i in 0..6 { @@ -515,7 +542,7 @@ mod tests { WeightsType::RvcV2 } else if i % 5 == 3 { WeightsType::StableDiffusionXL - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsType::SoVitsSvc } else { WeightsType::StableDiffusion15 @@ -529,7 +556,7 @@ mod tests { WeightsCategory::VoiceConversion } else if i % 5 == 3 { WeightsCategory::ImageGeneration - } else if i ^ 5 == 4 { + } else if (i ^ 5) == 4 { WeightsCategory::VoiceConversion } else { WeightsCategory::ImageGeneration @@ -539,10 +566,10 @@ mod tests { let number: u32 = rng.gen(); let model_weight_token = ModelWeightToken(number.to_string()); - + let args = CreateModelWeightsArgs { token: &model_weight_token, // replace with actual ModelWeightToken - weights_type:weights_type, // replace with actual WeightsType + weights_type: weights_type, // replace with actual WeightsType weights_category: weights_category, // replace with actual WeightsCategory title: format!("Title {}", i), maybe_thumbnail_token: Some(format!("Thumbnail {}", i)), @@ -572,28 +599,192 @@ mod tests { // try a query with all the filters let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::Tacotron2) - .weights_category(WeightsCategory::TextToSpeech) + .weights_type(Some(WeightsType::Tacotron2)) + .weights_category(Some(WeightsCategory::TextToSpeech)) .scope_creator_username(Some("hanashi")) .include_user_deleted_results(false); - let result = qb.perform_query_for_page(&pool).await?; assert_eq!(result.weights.len(), 2); let qb = ListWeightsQueryBuilder::new() - .weights_type(WeightsType::LoRA) - .weights_category(WeightsCategory::ImageGeneration) - .scope_creator_username(Some("hanashi")) - .include_user_deleted_results(false); + .weights_type(Some(WeightsType::LoRA)) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 1); assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); assert_eq!(result.weights[0].weights_type, WeightsType::LoRA); Ok(()) } -} + #[tokio::test] + #[serial] + async fn list_weights_query_builder_page_type_and_category() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + // using the seeded weights from the previous test + let creator_username = "hanashi".to_string(); + + for i in 0..4 { + let weights_types = vec![ + WeightsType::HifiganTacotron2, + WeightsType::LoRA, + WeightsType::RvcV2, + WeightsType::StableDiffusionXL + ]; + + let weights_type = weights_types[i as usize]; + + let qb = ListWeightsQueryBuilder::new() + .weights_type(Some(weights_type)) + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + println!("weights_type: {:?}", weights_type); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 20, "Should be 20 weights"); + assert_eq!( + result.weights[0].weights_type, + weights_type, + "Should be the same weights type" + ); + } + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_weight_query_builder_all() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .limit(11); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 11); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn list_limits() -> AnyhowResult<()> { + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(10); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), 10); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false); + + let result = qb.perform_query_for_page(&pool).await?; + + // by default 25 + assert_eq!(result.weights.len(), 25); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + // over query limit + let qb: ListWeightsQueryBuilder = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(80); + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), 60); + // pick a random value in here + assert_eq!(result.weights[9].weights_category, WeightsCategory::ImageGeneration); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn page_across_a_category() -> AnyhowResult<()> { + // paging across a category and it works + let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING; + // how we implement paging. + let page:u16 = 0; + let page_size: u16 = 20; + + let pool = MySqlPoolOptions::new() + .max_connections(3) + .connect(&db_connection_string).await + .unwrap(); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .sort_ascending(true) + .limit(page_size); + + let result = qb.perform_query_for_page(&pool).await?; + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let page = 1; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .sort_ascending(true) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + let page = 2; + let offset = page * page_size; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + let qb = ListWeightsQueryBuilder::new() + .scope_creator_username(Some("hanashi")) + .include_user_deleted_results(false) + .sort_ascending(true) + .weights_category(Some(WeightsCategory::ImageGeneration)) + .limit(page_size) + .offset(Some(offset.into())); + + let result = qb.perform_query_for_page(&pool).await?; + + assert_eq!(result.weights.len(), page_size as usize); + assert_eq!(result.weights[0].weights_category, WeightsCategory::ImageGeneration); + + Ok(()) + } +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs index cbe0ba55d7..33bb1ace32 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/update_weight.rs @@ -9,37 +9,38 @@ pub struct UpdateWeightArgs<'a> { pub weight_token: &'a ModelWeightToken, pub title: Option<&'a str>, pub maybe_thumbnail_token: Option<&'a str>, - pub description_markdown: &'a str, - pub description_rendered_html: &'a str, - pub creator_set_visibility: &'a Visibility, + pub description_markdown: Option<&'a str>, + pub description_rendered_html: Option<&'a str>, + pub creator_set_visibility: Option<&'a Visibility>, + pub weights_type: Option, + pub weights_category: Option, pub mysql_pool: &'a MySqlPool, } pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { - let mut transaction = args.mysql_pool.begin().await?; + let transaction = args.mysql_pool.begin().await?; - let query_result = sqlx - ::query!( - r#" - UPDATE model_weights - SET - title = ?, - description_markdown = ?, - maybe_thumbnail_token = ?, - description_rendered_html = ?, - creator_set_visibility = ?, - version = version + 1 - WHERE token = ? - LIMIT 1 - "#, - args.title, - args.description_markdown, - args.maybe_thumbnail_token, - args.description_rendered_html, - args.creator_set_visibility.to_str(), - args.weight_token.as_str() - ) - .execute(args.mysql_pool).await; + + let query_result = sqlx::query!( + r#" + UPDATE model_weights + SET + title = COALESCE(?, title), + description_markdown = COALESCE(?, description_markdown), + maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token), + description_rendered_html = COALESCE(?, description_rendered_html), + creator_set_visibility = COALESCE(?, creator_set_visibility), + version = version + 1 + WHERE token = ? + "#, + args.title.as_deref(), + args.description_markdown.as_deref(), + args.maybe_thumbnail_token.as_deref(), + args.description_rendered_html.as_deref(), + args.creator_set_visibility.as_deref(), + args.weight_token.as_str() + ) + .execute(args.mysql_pool).await; transaction.commit().await?; @@ -53,6 +54,7 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> { #[cfg(test)] mod tests { + // Template use sqlx::mysql::MySqlPoolOptions; use tokio; diff --git a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs index e81d9c339a..5ec24e4eeb 100644 --- a/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs +++ b/crates/lib/mysql_queries/src/queries/trending_model_analytics/list_trending_tts_models.rs @@ -63,7 +63,7 @@ pub async fn list_trending_tts_models( ]; let query = query_parts.join(" UNION "); - let mut query = sqlx::query_as::<_, RawTrendingModel>(&query); + let query = sqlx::query_as::<_, RawTrendingModel>(&query); let results = query.fetch_all(&mut **mysql_connection).await?; diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs index 23dabea29c..f5bd643639 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs @@ -159,7 +159,7 @@ impl ListAssignedTtsCategoriesQueryBuilder { // NB: Model token is always used for scoping. query = query.bind(&self.scope_tts_model_token); - let mut results = query.fetch_all(&mut **mysql_connection) + let results = query.fetch_all(&mut **mysql_connection) .await?; Ok(results) diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs index 490493cf27..0a97d15436 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs @@ -192,7 +192,7 @@ pub async fn mark_tts_upload_job_done( { let status = if success { "complete_success" } else { "complete_failure" }; - let query_result = sqlx::query!( + let _query_result = sqlx::query!( r#" UPDATE tts_model_upload_jobs SET diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs index d11c2b921c..0ed9b446a7 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs @@ -268,7 +268,7 @@ LEFT OUTER JOIN users } } - if let Some(username) = self.scope_creator_username.as_deref() { + if let Some(_username) = self.scope_creator_username.as_deref() { if !first_predicate_added { query.push_str(" WHERE users.username = ?"); first_predicate_added = true; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs index 1842de1a2a..f2cafbbef1 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs @@ -93,7 +93,7 @@ SET "#, query_inner_positions, query_inner_ip_addresses, user_token, rule_tokens); // NB: This, unfortunately, cannot be statically checked. - let mut query = sqlx::query(&query); + let query = sqlx::query(&query); let result = query.execute(mysql_pool).await; diff --git a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs index 55d93b4246..015e52f891 100644 --- a/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs +++ b/crates/lib/mysql_queries/src/queries/twitch/twitch_oauth/find.rs @@ -145,7 +145,7 @@ impl TwitchOauthTokenFinder { query = query.bind(twitch_username); } - let mut result = query.fetch_optional(mysql_pool).await; + let result = query.fetch_optional(mysql_pool).await; match result { Ok(Some(record)) => { diff --git a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs index a6decf5fff..b897d8daaf 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/create_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/create_user_bookmark.rs @@ -4,15 +4,15 @@ use anyhow::anyhow; use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct CreateFavoriteArgs<'e, 'c, E> +pub struct CreateUserBookmarkArgs<'e, 'c, E> where E: 'e + Executor<'c, Database = MySql> { - pub entity_token: &'e FavoriteEntityToken, + pub entity_token: &'e UserBookmarkEntityToken, pub user_token: &'e UserToken, @@ -23,19 +23,19 @@ pub struct CreateFavoriteArgs<'e, 'c, E> pub phantom: PhantomData<&'c E>, } -pub async fn create_favorite<'e, 'c : 'e, E>( - args: CreateFavoriteArgs<'e, 'c, E>, +pub async fn create_user_bookmark<'e, 'c : 'e, E>( + args: CreateUserBookmarkArgs<'e, 'c, E>, ) - -> AnyhowResult + -> AnyhowResult where E: 'e + Executor<'c, Database = MySql> { - let favorite_token = FavoriteToken::generate(); + let user_bookmark_token = UserBookmarkToken::generate(); let (entity_type, entity_token) = args.entity_token.get_composite_keys(); let query_result = sqlx::query!( r#" -INSERT INTO favorites +INSERT INTO user_bookmarks SET token = ?, user_token = ?, @@ -46,7 +46,7 @@ ON DUPLICATE KEY UPDATE deleted_at = NULL, version = version + 1 "#, - &favorite_token, + &user_bookmark_token, args.user_token, entity_type, entity_token, @@ -59,5 +59,5 @@ ON DUPLICATE KEY UPDATE Err(err) => return Err(anyhow!("Mysql error: {:?}", err)), }; - Ok(favorite_token) + Ok(user_bookmark_token) } diff --git a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs similarity index 61% rename from crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs index 0ec20c4e6a..025b6f51c7 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/delete_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/delete_user_bookmark.rs @@ -1,22 +1,22 @@ use sqlx::{Executor, MySql}; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; // NB: UserToken is only supplied as an optimistic form of validation -pub async fn delete_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - user_token: &'e UserToken, - mysql_executor: E +pub async fn delete_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + user_token: &'e UserToken, + mysql_executor: E ) - -> AnyhowResult<()> + -> AnyhowResult<()> where E: 'e + Executor<'c, Database = MySql> { sqlx::query!( r#" -UPDATE favorites +UPDATE user_bookmarks SET deleted_at = CURRENT_TIMESTAMP, version = version + 1 @@ -25,7 +25,7 @@ token = ? AND user_token = ? LIMIT 1 "#, - favorite_token, + user_bookmark_token, user_token ) .execute(mysql_executor) diff --git a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs similarity index 51% rename from crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs index 01d46404bc..efb6903070 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/get_favorite.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_user_bookmark.rs @@ -2,15 +2,15 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use sqlx::{Executor, MySql}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, @@ -23,21 +23,21 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn get_favorite<'e, 'c, E>( - favorite_token: &'e FavoriteToken, - mysql_executor: E +pub async fn get_user_bookmark<'e, 'c, E>( + user_bookmark_token: &'e UserBookmarkToken, + mysql_executor: E ) - -> AnyhowResult> + -> AnyhowResult> where E: 'e + Executor<'c, Database = MySql> { let maybe_results = sqlx::query_as!( - RawFavorite, + RawUserBookmark, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, @@ -50,29 +50,29 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE f.token = ? "#, - favorite_token + user_bookmark_token ) .fetch_one(mysql_executor) .await; match maybe_results { - Ok(favorite) => Ok(Some(Favorite { - token: favorite.token, - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - user_token: favorite.user_token, - username: favorite.username, - user_display_name: favorite.user_display_name, - user_gravatar_hash: favorite.user_gravatar_hash, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - maybe_deleted_at: favorite.deleted_at, + Ok(user_bookmark) => Ok(Some(UserBookmark { + token: user_bookmark.token, + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + user_token: user_bookmark.user_token, + username: user_bookmark.username, + user_display_name: user_bookmark.user_display_name, + user_gravatar_hash: user_bookmark.user_gravatar_hash, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + maybe_deleted_at: user_bookmark.deleted_at, })), Err(err) => match err { sqlx::Error::RowNotFound => Ok(None), @@ -81,10 +81,10 @@ WHERE } } -pub struct RawFavorite { - pub token: FavoriteToken, +pub struct RawUserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, pub user_token: UserToken, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs similarity index 76% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index f42c7da477..232c7344a6 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -2,24 +2,25 @@ use anyhow::anyhow; use log::warn; use sqlx::MySqlPool; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use errors::AnyhowResult; -use crate::queries::favorites::list_user_favorites_result::{RawUserFavoriteRecord, UserFavorite}; +use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord; +use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark; -pub async fn list_user_favorites( +pub async fn list_user_bookmarks( username: &str, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { /// TODO(bt,2023-11-21): Maybe this query can use a switch /// See: https://stackoverflow.com/questions/2761574/mysql-use-case-else-value-as-join-parameter let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -39,7 +40,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -65,28 +66,28 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub async fn list_user_favorites_by_entity_type( - username: &str, - entity_type: FavoriteEntityType, - mysql_pool: &MySqlPool -) -> AnyhowResult> { +pub async fn list_user_user_bookmarks_by_entity_type( + username: &str, + entity_type: UserBookmarkEntityType, + mysql_pool: &MySqlPool +) -> AnyhowResult> { let maybe_results= sqlx::query_as!( - RawUserFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, - f.entity_type as `entity_type: enums::by_table::favorites::favorite_entity_type::FavoriteEntityType`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, + f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`, f.entity_token, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, @@ -106,7 +107,7 @@ SELECT zs_voices.title as maybe_descriptive_text_zs_voice_title FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token @@ -134,12 +135,12 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs similarity index 67% rename from crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs index e1080ac694..3df0f0812c 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_favorites_for_entity.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_for_entity.rs @@ -4,13 +4,13 @@ use log::warn; use sqlx::MySqlPool; use errors::AnyhowResult; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -use crate::queries::favorites::favorite_entity_token::FavoriteEntityToken; +use crate::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, pub user_token: UserToken, pub username: String, @@ -22,18 +22,18 @@ pub struct Favorite { pub maybe_deleted_at: Option>, } -pub async fn list_favorites_for_entity( - favorite_entity_token: FavoriteEntityToken, +pub async fn list_user_bookmarks_for_entity( + user_bookmark_entity_token: UserBookmarkEntityToken, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { - let (entity_type, entity_token) = favorite_entity_token.get_composite_keys(); + let (entity_type, entity_token) = user_bookmark_entity_token.get_composite_keys(); let maybe_results= sqlx::query_as!( - RawFavoriteRecord, + RawUserBookmarkRecord, r#" SELECT - f.token as `token: tokens::tokens::favorites::FavoriteToken`, + f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`, f.user_token as `user_token: tokens::tokens::users::UserToken`, u.username, u.display_name as user_display_name, @@ -44,7 +44,7 @@ SELECT f.deleted_at FROM - favorites AS f + user_bookmarks AS f JOIN users AS u ON f.user_token = u.token WHERE @@ -64,18 +64,18 @@ LIMIT 50 Err(err) => match err { sqlx::Error::RowNotFound => Ok(Vec::new()), _ => { - warn!("list favorites db error: {:?}", err); + warn!("list user_bookmarks db error: {:?}", err); Err(anyhow!("error with query: {:?}", err)) } }, Ok(results) => Ok(results.into_iter() - .map(|favorite| favorite.into_public_type()) + .map(|user_bookmark| user_bookmark.into_public_type()) .collect()), } } -pub struct RawFavoriteRecord { - token: FavoriteToken, +pub struct RawUserBookmarkRecord { + token: UserBookmarkToken, user_token: UserToken, username: String, @@ -87,9 +87,9 @@ pub struct RawFavoriteRecord { deleted_at: Option>, } -impl RawFavoriteRecord { - pub fn into_public_type(self) -> Favorite { - Favorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs similarity index 66% rename from crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs rename to crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index c391a5d8e6..e56d1de9d8 100644 --- a/crates/lib/mysql_queries/src/queries/favorites/list_user_favorites_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -1,18 +1,18 @@ use chrono::{DateTime, Utc}; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_type::MediaFileType; -use tokens::tokens::favorites::FavoriteToken; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use tokens::tokens::users::UserToken; -pub struct UserFavorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, - /// Something descriptive about the favorited entity. + /// Something descriptive about the bookmarked entity. /// This might be TTS text, a username, etc. It depends on the entity type. pub maybe_entity_descriptive_text: Option, @@ -26,10 +26,10 @@ pub struct UserFavorite { pub maybe_deleted_at: Option>, } -pub struct RawUserFavoriteRecord { - pub (crate) token: FavoriteToken, +pub struct RawUserBookmarkRecord { + pub (crate) token: UserBookmarkToken, - pub (crate) entity_type: FavoriteEntityType, + pub (crate) entity_type: UserBookmarkEntityType, pub (crate) entity_token: String, pub (crate) user_token: UserToken, @@ -51,27 +51,27 @@ pub struct RawUserFavoriteRecord { pub (crate) maybe_descriptive_text_zs_voice_title: Option, } -impl RawUserFavoriteRecord { - pub fn into_public_type(self) -> UserFavorite { - UserFavorite { +impl RawUserBookmarkRecord { + pub fn into_public_type(self) -> UserBookmark { + UserBookmark { token: self.token, entity_type: self.entity_type, entity_token: self.entity_token, maybe_entity_descriptive_text: match self.entity_type { - FavoriteEntityType::User => self.maybe_descriptive_text_user_display_name, - FavoriteEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, - FavoriteEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, - FavoriteEntityType::W2lTemplate => None, - FavoriteEntityType::W2lResult => None, + UserBookmarkEntityType::User => self.maybe_descriptive_text_user_display_name, + UserBookmarkEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, + UserBookmarkEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, + UserBookmarkEntityType::W2lTemplate => None, + UserBookmarkEntityType::W2lResult => None, // TODO(bt,2023-11-21): Summary text needs to be better enriched. - FavoriteEntityType::MediaFile => self.maybe_media_file_type + UserBookmarkEntityType::MediaFile => self.maybe_media_file_type .and_then(|media_file_type| match media_file_type { MediaFileType::Audio => Some("audio media file".to_string()), MediaFileType::Image => Some("image media file".to_string()), MediaFileType::Video => Some("video media file".to_string()), }), - FavoriteEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, - FavoriteEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, + UserBookmarkEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, + UserBookmarkEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, }, user_token: self.user_token, username: self.username, diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs new file mode 100644 index 0000000000..df6d7f415c --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/mod.rs @@ -0,0 +1,7 @@ +pub mod delete_user_bookmark; +pub mod create_user_bookmark; +pub mod user_bookmark_entity_token; +pub mod get_user_bookmark; +pub mod list_user_bookmarks_for_entity; +pub mod list_user_bookmarks; +pub mod list_user_bookmarks_result; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs new file mode 100644 index 0000000000..5fbf8224ad --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs @@ -0,0 +1,48 @@ +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use tokens::tokens::zs_voices::ZsVoiceToken; + +pub enum UserBookmarkEntityToken { + User(UserToken), + TtsModel(TtsModelToken), + TtsResult(TtsResultToken), + W2lTemplate(W2lTemplateToken), + W2lResult(W2lResultToken), + MediaFile(MediaFileToken), + VoiceConversionModel(VoiceConversionModelToken), + ZsVoice(ZsVoiceToken), +} + +impl UserBookmarkEntityToken { + pub fn from_entity_type_and_token(entity_type: UserBookmarkEntityType, token: &str) -> Self { + match entity_type { + UserBookmarkEntityType::User => Self::User(UserToken::new_from_str(token)), + UserBookmarkEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), + UserBookmarkEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), + UserBookmarkEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), + UserBookmarkEntityType::W2lResult => Self::W2lResult(W2lResultToken::new_from_str(token)), + UserBookmarkEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), + UserBookmarkEntityType::VoiceConversionModel => Self::VoiceConversionModel(VoiceConversionModelToken::new_from_str(token)), + UserBookmarkEntityType::ZsVoice => Self::ZsVoice(ZsVoiceToken::new_from_str(token)), + } + } + + pub fn get_composite_keys(&self) -> (UserBookmarkEntityType, &str) { + match self { + UserBookmarkEntityToken::User(user_token) => (UserBookmarkEntityType::User, user_token.as_str()), + UserBookmarkEntityToken::TtsModel(tts_model_token) => (UserBookmarkEntityType::TtsModel, tts_model_token.as_str()), + UserBookmarkEntityToken::TtsResult(tts_result_token) => (UserBookmarkEntityType::TtsResult, tts_result_token.as_str()), + UserBookmarkEntityToken::W2lTemplate(w2l_template_token) => (UserBookmarkEntityType::W2lTemplate, w2l_template_token.as_str()), + UserBookmarkEntityToken::W2lResult(w2l_result_token) => (UserBookmarkEntityType::W2lResult, w2l_result_token.as_str()), + UserBookmarkEntityToken::MediaFile(media_file_token) => (UserBookmarkEntityType::MediaFile, media_file_token.as_str()), + UserBookmarkEntityToken::VoiceConversionModel(voice_conversion_model_token) => (UserBookmarkEntityType::VoiceConversionModel, voice_conversion_model_token.as_str()), + UserBookmarkEntityToken::ZsVoice(zs_voice_token) => (UserBookmarkEntityType::ZsVoice, zs_voice_token.as_str()), + } + } +} diff --git a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs index e00ed36aa6..d9f623e33e 100644 --- a/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs +++ b/crates/lib/mysql_queries/src/queries/users/user/list_users_query_builder.rs @@ -65,10 +65,10 @@ impl ListUsersQueryBuilder { let internal_results = self.internal_query(mysql_pool).await?; - let mut first_id = internal_results.first() + let first_id = internal_results.first() .map(|raw_result| raw_result.user_id); - let mut last_id = internal_results.last() + let last_id = internal_results.last() .map(|raw_result| raw_result.user_id); let users = internal_results @@ -106,17 +106,17 @@ impl ListUsersQueryBuilder { ) -> AnyhowResult> { let query = self.build_query_string(); - let mut query = sqlx::query_as::<_, UserForListRaw>(&query); + let query = sqlx::query_as::<_, UserForListRaw>(&query); - let mut results = query.fetch_all(mysql_pool) + let results = query.fetch_all(mysql_pool) .await?; Ok(results) } pub fn build_query_string(&self) -> String { - let mut query_string = r#" + let query_string = r#" SELECT id as user_id, token as user_token, diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs index c12bf4e62a..0abda7cd4b 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use sqlx::MySqlPool; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; diff --git a/crates/lib/password/Cargo.toml b/crates/lib/password/Cargo.toml index f8063637b3..a158566eb1 100644 --- a/crates/lib/password/Cargo.toml +++ b/crates/lib/password/Cargo.toml @@ -17,9 +17,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bcrypt = "0.8" # password hashes -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } #hmac = "0.10.0" # jwt signing #jwt = "0.13.0" #sha2 = "0.9.5" # hmac signing diff --git a/crates/lib/redis_common/Cargo.toml b/crates/lib/redis_common/Cargo.toml index 28aa9fc780..a22cd5841c 100644 --- a/crates/lib/redis_common/Cargo.toml +++ b/crates/lib/redis_common/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" # Internal container_common = { path = "../deprecated/container_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" @@ -27,4 +30,3 @@ serde_json = "1.0.108" # TODO: Get rid of in favor of std::time time = "0.3.30" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/server_environment/Cargo.toml b/crates/lib/server_environment/Cargo.toml index 5ece74511a..2124b7857c 100644 --- a/crates/lib/server_environment/Cargo.toml +++ b/crates/lib/server_environment/Cargo.toml @@ -14,8 +14,14 @@ name = "server_environment" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] # None yet diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index 57245fc712..a26c50f6d5 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -16,8 +16,11 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/lib/storyteller_root/Cargo.toml b/crates/lib/storyteller_root/Cargo.toml index 60143bba40..c32ec7493e 100644 --- a/crates/lib/storyteller_root/Cargo.toml +++ b/crates/lib/storyteller_root/Cargo.toml @@ -13,9 +13,15 @@ name = "storyteller_root" path = "src/lib.rs" [dependencies] -# External +# Internal +# (None yet) + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } +# External +# (None yet) + [dev-dependencies] serial_test = "2.0.0" tempdir = "0.3.7" diff --git a/crates/lib/testing/Cargo.toml b/crates/lib/testing/Cargo.toml index de25b327df..e90fb429ad 100644 --- a/crates/lib/testing/Cargo.toml +++ b/crates/lib/testing/Cargo.toml @@ -18,6 +18,8 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 985c8e11fc..d6e4f622b9 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -18,6 +18,9 @@ path = "src/lib.rs" container_common = { path = "../deprecated/container_common" } user_input_common = { path = "../user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -29,4 +32,3 @@ unicode-segmentation = "1.10.1" # Serialization serde = "1.0.125" serde_derive = "1.0.125" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/url_config/Cargo.toml b/crates/lib/url_config/Cargo.toml index 91ee274c39..f458e4cb1a 100644 --- a/crates/lib/url_config/Cargo.toml +++ b/crates/lib/url_config/Cargo.toml @@ -18,7 +18,9 @@ path = "src/lib.rs" http_server_common = { path = "../deprecated/http_server_common" } reusable_types = { path = "../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External actix-web = "4.4.0" anyhow = "1.0.75" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml index d737ce2074..f0a2c6db70 100644 --- a/crates/lib/user_input_common/Cargo.toml +++ b/crates/lib/user_input_common/Cargo.toml @@ -13,10 +13,11 @@ name = "user_input_common" path = "src/lib.rs" [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External once_cell = "1.18.0" pulldown-cmark = "0.9.3" # Markdown regex = "1.5.4" unicode-segmentation = "1.10.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index c9f8bced1e..ef1916e6e7 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -17,6 +17,9 @@ path = "src/lib.rs" enums = { path = "../../_types/enums" } errors = { path = "../../lib/errors" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + # External bytes = "1.3" chrono = { version = "0.4.22", features = ["serde"] } @@ -35,7 +38,6 @@ reqwest = { version = "0.11", features = ["json"] } # HTTP Client # Scraping rss = "2.0.2" scraper = "0.17.1" -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] # None yet diff --git a/crates/my-workspace-hack/Cargo.toml b/crates/my-workspace-hack/Cargo.toml index 22f41a903d..dddf543e0b 100644 --- a/crates/my-workspace-hack/Cargo.toml +++ b/crates/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -40,12 +40,12 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } getrandom-c65f7effa3be6d31 = { package = "getrandom", version = "0.1.16", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } hyper = { version = "0.14.27", features = ["client", "http1", "http2", "runtime", "server"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } miniz_oxide = { version = "0.7.1", features = ["simd"] } @@ -67,19 +67,20 @@ sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } simd-adler32 = { version = "0.3.7" } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } -sqlx = { version = "0.7.2", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx = { version = "0.7.3", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } time = { version = "0.3.30", features = ["macros", "serde-well-known"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } twitch_oauth2 = { version = "0.6.1", features = ["reqwest_client"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } void = { version = "1.0.2" } +zip = { version = "0.6.6" } [build-dependencies] ahash = { version = "0.8.6" } @@ -88,11 +89,13 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } +deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0", features = ["serde"] } +flate2 = { version = "1.0.28" } flume = { version = "0.11.0", default-features = false, features = ["async"] } futures-channel = { version = "0.3.29", features = ["sink"] } futures-core = { version = "0.3.29" } @@ -102,13 +105,15 @@ futures-sink = { version = "0.3.29" } futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] } getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } +hashbrown = { version = "0.14.2", features = ["raw"] } heck = { version = "0.4.1", features = ["unicode"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } +indexmap = { version = "2.1.0", features = ["serde"] } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.150", features = ["align", "extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } memchr = { version = "2.6.4", features = ["use_std"] } +miniz_oxide = { version = "0.7.1", features = ["simd"] } mio = { version = "0.8.9", features = ["net", "os-ext"] } native-tls = { version = "0.2.11", default-features = false, features = ["vendored"] } num-integer = { version = "0.1.45", features = ["i128"] } @@ -124,30 +129,33 @@ serde = { version = "1.0.193", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } -sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } -sqlx-macros = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-macros-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } -sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +simd-adler32 = { version = "0.3.7" } +sqlx-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] } +sqlx-macros = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-macros-core = { version = "0.7.3", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] } +sqlx-mysql = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.7.3", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.39", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +time = { version = "0.3.30", features = ["macros", "serde-well-known"] } time-macros = { version = "0.2.15", default-features = false, features = ["formatting", "parsing", "serde"] } -tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.14", features = ["fs"] } tracing = { version = "0.1.40", features = ["log"] } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -url = { version = "2.4.1", features = ["serde"] } +url = { version = "2.5.0", features = ["serde"] } +zip = { version = "0.6.6" } [target.x86_64-unknown-linux-gnu.dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-unknown-linux-gnu.build-dependencies] -openssl = { version = "0.10.59", features = ["vendored"] } -openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.60", features = ["vendored"] } +openssl-sys = { version = "0.9.96", default-features = false, features = ["vendored"] } rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } @@ -163,13 +171,13 @@ spin = { version = "0.9.8" } [target.x86_64-pc-windows-msvc.dependencies] spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } [target.x86_64-pc-windows-msvc.build-dependencies] socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } -winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } +winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] } ### END HAKARI SECTION diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index a1dac56f4d..ed1487b8a3 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } tokens = { path = "../../../_types/tokens" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -36,10 +39,9 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Serialization serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 42bcfa7769..e97068915d 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -33,6 +33,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } subprocess_common = { path = "../../../lib/subprocess_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -50,7 +53,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -60,4 +63,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 965444fc3a..733388d420 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -41,8 +41,8 @@ testing = { path = "../../../lib/testing" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } -# Vendored code -# newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" @@ -63,7 +63,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -82,4 +82,3 @@ serde_json = "1.0.108" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index fb451aa8a1..cdc9aaf01a 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -38,9 +38,6 @@ subprocess_common = { path = "../../../lib/subprocess_common" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } - # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -59,7 +56,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 15ab7f6497..b911d80f6e 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -18,11 +18,11 @@ REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" # should rename my folder # abs path to the folder that contains the inference code need to get that container in the repo need basepath? relative to everyones env VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" -#VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" -VALL_E_X_INFERENCE_COMMAND = "python3 main.py" -VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .venv/bin/activate" +#VALL_E_X_INFERENCE_ROOT_DIRECTORY="/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X" +VALL_E_X_INFERENCE_COMMAND="python3 main.py" +VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .venv/bin/activate" -# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE = "" # fill this out for dev? +# VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev? #VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" #VALL_E_X_INFERENCE_COMMAND="python3 main.py" #VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate" @@ -82,6 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" +TT2_AS_MEDIA_FILES="true" # ================================= # # # diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 19c0d958c9..e90e3fc887 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -5,14 +5,17 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; use mysql_queries::queries::voice_conversion::results::get_voice_conversion_result_for_inference::get_voice_conversion_result_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -23,86 +26,47 @@ pub struct AudioFile { pub filesystem_path: PathBuf, } -pub async fn download_audio_file( - audio_source: &LipsyncAnimationAudioSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadAudioFileArgs<'a> { + pub audio_source: &'a LipsyncAnimationAudioSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result { let bucket_object_path; - match audio_source { + match args.audio_source { LipsyncAnimationAudioSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationAudioSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } - LipsyncAnimationAudioSource::T(tts_result_token) => { - // TODO(bt, 2023-09-08): Implement + LipsyncAnimationAudioSource::T(_tts_result_token) => { + // NB(bt,2023-12-08): This will likely never be implemented now that TTS is on media files. return Err(ProcessSingleJobError::NotYetImplemented) } LipsyncAnimationAudioSource::V(voice_conversion_result_token) => { - let voice_conversion_result = get_voice_conversion_result_for_inference( - voice_conversion_result_token, - false, - &mysql_pool, - ).await; - - let voice_conversion_result = match voice_conversion_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) - } - Err(e) => { - error!("could not query voice conversion result: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); - bucket_object_path = result_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_voice_conversion_result(voice_conversion_result_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("audio.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "audio file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -110,3 +74,82 @@ pub async fn download_audio_file( filesystem_path: downloaded_filesystem_path, }) } + +async fn from_media_file(media_file_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file: {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} + +async fn from_voice_conversion_result(voice_conversion_result_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result { + let voice_conversion_result = get_voice_conversion_result_for_inference( + voice_conversion_result_token, + false, + &args.mysql_pool, + ).await; + + let voice_conversion_result = match voice_conversion_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find voice conversion result: {:?}", voice_conversion_result_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token))) + } + Err(e) => { + error!("could not query voice conversion result: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash); + + Ok(result_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 6ac3b4d3e9..8285a78c5a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -5,12 +5,15 @@ use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -21,59 +24,40 @@ pub struct ImageFile { pub filesystem_path: PathBuf, } -pub async fn download_image_file( - image_source: &LipsyncAnimationImageSource, - public_bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job: &AvailableInferenceJob, - temp_dir_creator: &ScopedTempDirCreator, - work_temp_dir: &TempDir, - mysql_pool: &MySqlPool, -) -> Result { +pub struct DownloadImageFileArgs<'a> { + pub image_source: &'a LipsyncAnimationImageSource, + pub public_bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job: &'a AvailableInferenceJob, + pub temp_dir_creator: &'a ScopedTempDirCreator, + pub work_temp_dir: &'a TempDir, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn download_image_file(args: DownloadImageFileArgs<'_>) -> Result { let bucket_object_path; - match image_source { + match args.image_source { LipsyncAnimationImageSource::F(media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + bucket_object_path = from_media_file(media_file_token, &args).await?; } LipsyncAnimationImageSource::U(media_upload_token) => { - let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); - let media_upload_result = get_media_upload_for_inference( - &media_upload_token, - mysql_pool - ).await; - - let media_upload_result = match media_upload_result { - Ok(Some(result)) => result, - Ok(None) => { - warn!("could not find media upload: {:?}", media_upload_token); - return Err(ProcessSingleJobError::from_anyhow_error( - anyhow!("could not find media upload: {:?}", media_upload_token))) - } - Err(e) => { - error!("could not query media upload: {:?}", e); - return Err(ProcessSingleJobError::from_anyhow_error(e)) - } - }; - - let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); - bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + bucket_object_path = from_media_upload(media_upload_token, &args).await?; } } - let downloaded_filesystem_path = work_temp_dir.path().join("image.bin"); + let downloaded_filesystem_path = args.work_temp_dir.path().join("image.bin"); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "image file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, - bucket_client: public_bucket_client, - job_progress_reporter, + bucket_client: args.public_bucket_client, + job_progress_reporter: args.job_progress_reporter, job_progress_update_description: "downloading", - job_id: job.id.0, - scoped_tempdir_creator: &temp_dir_creator, + job_id: args.job.id.0, + scoped_tempdir_creator: &args.temp_dir_creator, maybe_existing_file_minimum_size_required: None, }).await?; @@ -81,3 +65,56 @@ pub async fn download_image_file( filesystem_path: downloaded_filesystem_path, }) } + +async fn from_media_file(media_file_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + true, + args.mysql_pool + ).await; + + let media_file_result = match media_file_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media file : {:?}", media_file_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file : {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file : {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_file_bucket_path = MediaFileBucketPath::from_object_hash( + &media_file_result.public_bucket_directory_hash, + media_file_result.maybe_public_bucket_prefix.as_deref(), + media_file_result.maybe_public_bucket_extension.as_deref()); + + Ok(media_file_bucket_path.to_full_object_pathbuf()) +} + +async fn from_media_upload(media_upload_token: &str, args: &DownloadImageFileArgs<'_>) -> Result { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + args.mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + Ok(media_upload_bucket_path.to_full_object_pathbuf()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 14fd9a32e0..57852d2b4b 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -4,7 +4,7 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; @@ -19,8 +19,8 @@ use tokens::tokens::users::UserToken; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; -use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; -use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; +use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs}; +use crate::job::job_types::lipsync::sad_talker::download_image_file::{download_image_file, DownloadImageFileArgs}; use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; @@ -97,27 +97,27 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result { - creator_user_token = UserToken::new_from_str(token); + UserToken::new_from_str(token) }, None => { return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))); } - } + }; // STEP 1. SETUP A TEMP DIRECTORY let work_temp_dir = format!("/tmp/temp_zeroshot_create_voice_{}", job.id.0); diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index 20359f1e0d..76f4940796 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -5,9 +5,9 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; -use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; +use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; +use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 082dd95eac..fb444a2a07 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -5,8 +5,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 69337d467f..65b6fa34a6 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -4,8 +4,8 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs index e05d695841..6ec512d1ea 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -4,9 +4,8 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; -use buckets::public::media_files::original_file::MediaFileBucketPath; - -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index 0df0d80325..705671b1c1 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -4,8 +4,8 @@ use std::time::Instant; use anyhow::{anyhow, Error}; use log::{error, info, warn}; use serde::{Deserialize, Serialize}; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; -use buckets::public::media_files::original_file::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::check_file_exists::check_file_exists; use filesys::file_size::file_size; diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 504d2fb08d..79d8d703c1 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -22,7 +22,6 @@ use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmen use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; -use newrelic_telemetry::Client as NewRelicClient; use crate::job_specific_dependencies::JobSpecificDependencies; use crate::util::scoped_execution::ScopedExecution; @@ -135,10 +134,6 @@ pub struct ClientDependencies { pub job_progress_reporter: Box, pub firehose_publisher: FirehosePublisher, - - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, } pub struct FileSystemDetails { diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 7779535799..186e87c9f4 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -44,7 +44,6 @@ use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use memory_caching::ttl_key_counter::TtlKeyCounter; use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use newrelic_telemetry::ClientBuilder; use crate::http_server::run_http_server::CreateServerArgs; use crate::http_server::run_http_server::launch_http_server; @@ -149,12 +148,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -237,8 +230,6 @@ async fn main() -> AnyhowResult<()> { clients: ClientDependencies { job_progress_reporter, firehose_publisher, - newrelic_client, - newrelic_disabled, }, job: JobSystemDependencies { system: JobSystemControls { diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index 64fe625e3a..07bfd1b465 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -42,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 6ac55b3d1d..2d1f6fa732 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -27,8 +27,8 @@ memory_caching = { path = "../../../lib/caching/memory_caching" } mysql_queries = { path = "../../../lib/mysql_queries" } tts_common = { path = "../../../lib/tts_common" } -# Vendored code -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" @@ -48,7 +48,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -58,4 +58,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs index 433c04771f..3b4296a0d3 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/job_args.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/job_args.rs @@ -10,7 +10,6 @@ use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::TtsModelForInferenceRecord; -use newrelic_telemetry::Client as NewRelicClient; use crate::caching::cache_miss_strategizer_multi::SyncMultiCacheMissStrategizer; use crate::caching::virtual_lfu_cache::SyncVirtualLfuCache; @@ -38,10 +37,6 @@ pub struct JobArgs { pub job_stats: JobStats, - pub newrelic_client: NewRelicClient, - - pub newrelic_disabled: bool, - pub worker_details: JobWorkerDetails, // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index bb9cd522a8..1767a674e2 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -6,14 +6,13 @@ use std::fs::File; use std::io::Read; use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::anyhow; use log::{error, info, warn}; use tempdir::TempDir; use container_common::anyhow_result::AnyhowResult; -use container_common::token::random_uuid::generate_random_uuid; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; @@ -24,7 +23,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_done use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceRecord}; use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; -use newrelic_telemetry::Span; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; @@ -56,34 +54,13 @@ pub async fn process_single_job( job_args: &JobArgs, job: &AvailableTtsInferenceJob, cached_model_record: &TtsModelForInferenceRecord, -) -> Result<(Span, Span), ProcessSingleJobError> { +) -> Result<(), ProcessSingleJobError> { // NB: Hack to allow cache bypass for debug requests // This will let us iterate faster with model changes. let mut maybe_queried_model = None; let mut model_record = cached_model_record; - let start = Instant::now(); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - let maybe_user_token = job.maybe_creator_user_token.as_deref().unwrap_or(""); - - let mut job_iteration_span = Span::new(&span_id, &trace_id, get_timestamp_millis()) - .name("single job execution") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - - let span_id = generate_random_uuid(); - let trace_id = generate_random_uuid(); - - let created_at_timestamp = (job.created_at.timestamp() as u64) * 1000; - - let mut since_creation_span = Span::new(&span_id, &trace_id, created_at_timestamp) - .name("job since creation") - .attribute("user_token", maybe_user_token) - .service_name("tts-inference-job"); - let mut job_progress_reporter = job_args .job_progress_reporter .new_tts_inference(&job.inference_job_token) @@ -100,15 +77,7 @@ pub async fn process_single_job( if !lock_acquired { warn!("Could not acquire job lock for: {:?}", &job.id); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "failure"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "failure"); - job_iteration_span.set_duration(duration); - - return Ok((since_creation_span, job_iteration_span)); + return Ok(()); } info!("Lock acquired for job: {}", job.inference_job_token); @@ -448,15 +417,7 @@ pub async fn process_single_job( info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", job.id, id, &inference_result_token); - let duration = start.elapsed(); - - since_creation_span.set_attribute("status", "success"); - since_creation_span.set_duration(duration); - - job_iteration_span.set_attribute("status", "success"); - job_iteration_span.set_duration(duration); - - Ok((since_creation_span, job_iteration_span)) + Ok(()) } fn get_timestamp_millis() -> u64 { diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index 9738caa13c..db471482ac 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -45,8 +45,6 @@ use mysql_queries::queries::tts::tts_inference_jobs::list_available_tts_inferenc use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; -use newrelic_telemetry::ClientBuilder; -use newrelic_telemetry::Span; use crate::caching::cache_miss_strategizer::CacheMissStrategizer; use crate::caching::cache_miss_strategizer::CacheMissStrategy; @@ -253,12 +251,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("TTS_MODEL_RECORD_CACHE_MILLIS", 300_000)?, // Five minutes ); - let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; - - let newrelic_disabled = easyenv::get_env_bool_or_default("IS_NEWRELIC_DISABLED", false); - - let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); - let maybe_minimum_priority = easyenv::get_env_string_optional("MAYBE_MINIMUM_PRIORITY") .map(|priority_string| { priority_string.parse::() @@ -301,8 +293,6 @@ async fn main() -> AnyhowResult<()> { tts_sidecar_health_check_client, }, job_stats: JobStats::new(), - newrelic_client, - newrelic_disabled, worker_details: JobWorkerDetails { is_on_prem, worker_hostname: server_hostname.clone(), @@ -344,8 +334,6 @@ async fn main_loop(job_args: JobArgs) { let mut noop_logger = NoOpLogger::new(job_args.no_op_logger_millis as i64); - let mut span_batch = Vec::new(); - let mut sort_by_priority = true; let mut sort_by_priority_count = 0; @@ -411,23 +399,11 @@ async fn main_loop(job_args: JobArgs) { needs_health_check_at_start = false; } - let mut spans = match batch_result { - Ok(spans) => spans, - Err(e) => { - warn!("Error running job batch: {:?}", e); - std::thread::sleep(Duration::from_millis(error_timeout_millis)); - error_timeout_millis += INCREASE_TIMEOUT_MILLIS; - continue; - } - }; - - if !job_args.newrelic_disabled { - span_batch.append(&mut spans); - - if span_batch.len() > 50 { - let spans_to_send = span_batch.split_off(0).into(); - job_args.newrelic_client.send_spans(spans_to_send).await; - } + if let Err(e) = batch_result { + warn!("Error running job batch: {:?}", e); + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; } error_timeout_millis = START_TIMEOUT_MILLIS; // reset @@ -441,7 +417,7 @@ async fn process_jobs( inferencer: &JobArgs, jobs: Vec, needs_health_check_at_start: bool, -) -> AnyhowResult> { +) -> AnyhowResult<()> { if needs_health_check_at_start { maybe_block_on_sidecar_health_check(&inferencer.http_clients.tts_sidecar_health_check_client).await; @@ -449,8 +425,6 @@ async fn process_jobs( let mut maybe_sidecar_health_issue = false; - let mut batch_spans = Vec::new(); - for job in jobs.into_iter() { let model_state_result = ModelState::query_model_and_check_filesystem( &job, @@ -555,44 +529,38 @@ async fn process_jobs( } let result = process_single_job(inferencer, &job, &model_state.model_record).await; - match result { - Ok((span1, span2)) => { - batch_spans.push(span1); - batch_spans.push(span2); - }, - Err(e) => { - warn!("Failure to process job: {:?}", e); + if let Err(e) = result { + warn!("Failure to process job: {:?}", e); - record_failure_and_maybe_slow_down(&inferencer.job_stats); + record_failure_and_maybe_slow_down(&inferencer.job_stats); - maybe_sidecar_health_issue = true; + maybe_sidecar_health_issue = true; - let failure_reason = "failure processing job"; - let internal_debugging_failure_reason = format!("job error: {:?}", e); + let failure_reason = "failure processing job"; + let internal_debugging_failure_reason = format!("job error: {:?}", e); - let _r = mark_tts_inference_job_failure( - &inferencer.mysql_pool, - &job, - failure_reason, - &internal_debugging_failure_reason, - inferencer.job_max_attempts, - &inferencer.get_worker_name(), - ).await; + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + &internal_debugging_failure_reason, + inferencer.job_max_attempts, + &inferencer.get_worker_name(), + ).await; - match e { - ProcessSingleJobError::Other(_) => {} // No-op - ProcessSingleJobError::FilesystemFull => { - // TODO: Refactor - we should stop processing all of these jobs as we'll lose out - // on this entire batch by attempting to clear the filesystem. This should be handled - // in the calling code. - delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; - } + match e { + ProcessSingleJobError::Other(_) => {} // No-op + ProcessSingleJobError::FilesystemFull => { + // TODO: Refactor - we should stop processing all of these jobs as we'll lose out + // on this entire batch by attempting to clear the filesystem. This should be handled + // in the calling code. + delete_tts_synthesizers_from_cache(&inferencer.semi_persistent_cache)?; } } } } - Ok(batch_spans) + Ok(()) } fn record_failure_and_maybe_slow_down(job_stats: &JobStats) { diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 2d3d54d8d0..7819055ed0 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -42,7 +45,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -52,4 +55,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 84c844bd40..ac801c96bd 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -24,6 +24,9 @@ google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } jobs_common = { path = "../../../lib/jobs_common" } mysql_queries = { path = "../../../lib/mysql_queries" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } @@ -40,7 +43,7 @@ r2d2_redis = "0.14.0" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -50,4 +53,3 @@ hyper = { version = "0.14.24" } serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index e9c1bb359a..58a1486caf 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -25,6 +25,9 @@ tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -40,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool @@ -81,7 +84,6 @@ toml = "0.5.8" # Text processing pulldown-cmark = "0.9.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 9623a9f9dc..869eee7790 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -21,6 +21,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } url_config = { path = "../../../lib/url_config" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -51,10 +54,9 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt async-trait = "0.1.58" sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } [dev-dependencies] mockall = "0.11.3" # NB: Tokio version matches `async-stripe` transitive dep; features are for testing -tokio = { version = "1.13.0", features = ["rt", "macros"] } +tokio = { version = "1.34.0", features = ["rt", "macros"] } diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index cc4df2852d..df5bae0773 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,8 +1,7 @@ use actix_web::HttpRequest; -// use mockall::automock; /// Allows us to purge session caches that might keep stale premium plan/billing data around -// #[cfg_attr(test, automock)] +#[cfg_attr(test, mockall::automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. diff --git a/crates/service/plugins/inference_workloads/Cargo.toml b/crates/service/plugins/inference_workloads/Cargo.toml index 6d1f817577..3b3bee9e34 100644 --- a/crates/service/plugins/inference_workloads/Cargo.toml +++ b/crates/service/plugins/inference_workloads/Cargo.toml @@ -14,5 +14,5 @@ path = "src/lib.rs" [dependencies] -# External +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index ca9835c9ae..86db221476 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -31,6 +31,9 @@ server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } user_input_common = { path = "../../../lib/user_input_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -65,4 +68,3 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", r2d2_redis = "0.14.0" byteorder = "1.5.0" strum_macros = "0.25.3" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index 5ed83e2aa9..4ea1672720 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -20,6 +20,9 @@ easyenv = { path = "../../../lib/easyenv" } errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External chrono = { version = "0.4.22", features = ["serde"] } dotenv = "0.15.0" @@ -29,7 +32,7 @@ hostname = "0.3.1" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures-util = "0.3.29" @@ -45,4 +48,3 @@ actix-web = "4.4.0" serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.108" -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index f015262c17..bc0e4b61bc 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -95,7 +95,7 @@ native-tls = { version = "0.2.8", features = ["vendored"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "1.34.0", features = ["macros"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool @@ -162,7 +162,8 @@ pulldown-cmark = "0.9.3" # The initial round of stabilizations for the async/await language feature in Rust 1.39 did not # include support for async fn in traits. async-trait = "0.1.58" -utoipa = { version = "4", features = ["actix_extras"] } + +utoipa = { version = "4", features = ["chrono","actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } http = "0.2.11" diff --git a/crates/service/web/storyteller_web/config/storyteller-web.development.env b/crates/service/web/storyteller_web/config/storyteller-web.development.env index 01ffcc593e..c890b5ef3d 100644 --- a/crates/service/web/storyteller_web/config/storyteller-web.development.env +++ b/crates/service/web/storyteller_web/config/storyteller-web.development.env @@ -5,6 +5,9 @@ COOKIE_SECURE=false COOKIE_DOMAIN=.jungle.horse WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse +# Test the rate limiter bypass (obviously these are dummy values) +AI_STREAMER_USERNAMES="PewDiePie,xQc" + # Stripe configs # All secrets are development and okay to check in STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index f76feb1e76..1d96b220e0 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -1,11 +1,49 @@ -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; -use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; -use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; +use crate::http_server::endpoints::weights::list_available_weights::*; +use crate::http_server::web_utils::response_success_helpers::*; +use crate::http_server::endpoints::weights::delete_weight::*; +use crate::http_server::endpoints::weights::get_weight::*; +use crate::http_server::endpoints::weights::update_weight::*; +use crate::http_server::endpoints::weights::list_weights_by_user::*; +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::generic_inference_jobs::*; +use tokens::tokens::model_weights::*; +use tokens::tokens::users::*; +use enums::common::visibility::Visibility; +use chrono::{DateTime, Utc}; use utoipa::OpenApi; + + #[derive(OpenApi)] -#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), - components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, + crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler, + crate::http_server::endpoints::weights::delete_weight::delete_weight_handler, + crate::http_server::endpoints::weights::update_weight::update_weight_handler, + crate::http_server::endpoints::weights::get_weight::get_weight_handler, + crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler + ),components(schemas( + // Tokens + UserToken,Visibility, + // + SimpleGenericJsonSuccess, + // Inference + EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken, + //Model Weights + ModelWeightToken,WeightsCategory,WeightsType, + GetWeightPathInfo,GetWeightResponse,GetWeightError, + UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError, + DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError, + ListWeightError,ModelWeightForList,ListWeightError, + ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, + ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight, + UserDetailsLight,DefaultAvatarInfo + )))] pub struct ApiDoc; + diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs new file mode 100644 index 0000000000..824be656d0 --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs @@ -0,0 +1,2 @@ +pub mod redis_rate_limiters; +pub mod username_set; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs new file mode 100644 index 0000000000..3aa3df32cf --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -0,0 +1,130 @@ +use std::time::Duration; + +use log::info; + +use config::common_env::CommonEnv; +use errors::AnyhowResult; +use limitation::Limiter; +use crate::configs::app_startup::username_set::UsernameSet; + +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::server_state::RedisRateLimiters; + + +/// Build the various rate limiters +pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult { + // TODO(bt,2023-12-13): Kill CommonEnv. + + info!("Setting up Redis rate limiters..."); + + let logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) + }; + + let logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) + }; + + let api_high_priority_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) + }; + + // This is for the following users: + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let api_ai_streamers_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_AI_STREAMERS_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_ai_streamers", limiter_enabled) + }; + + let ai_streamer_usernames = + UsernameSet::from_comma_separated(&easyenv::get_env_string_or_default("AI_STREAMER_USERNAMES", "")); + + info!("AI Streamers that can bypass the normal rate limiter ({}): {:?}", + ai_streamer_usernames.len(), + ai_streamer_usernames.list_names()); + + let model_upload_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) + }; + + let file_upload_logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_FILE_UPLOAD_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_FILE_UPLOAD_LOGGED_OUT_MAX_REQUESTS", 4)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_FILE_UPLOAD_LOGGED_OUT_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "file_upload_logged_out", limiter_enabled) + }; + + let file_upload_logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_FILE_UPLOAD_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_FILE_UPLOAD_LOGGED_IN_MAX_REQUESTS", 6)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_FILE_UPLOAD_LOGGED_IN_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "file_upload_logged_in", limiter_enabled) + }; + + Ok(RedisRateLimiters { + logged_out: logged_out_redis_rate_limiter, + logged_in: logged_in_redis_rate_limiter, + api_high_priority: api_high_priority_redis_rate_limiter, + api_ai_streamers: api_ai_streamers_redis_rate_limiter, + api_ai_streamer_username_set: ai_streamer_usernames, + model_upload: model_upload_rate_limiter, + file_upload_logged_out: file_upload_logged_out_redis_rate_limiter, + file_upload_logged_in: file_upload_logged_in_redis_rate_limiter, + }) +} diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs new file mode 100644 index 0000000000..0f48cb5350 --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -0,0 +1,103 @@ +use std::collections::{BTreeSet, HashSet}; + +#[derive(Clone)] +pub struct UsernameSet { + /// "usernames" in our service are always lowercase. + /// "display names" are case sensitive. + usernames: HashSet, +} + +impl UsernameSet { + pub fn from_comma_separated(comma_separated_list: &str) -> Self { + let usernames = comma_separated_list.split(",") + .into_iter() + .map(|item| item.trim()) + .map(|item| item.to_lowercase()) + .filter(|item| !item.is_empty()) + .collect::>(); + Self { + usernames + } + } + + pub fn len(&self) -> usize { + self.usernames.len() + } + + pub fn list_names(&self) -> BTreeSet<&str> { + // NB: BTreeSet for sorting, which is good for logging at app startup + self.usernames.iter().map(|item| item.as_str()).collect::<_>() + } + + pub fn username_is_in_set(&self, username: &str) -> bool { + // We lowercase the input username just in case a display name is passed + let username = username.to_lowercase(); + self.usernames.contains(&username) + } +} + +#[cfg(test)] +mod tests { + use crate::configs::app_startup::username_set::UsernameSet; + fn build_set() -> UsernameSet { + UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") + } + + #[test] + fn test_deduplication() { + let set = build_set(); + assert_eq!(set.len(), 3); + } + + #[test] + fn test_in_set() { + let set = build_set(); + assert!(set.username_is_in_set("bob")); + assert!(set.username_is_in_set("alice")); + assert!(set.username_is_in_set("john")); + } + + #[test] + fn test_in_set_case_insensitive() { + let set = build_set(); + + assert!(set.username_is_in_set("Bob")); + assert!(set.username_is_in_set("BOB")); + + assert!(set.username_is_in_set("AlIcE")); + assert!(set.username_is_in_set("ALICE")); + + assert!(set.username_is_in_set("jOHN")); + assert!(set.username_is_in_set("JOHN")); + } + + #[test] + fn test_not_in_set() { + let set = build_set(); + + // NB: The build_set() function might have attempted to introduce these, + // but the struct constructor should filter blank spaces out + assert!(!set.username_is_in_set("")); + assert!(!set.username_is_in_set(" ")); + + // Obviously not in set + assert!(!set.username_is_in_set("mary")); + assert!(!set.username_is_in_set("bill")); + + // These usernames are in the set, but not with padding + assert!(!set.username_is_in_set(" alice ")); + assert!(!set.username_is_in_set("bob ")); + assert!(!set.username_is_in_set(" john")); + } + + #[test] + fn test_username_return() { + let set = build_set(); + let usernames = set.list_names(); + assert_eq!(usernames.len(), 3); + assert!(usernames.contains(&"alice")); + assert!(usernames.contains(&"bob")); + assert!(usernames.contains(&"john")); + assert!(!usernames.contains(&"mary")); + } +} diff --git a/crates/service/web/storyteller_web/src/configs/mod.rs b/crates/service/web/storyteller_web/src/configs/mod.rs index 0a735a5b5c..806e1e3209 100644 --- a/crates/service/web/storyteller_web/src/configs/mod.rs +++ b/crates/service/web/storyteller_web/src/configs/mod.rs @@ -1,2 +1,3 @@ +pub mod app_startup; pub mod plans; pub mod static_api_tokens; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index d1353f60f7..582296e109 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -2,9 +2,9 @@ use tokens::tokens::users::UserToken; use crate::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username; use crate::user_avatars::default_avatar_from_username::default_avatar_from_username; - +use utoipa::ToSchema; /// Everything we need to refer to a user on the public web interface. -#[derive(Clone, Serialize)] +#[derive(Clone, Serialize,ToSchema)] pub struct UserDetailsLight { /// The token for the user pub user_token: UserToken, @@ -82,7 +82,8 @@ impl UserDetailsLight { } } -#[derive(Clone, Serialize)] + +#[derive(Clone, Serialize,ToSchema)] pub struct DefaultAvatarInfo { pub image_index: u8, pub color_index: u8, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs similarity index 85% rename from crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs index 8d1e536dcc..26b6de4369 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs @@ -37,7 +37,7 @@ const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; #[derive(Deserialize)] -pub struct EnqueueLipsyncAnimationRequest { +pub struct EnqueueFaceAnimationRequest { uuid_idempotency_token: String, audio_source: AudioSource, @@ -117,35 +117,35 @@ pub struct ImageSource { } #[derive(Serialize)] -pub struct EnqueueLipsyncAnimationSuccessResponse { +pub struct EnqueueFaceAnimationSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] -pub enum EnqueueLipsyncAnimationError { +pub enum EnqueueFaceAnimationError { BadInput(String), NotAuthorized, ServerError, RateLimited, } -impl ResponseError for EnqueueLipsyncAnimationError { +impl ResponseError for EnqueueFaceAnimationError { fn status_code(&self) -> StatusCode { match *self { - EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, - EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, - EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + EnqueueFaceAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueFaceAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueFaceAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueFaceAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(), - EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(), - EnqueueLipsyncAnimationError::ServerError => "server error".to_string(), - EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(), + EnqueueFaceAnimationError::BadInput(reason) => reason.to_string(), + EnqueueFaceAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueFaceAnimationError::ServerError => "server error".to_string(), + EnqueueFaceAnimationError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -153,16 +153,16 @@ impl ResponseError for EnqueueLipsyncAnimationError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl std::fmt::Display for EnqueueLipsyncAnimationError { +impl std::fmt::Display for EnqueueFaceAnimationError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) } } -pub async fn enqueue_lipsync_animation_handler( +pub async fn enqueue_face_animation_handler( http_request: HttpRequest, - request: web::Json, - server_state: web::Data>) -> Result + request: web::Json, + server_state: web::Data>) -> Result { let mut maybe_user_token : Option = None; @@ -171,7 +171,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); @@ -184,7 +184,7 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; if let Some(user_session) = maybe_user_session.as_ref() { @@ -209,20 +209,24 @@ pub async fn enqueue_lipsync_animation_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + warn!("User is not authorized because they are banned."); + return Err(EnqueueFaceAnimationError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueLipsyncAnimationError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(EnqueueLipsyncAnimationError::RateLimited); + return Err(EnqueueFaceAnimationError::RateLimited); } // ==================== LOOK UP MODEL INFO ==================== // @@ -239,7 +243,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token { LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("audio source not fully specified".to_string())); } }; @@ -249,7 +253,7 @@ pub async fn enqueue_lipsync_animation_handler( } else if let Some(ref token) = request.image_source.maybe_media_upload_token { LipsyncAnimationImageSource::media_upload_token(token.as_str()) } else { - return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string())); + return Err(EnqueueFaceAnimationError::BadInput("image source not fully specified".to_string())); } }; @@ -329,7 +333,7 @@ pub async fn enqueue_lipsync_animation_handler( Ok((job_token, _id)) => job_token, Err(err) => { warn!("New generic inference job creation DB error: {:?}", err); - return Err(EnqueueLipsyncAnimationError::ServerError); + return Err(EnqueueFaceAnimationError::ServerError); } }; @@ -341,16 +345,16 @@ pub async fn enqueue_lipsync_animation_handler( .await .map_err(|e| { warn!("error publishing event: {:?}", e); - EnqueueLipsyncAnimationError::ServerError + EnqueueFaceAnimationError::ServerError })?; - let response: EnqueueLipsyncAnimationSuccessResponse = EnqueueLipsyncAnimationSuccessResponse { + let response: EnqueueFaceAnimationSuccessResponse = EnqueueFaceAnimationSuccessResponse { success: true, inference_job_token: job_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?; + .map_err(|_e| EnqueueFaceAnimationError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs index 5a9c5c080e..1a000a595b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -1,2 +1,2 @@ -pub mod enqueue_lipsync_animation; -pub mod enqueue_rerender_animation; \ No newline at end of file +pub mod enqueue_rerender_animation; +pub mod enqueue_face_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs deleted file mode 100644 index efbb773b84..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_entity_handler.rs +++ /dev/null @@ -1,122 +0,0 @@ -// NB: Incrementally getting rid of build warnings... -#![forbid(unused_imports)] -#![forbid(unused_mut)] -#![forbid(unused_variables)] - -use std::fmt; -use std::sync::Arc; - -use actix_web::{HttpRequest, HttpResponse, web}; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; -use actix_web::web::Path; -use chrono::{DateTime, Utc}; -use log::warn; - -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::list_favorites_for_entity::list_favorites_for_entity; -use tokens::tokens::favorites::FavoriteToken; - -use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::server_state::ServerState; - -/// For the URL PathInfo -#[derive(Deserialize)] -pub struct ListFavoritesPathInfo { - entity_type: FavoriteEntityType, - entity_token: String, -} - -#[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { - pub success: bool, - pub favorites: Vec, -} - -#[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, - - pub user: UserDetailsLight, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug)] -pub enum ListFavoritesError { - ServerError, -} - -impl ResponseError for ListFavoritesError { - fn status_code(&self) -> StatusCode { - match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, - } - } - - fn error_response(&self) -> HttpResponse { - let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - }; - - to_simple_json_error(&error_reason, self.status_code()) - } -} - -// NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -pub async fn list_favorites_for_entity_handler( - _http_request: HttpRequest, - path: Path, - server_state: web::Data> -) -> Result -{ - let entity_token = FavoriteEntityToken::from_entity_type_and_token( - path.entity_type, &path.entity_token); - - let query_results = list_favorites_for_entity( - entity_token, - &server_state.mysql_pool, - ).await; - - let favorites = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); - } - }; - - let response = ListFavoritesSuccessResponse { - success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - user: UserDetailsLight { - user_token: favorite.user_token.clone(), - username: favorite.username.to_string(), // NB: Cloned because of ref use for avatar below - display_name: favorite.user_display_name.clone(), - gravatar_hash: favorite.user_gravatar_hash.clone(), - default_avatar: DefaultAvatarInfo::from_username(&favorite.username), - }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, - }) - .collect(), - }; - - let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs deleted file mode 100644 index 4529d2f18b..0000000000 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod create_favorite_handler; -pub mod delete_favorite_handler; -pub mod list_favorites_for_entity_handler; -pub mod list_favorites_for_session_handler; -pub mod list_favorites_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index d227585c20..5befd6c91e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -9,8 +9,8 @@ use chrono::{DateTime, Utc}; use log::error; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::common::job_status_plus::JobStatusPlus; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs index 021d4db258..b18f8af8da 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::warn; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use mysql_queries::queries::media_files::get_media_file::get_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 792af74603..7fe2872d84 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -5,7 +5,7 @@ use actix_multipart::Multipart; use actix_web::{HttpRequest, web}; use log::{error, info, warn}; -use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::common::visibility::Visibility; @@ -66,26 +66,25 @@ pub async fn handle_upload( UploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(UploadError::RateLimited); } - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - // ==================== READ MULTIPART REQUEST ==================== // let upload_media_request = drain_multipart_request(multipart_payload) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 81af5ee4b2..0a6ed9375d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -25,4 +25,5 @@ pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; pub mod w2l; -pub mod favorites; \ No newline at end of file +pub mod user_bookmarks; +pub mod weights; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index b01dd37a5d..2001562038 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -32,6 +32,7 @@ use tts_common::priority::{FAKEYOU_DEFAULT_VALID_API_TOKEN_PRIORITY_LEVEL, FAKEY use user_input_common::check_for_slurs::contains_slurs; use users_component::utils::user_session_extended::UserSessionExtended; +use crate::configs::app_startup::username_set::UsernameSet; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; use crate::configs::plans::plan::Plan; use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; @@ -240,30 +241,44 @@ pub async fn enqueue_infer_tts_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + warn!("User is not authorized to use TTS model because they are banned."); + return Err(InferTtsError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { - let mut rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - warn!("User is not authorized to use TTS model because they are banned."); - return Err(InferTtsError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in + let maybe_username = maybe_user_session + .as_ref() + .map(|session| session.user.username.as_str()); + + // FIXME(bt,2023-12-13): These boolean flags are a bit dangerous. + let rate_limiter_type = get_rate_limiter_type( + maybe_username, + &server_state.redis_rate_limiters.api_ai_streamer_username_set, + is_investor, + is_from_api, + use_high_priority_rate_limiter + ); + + let rate_limiter = match rate_limiter_type { + RateLimiterType::LoggedOut => &server_state.redis_rate_limiters.logged_out, + RateLimiterType::LoggedIn => &server_state.redis_rate_limiters.logged_in, + RateLimiterType::ApiHighPriority => { + info!("Using API high priority rate limiter"); + &server_state.redis_rate_limiters.api_high_priority + }, + RateLimiterType::ApiAiStreamer => { + info!("Using AI streamer rate limiter"); + &server_state.redis_rate_limiters.api_ai_streamers }, }; - // TODO/TEMP - if is_investor || is_from_api { - rate_limiter = &server_state.redis_rate_limiters.logged_in; - } - - // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters - if use_high_priority_rate_limiter { - rate_limiter = &server_state.redis_rate_limiters.api_high_priority; - } - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } @@ -476,6 +491,54 @@ pub async fn enqueue_infer_tts_handler( .body(body)) } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RateLimiterType { + LoggedOut, + LoggedIn, + ApiHighPriority, + ApiAiStreamer, +} + +// FIXME(bt,2023-12-13): The boolean flags and disjoint decision making are kind of terrible. +fn get_rate_limiter_type( + maybe_username: Option<&str>, + ai_streamer_username_set: &UsernameSet, + is_investor: bool, + is_from_api: bool, + use_high_priority_rate_limiter: bool, +) -> RateLimiterType { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + + let mut rate_limiter = match maybe_username { + None => RateLimiterType::LoggedOut, + Some(username) => { + if ai_streamer_username_set.username_is_in_set(username) { + is_ai_streamer = true; + } + + RateLimiterType::LoggedIn + }, + }; + + // TODO/TEMP + if is_investor || is_from_api { + rate_limiter = RateLimiterType::LoggedIn; + } + + // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters + if use_high_priority_rate_limiter { + rate_limiter = RateLimiterType::ApiHighPriority; + } + + if is_ai_streamer { + rate_limiter = RateLimiterType::ApiAiStreamer; + } + + rate_limiter +} + pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> { if text.len() < 3 { return Err("text is too short".to_string()); @@ -542,3 +605,37 @@ fn random_troll_text() -> &'static str { let random = texts.choose(&mut rand::thread_rng()); random.map(|r| *r).unwrap_or(texts[0]) } + + +#[cfg(test)] +mod tests { + + mod rate_limiter_tests { + use crate::configs::app_startup::username_set::UsernameSet; + use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::{get_rate_limiter_type, RateLimiterType}; + + fn build_username_set() -> UsernameSet { + UsernameSet::from_comma_separated("ai_streamer") + } + + #[test] + fn test_logged_out() { + let rate_limiter = get_rate_limiter_type(None, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedOut); + } + + #[test] + fn test_logged_in() { + let username = Some("not_ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedIn); + } + + #[test] + fn test_ai_streamer() { + let username = Some("ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::ApiAiStreamer); + } + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index be43c33d8a..20dab038b7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -8,7 +8,7 @@ use actix_web::web::Path; use chrono::{DateTime, Utc}; use log::{error, log}; use r2d2_redis::redis::{Commands, RedisResult}; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs similarity index 52% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs index cb4aa5f5f4..b62fe2feeb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/create_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs @@ -11,46 +11,46 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::favorite_entity_token::FavoriteEntityToken; -use mysql_queries::queries::favorites::create_favorite::{create_favorite, CreateFavoriteArgs}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct CreateFavoriteRequest { +pub struct CreateUserBookmarkRequest { entity_token: String, - entity_type: FavoriteEntityType, + entity_type: UserBookmarkEntityType, } #[derive(Serialize)] -pub struct CreateFavoriteSuccessResponse { +pub struct CreateUserBookmarkSuccessResponse { pub success: bool, - pub favorite_token: FavoriteToken, + pub user_bookmark_token: UserBookmarkToken, } #[derive(Debug)] -pub enum CreateFavoriteError { +pub enum CreateUserBookmarkError { BadInput(String), NotAuthorized, ServerError, } -impl ResponseError for CreateFavoriteError { +impl ResponseError for CreateUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - CreateFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - CreateFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - CreateFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + CreateUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - CreateFavoriteError::BadInput(reason) => reason.to_string(), - CreateFavoriteError::NotAuthorized => "unauthorized".to_string(), - CreateFavoriteError::ServerError => "server error".to_string(), + CreateUserBookmarkError::BadInput(reason) => reason.to_string(), + CreateUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + CreateUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -58,24 +58,24 @@ impl ResponseError for CreateFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for CreateFavoriteError { +impl fmt::Display for CreateUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn create_favorite_handler( +pub async fn create_user_bookmark_handler( http_request: HttpRequest, - request: web::Json, + request: web::Json, server_state: web::Data>, -) -> Result +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -84,42 +84,42 @@ pub async fn create_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - CreateFavoriteError::ServerError + CreateUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { warn!("not logged in"); - return Err(CreateFavoriteError::NotAuthorized); + return Err(CreateUserBookmarkError::NotAuthorized); } }; - let entity_token = FavoriteEntityToken::from_entity_type_and_token( + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( request.entity_type, &request.entity_token); - let query_result = create_favorite(CreateFavoriteArgs { + let query_result = create_user_bookmark(CreateUserBookmarkArgs { entity_token: &entity_token, user_token: &user_session.user_token_typed, mysql_executor: &mut *mysql_connection, phantom: Default::default(), }).await; - let favorite_token = match query_result { + let user_bookmark_token = match query_result { Ok(token) => token, Err(err) => { - warn!("error inserting favorite: {:?}", err); - return Err(CreateFavoriteError::ServerError); + warn!("error inserting user_bookmark: {:?}", err); + return Err(CreateUserBookmarkError::ServerError); } }; - let response = CreateFavoriteSuccessResponse { + let response = CreateUserBookmarkSuccessResponse { success: true, - favorite_token, + user_bookmark_token, }; let body = serde_json::to_string(&response) - .map_err(|_e| CreateFavoriteError::ServerError)?; + .map_err(|_e| CreateUserBookmarkError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs similarity index 58% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs index be28d24534..d68fc29e75 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/delete_favorite_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/delete_user_bookmark_handler.rs @@ -12,8 +12,8 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use log::warn; -use mysql_queries::queries::favorites::delete_favorite::delete_favorite; -use tokens::tokens::favorites::FavoriteToken; +use mysql_queries::queries::user_bookmarks::delete_user_bookmark::delete_user_bookmark; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -21,40 +21,40 @@ use crate::server_state::ServerState; /// For the URL PathInfo #[derive(Deserialize)] -pub struct DeleteFavoritePathInfo { - favorite_token: FavoriteToken, +pub struct DeleteUserBookmarkPathInfo { + user_bookmark_token: UserBookmarkToken, } #[derive(Deserialize)] -pub struct DeleteFavoriteRequest { +pub struct DeleteUserBookmarkRequest { /// NB: this is only to disambiguate when a user is both a mod and an author. as_mod: Option, } #[derive(Debug)] -pub enum DeleteFavoriteError { +pub enum DeleteUserBookmarkError { BadInput(String), NotAuthorized, NotFound, ServerError, } -impl ResponseError for DeleteFavoriteError { +impl ResponseError for DeleteUserBookmarkError { fn status_code(&self) -> StatusCode { match *self { - DeleteFavoriteError::BadInput(_) => StatusCode::BAD_REQUEST, - DeleteFavoriteError::NotAuthorized => StatusCode::UNAUTHORIZED, - DeleteFavoriteError::NotFound => StatusCode::NOT_FOUND, - DeleteFavoriteError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + DeleteUserBookmarkError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteUserBookmarkError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteUserBookmarkError::NotFound => StatusCode::NOT_FOUND, + DeleteUserBookmarkError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - DeleteFavoriteError::BadInput(reason) => reason.to_string(), - DeleteFavoriteError::NotAuthorized => "unauthorized".to_string(), - DeleteFavoriteError::NotFound => "not found".to_string(), - DeleteFavoriteError::ServerError => "server error".to_string(), + DeleteUserBookmarkError::BadInput(reason) => reason.to_string(), + DeleteUserBookmarkError::NotAuthorized => "unauthorized".to_string(), + DeleteUserBookmarkError::NotFound => "not found".to_string(), + DeleteUserBookmarkError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -62,24 +62,24 @@ impl ResponseError for DeleteFavoriteError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for DeleteFavoriteError { +impl fmt::Display for DeleteUserBookmarkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn delete_favorite_handler( +pub async fn delete_user_bookmark_handler( http_request: HttpRequest, - path: Path, - _request: web::Json, + path: Path, + _request: web::Json, server_state: web::Data> -) -> Result { +) -> Result { let mut mysql_connection = server_state.mysql_pool .acquire() .await .map_err(|err| { warn!("MySql pool error: {:?}", err); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let maybe_user_session = server_state @@ -88,18 +88,18 @@ pub async fn delete_favorite_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - DeleteFavoriteError::ServerError + DeleteUserBookmarkError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { - return Err(DeleteFavoriteError::NotAuthorized); + return Err(DeleteUserBookmarkError::NotAuthorized); } }; - let query_result = delete_favorite( - &path.favorite_token, + let query_result = delete_user_bookmark( + &path.user_bookmark_token, &user_session.user_token_typed, &mut *mysql_connection ).await; @@ -108,7 +108,7 @@ pub async fn delete_favorite_handler( Ok(_) => {}, Err(err) => { warn!("Update tts mod approval status DB error: {:?}", err); - return Err(DeleteFavoriteError::ServerError); + return Err(DeleteUserBookmarkError::ServerError); } }; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs new file mode 100644 index 0000000000..0fa28887fd --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_entity_handler.rs @@ -0,0 +1,122 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks_for_entity::list_user_bookmarks_for_entity; +use tokens::tokens::user_bookmarks::UserBookmarkToken; + +use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct ListUserBookmarkPathInfo { + entity_type: UserBookmarkEntityType, + entity_token: String, +} + +#[derive(Serialize)] +pub struct ListUserBookmarksSuccessResponse { + pub success: bool, + pub user_bookmarks: Vec, +} + +#[derive(Serialize)] +pub struct UserBookmark { + pub token: UserBookmarkToken, + + pub user: UserDetailsLight, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ListUserBookmarksError { + ServerError, +} + +impl ResponseError for ListUserBookmarksError { + fn status_code(&self) -> StatusCode { + match *self { + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListUserBookmarksError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListUserBookmarksError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_user_bookmarks_for_entity_handler( + _http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + let entity_token = UserBookmarkEntityToken::from_entity_type_and_token( + path.entity_type, &path.entity_token); + + let query_results = list_user_bookmarks_for_entity( + entity_token, + &server_state.mysql_pool, + ).await; + + let user_bookmarks = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListUserBookmarksError::ServerError); + } + }; + + let response = ListUserBookmarksSuccessResponse { + success: true, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + user: UserDetailsLight { + user_token: user_bookmark.user_token.clone(), + username: user_bookmark.username.to_string(), // NB: Cloned because of ref use for avatar below + display_name: user_bookmark.user_display_name.clone(), + gravatar_hash: user_bookmark.user_gravatar_hash.clone(), + default_avatar: DefaultAvatarInfo::from_username(&user_bookmark.username), + }, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, + }) + .collect(), + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| ListUserBookmarksError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs index cdc11f8550..dc1f17d90c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_session_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_session_handler.rs @@ -13,38 +13,38 @@ use actix_web::web::Query; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -55,23 +55,23 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, NotAuthorizedError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - ListFavoritesError::NotAuthorizedError => StatusCode::UNAUTHORIZED, + ListUserBookmarksError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::NotAuthorizedError => StatusCode::UNAUTHORIZED, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), - ListFavoritesError::NotAuthorizedError => "not authorized".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), + ListUserBookmarksError::NotAuthorizedError => "not authorized".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -79,61 +79,61 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_session_handler( +pub async fn list_user_bookmarks_for_session_handler( http_request: HttpRequest, - query: Query, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let user_session = server_state.session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); - ListFavoritesError::ServerError + ListUserBookmarksError::ServerError })? - .ok_or(ListFavoritesError::NotAuthorizedError)?; + .ok_or(ListUserBookmarksError::NotAuthorizedError)?; let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&user_session.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&user_session.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&user_session.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, maybe_summary_text: None, maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs similarity index 51% rename from crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs index 071b1fcb0e..ccaf8b61f4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/favorites/list_favorites_for_user_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/list_user_bookmarks_for_user_handler.rs @@ -13,43 +13,43 @@ use actix_web::web::{Path, Query}; use chrono::{DateTime, Utc}; use log::warn; -use enums::by_table::favorites::favorite_entity_type::FavoriteEntityType; -use mysql_queries::queries::favorites::list_user_favorites::{list_user_favorites, list_user_favorites_by_entity_type}; -use tokens::tokens::favorites::FavoriteToken; +use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; +use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type}; +use tokens::tokens::user_bookmarks::UserBookmarkToken; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; #[derive(Deserialize)] -pub struct ListFavoritesPathInfo { +pub struct ListUserBookmarksPathInfo { username: String, } #[derive(Deserialize)] -pub struct ListFavoritesQueryData { - maybe_scoped_entity_type: Option, +pub struct ListUserBookmarksQueryData { + maybe_scoped_entity_type: Option, } #[derive(Serialize)] -pub struct ListFavoritesSuccessResponse { +pub struct ListUserBookmarksSuccessResponse { pub success: bool, - pub favorites: Vec, + pub user_bookmarks: Vec, } #[derive(Serialize)] -pub struct Favorite { - pub token: FavoriteToken, +pub struct UserBookmark { + pub token: UserBookmarkToken, - pub details: FavoriteDetails, + pub details: UserBookmarkDetails, pub created_at: DateTime, pub updated_at: DateTime, } #[derive(Serialize)] -pub struct FavoriteDetails { +pub struct UserBookmarkDetails { // TODO: This needs titles or some other summary metadata. - pub entity_type: FavoriteEntityType, + pub entity_type: UserBookmarkEntityType, pub entity_token: String, // TODO: Populate this for TTS @@ -60,20 +60,20 @@ pub struct FavoriteDetails { } #[derive(Debug)] -pub enum ListFavoritesError { +pub enum ListUserBookmarksError { ServerError, } -impl ResponseError for ListFavoritesError { +impl ResponseError for ListUserBookmarksError { fn status_code(&self) -> StatusCode { match *self { - ListFavoritesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - ListFavoritesError::ServerError => "server error".to_string(), + ListUserBookmarksError::ServerError => "server error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -81,56 +81,56 @@ impl ResponseError for ListFavoritesError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for ListFavoritesError { +impl fmt::Display for ListUserBookmarksError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn list_favorites_for_user_handler( +pub async fn list_user_bookmarks_for_user_handler( _http_request: HttpRequest, - path: Path, - query: Query, + path: Path, + query: Query, server_state: web::Data> -) -> Result +) -> Result { let query_results = match query.maybe_scoped_entity_type { - None => list_user_favorites(&path.username, &server_state.mysql_pool).await, + None => list_user_bookmarks(&path.username, &server_state.mysql_pool).await, Some(entity_type) => - list_user_favorites_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) + list_user_user_bookmarks_by_entity_type(&path.username, entity_type, &server_state.mysql_pool) .await, }; - let favorites = match query_results { + let user_bookmarks = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); - return Err(ListFavoritesError::ServerError); + return Err(ListUserBookmarksError::ServerError); } }; - let response = ListFavoritesSuccessResponse { + let response = ListUserBookmarksSuccessResponse { success: true, - favorites: favorites.into_iter() - .map(|favorite| Favorite { - token: favorite.token, - details: FavoriteDetails { - entity_type: favorite.entity_type, - entity_token: favorite.entity_token, - maybe_summary_text: favorite.maybe_entity_descriptive_text, + user_bookmarks: user_bookmarks.into_iter() + .map(|user_bookmark| UserBookmark { + token: user_bookmark.token, + details: UserBookmarkDetails { + entity_type: user_bookmark.entity_type, + entity_token: user_bookmark.entity_token, + maybe_summary_text: user_bookmark.maybe_entity_descriptive_text, // TODO(bt,2023-11-21): Thumbnails need proper support. We should build them as a // first-class system before handling the backfill here. maybe_thumbnail_url: None, }, - created_at: favorite.created_at, - updated_at: favorite.updated_at, + created_at: user_bookmark.created_at, + updated_at: user_bookmark.updated_at, }) .collect(), }; let body = serde_json::to_string(&response) - .map_err(|_e| ListFavoritesError::ServerError)?; + .map_err(|_e| ListUserBookmarksError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs new file mode 100644 index 0000000000..11881f5bb3 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/mod.rs @@ -0,0 +1,5 @@ +pub mod create_user_bookmark_handler; +pub mod delete_user_bookmark_handler; +pub mod list_user_bookmarks_for_entity_handler; +pub mod list_user_bookmarks_for_session_handler; +pub mod list_user_bookmarks_for_user_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index 22131729c0..7be7cec00d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -205,17 +205,20 @@ pub async fn enqueue_voice_conversion_inference_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _session) => &server_state.redis_rate_limiters.logged_in }; // TODO/TEMP diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 32ae10d58d..7e09f40782 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,7 +1,6 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] -use utoipa::ToSchema; use std::fmt::Debug; use std::sync::Arc; @@ -12,6 +11,7 @@ use actix_web::http::StatusCode; use log::warn; use serde::Deserialize; use serde::Serialize; +use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; @@ -43,23 +43,20 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; -#[derive(Deserialize)] -#[derive(ToSchema)] +#[derive(Deserialize,ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, voice_token: String, } -#[derive(Serialize)] -#[derive(ToSchema)] +#[derive(Serialize,ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } -#[derive(Debug)] -#[derive(ToSchema)] +#[derive(Debug,ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, @@ -156,16 +153,19 @@ pub async fn enqueue_tts_request( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueTTSRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueTTSRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index b257ce9c14..e89e18eca9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -6,7 +6,7 @@ use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use hyper::StatusCode; use log::{error, info, warn}; use once_cell::sync::Lazy; -use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType::MediaUpload; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; @@ -26,6 +26,7 @@ use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; @@ -111,16 +112,19 @@ pub async fn upload_zs_sample_handler( UploadSampleError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadSampleError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadSampleError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _user) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs index b47de2c732..6654843007 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs @@ -30,6 +30,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -138,16 +139,19 @@ pub async fn create_voice_handler( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueCreateVoiceRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueCreateVoiceRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index eee83b0dcc..8eea038050 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -102,16 +102,19 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_session { + if user.is_banned { + return Err(InferW2lWithUploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(InferW2lWithUploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs new file mode 100644 index 0000000000..1bd1a0e50d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/create_weight.rs @@ -0,0 +1,119 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use mysql_queries::queries::model_weights::create_weight::{self, create_weight}; +use once_cell::sync::Lazy; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{error, info, warn}; + +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload}; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; + +#[derive(Serialize,ToSchema)] +pub struct UploadWeightsSuccessResponse { + pub success: bool, + pub upload_token: ModelWeightToken, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + "binary/octet-stream", + ]) +}); + +// pub async fn create_weight_handler( +// http_request: HttpRequest, +// server_state: web::Data>, +// mut multipart_payload: Multipart, +// ) -> Result { + + +// let weight_upload_token = response.to_model_token(); + +// let response: UploadWeightsSuccessResponse = UploadWeightsSuccessResponse { +// success: true, +// upload_token: weight_upload_token, +// }; + +// let body = serde_json::to_string(&response) +// .map_err(|e| UploadError::ServerError)?; + +// return Ok(HttpResponse::Ok() +// .content_type("application/json") +// .body(body)); +// } + +// pub enum SuccessCase { +// WeightAlreadyUploaded { +// existing_upload_token: ModelWeightToken, +// }, +// WeightSuccessfullyUploaded { +// upload_token: ModelWeightToken, +// } +// } + +// impl SuccessCase { +// pub fn to_model_token(self) -> ModelWeightToken { +// match self { +// SuccessCase::WeightAlreadyUploaded { existing_upload_token } => existing_upload_token, +// SuccessCase::WeightSuccessfullyUploaded { upload_token } => upload_token, +// } +// } +// } + +// let token = create_weight(CreateModelWeightsArgs { +// token: &ModelWeightToken::generate(), +// weights_type: WeightsType::LoRA, // to do parse multi part for this +// weights_category: WeightsCategory::ImageGeneration, +// title: upload_weights_request.file_name.as_deref(), +// maybe_thumbnail_token: &ThumbnailToken::generate(), +// description_markdown: "", +// description_rendered_html: "", +// creator_user_token: maybe_user_token.as_str(), +// creator_ip_address: &ip_address, +// creator_set_visibility, +// maybe_last_update_user_token: "", +// original_download_url: "", +// original_filename: upload_weights_request.file_name.as_deref(), +// file_size_bytes: file_size_bytes as i32, +// file_checksum_sha2: hash, +// private_bucket_hash: private_bucket_hash, +// maybe_private_bucket_prefix: None, +// maybe_private_bucket_extension: None, +// cached_user_ratings_total_count: 0, +// cached_user_ratings_positive_count: 0, +// cached_user_ratings_negative_count: 0, +// maybe_cached_user_ratings_ratio: None, +// version: 0, +// mysql_pool: &server_state.mysql_pool, +// }).await +// .map_err(|e| { +// error!("Error creating weights upload: {:?}", e); +// UploadError::ServerError +// })?; + +// info!("new weights upload id: {} token: {:?}", record_id, &token); + +// Ok() +// } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs new file mode 100644 index 0000000000..33f9ac75d7 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/delete_weight.rs @@ -0,0 +1,193 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, log, warn}; +use enums::by_table::audit_logs::audit_log_entity_action::AuditLogEntityAction::Delete; + +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::delete_weights::{ + delete_weights_as_user, + delete_weights_as_mod, + undelete_weights_as_user, + undelete_weights_as_mod +}; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; + +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + +use utoipa::ToSchema; + +#[derive(Deserialize,ToSchema)] +pub struct DeleteWeightPathInfo { + weight_token: String, +} + +#[derive(Deserialize,ToSchema)] +pub struct DeleteWeightRequest { + set_delete: bool, + as_mod: bool +} + + +// =============== Error Response =============== + +#[derive(Debug, Serialize,ToSchema)] +pub enum DeleteWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteWeightError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteWeightError::NotFound => StatusCode::NOT_FOUND, + DeleteWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + + +#[utoipa::path( + delete, + path = "/v1/weights/weight/{weight_token}", + responses( + (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = DeleteWeightError), + (status = 401, description = "Not authorized", body = DeleteWeightError), + (status = 500, description = "Server error", body = DeleteWeightError), + ), + params( + ("request" = DeleteWeightRequest, description = "Payload for Request"), + ("path" = DeleteWeightPathInfo, description = "Path for Request") + ) +)] +pub async fn delete_weight_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> + ) -> Result{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(DeleteWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to delete this weight: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + + let delete_role = delete_role_disambiguation(is_mod, is_creator, Some(request.as_mod)); + + let query_result = if request.set_delete { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_weights_as_mod( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + } else { + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to undelete weights: {}", user_session.user_token); + return Err(DeleteWeightError::NotAuthorized); + } + DeleteRole::AsUser => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_weights_as_user( + &ModelWeightToken::new_from_str(&path.weight_token), + &server_state.mysql_pool + ).await + } + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update weight mod approval status DB error: {:?}", err); + return Err(DeleteWeightError::ServerError); + } + }; + + Ok(simple_json_success()) + } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs new file mode 100644 index 0000000000..02be96fef3 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -0,0 +1,170 @@ +use std::fmt; +use std::sync::Arc; +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; +use enums::common::visibility::Visibility; +use tokens::tokens::users::UserToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +use tokens::tokens::model_weights::ModelWeightToken; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; + +use enums::by_table::model_weights::weights_types::WeightsType; +use enums::by_table::model_weights::weights_category::WeightsCategory; + +use utoipa::ToSchema; + +#[derive(Serialize, Clone, ToSchema)] +pub struct GetWeightResponse { + success: bool, + weight_token: ModelWeightToken, + title: String, + weights_type: WeightsType, + weights_category: WeightsCategory, + maybe_thumbnail_token: Option, + description_markdown: String, + description_rendered_html: String, + creator_set_visibility: Visibility, + + file_size_bytes: i32, + file_checksum_sha2: String, + + cached_user_ratings_negative_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_total_count: u32, + + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + version: i32, + created_at: DateTime, +} + +#[derive(Deserialize,ToSchema)] +pub struct GetWeightPathInfo { + weight_token: String, +} + +#[derive(Debug,ToSchema)] +pub enum GetWeightError { + NotAuthorized, + NotFound, + ServerError, +} + +impl fmt::Display for GetWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for GetWeightError { + fn status_code(&self) -> StatusCode { + match *self { + GetWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + GetWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetWeightError::NotFound => StatusCode::NOT_FOUND, + } + } +} +#[utoipa::path( + get, + path = "/v1/weights/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = GetWeightResponse), + (status = 400, description = "Bad input", body = GetWeightError), + (status = 401, description = "Not authorized", body = GetWeightError), + (status = 500, description = "Server error", body = GetWeightError), + ), + params( + ("path" = GetWeightPathInfo, description = "Path for Request") + ) + )] +pub async fn get_weight_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool, + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(GetWeightError::NotFound); + }, + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(GetWeightError::ServerError); + } + }; + + // if the weight is private, only the creator can view it + let is_private = weight.creator_set_visibility == Visibility::Private; + + if is_private && creator_user_token.as_str() != &user_session.user_token { + warn!("user is not allowed to view this weight: {}", user_session.user_token); + return Err(GetWeightError::NotAuthorized); + } + + + let response = GetWeightResponse { + success: true, + weight_token: weight.token, + title: weight.title, + weights_type: weight.weights_type, + weights_category: weight.weights_category, + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + creator_set_visibility: weight.creator_set_visibility, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, + version: weight.version, + created_at: weight.created_at + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetWeightError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs new file mode 100644 index 0000000000..f191e40410 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_available_weights.rs @@ -0,0 +1,273 @@ +use std::sync::Arc; +use actix_web::web::Query; +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{ DateTime, Utc }; +use log::{ info, warn }; +use rand::Rng; +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder; + +use enums::by_table::model_weights::{ + weights_types::WeightsType, + weights_category::WeightsCategory, +}; + +use tokens::tokens::users::UserToken; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::{UserDetailsLight,DefaultAvatarInfo}; +use crate::server_state::ServerState; + +use utoipa::ToSchema; + + +#[derive(Deserialize,ToSchema)] +pub struct ListAvailableWeightsQuery { + pub sort_ascending: Option, + pub page_size: u16, + pub page_index : u16, + pub username: Option, + pub weights_type: Option, + pub weights_category: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ListAvailableWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, + pub page_size: u16, + pub page_index : u16, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize,ToSchema)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub description_markdown: String, + pub description_rendered_html: String, + + pub creator_user_token: UserToken, + pub creator_set_visibility: Visibility, + + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + + pub version: i32, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + // additional fields to be added when tables are around + pub likes: u32, + pub bookmarks: bool, + +} + +#[derive(Debug,ToSchema)] +pub enum ListWeightError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListWeightError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/weights/list", + responses( + (status = 200, description = "List Weights", body = ListAvailableWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightError), + (status = 500, description = "Server error", body = ListWeightError), + ), + params( + ("request" = ListAvailableWeightsQuery, description = "Payload for Request"), + ) +)] +pub async fn list_available_weights_handler( + http_request: HttpRequest, + query: web::Query, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightError::ServerError + })?; + + let mut is_mod = false; + let user_session = match maybe_user_session { + Some(session) => { + is_mod = session.can_ban_users; + session + } + None => { + info!("not logged in"); + return Err(ListWeightError::NotAuthorized); + } + }; + + let limit = query.page_size; + + let sort_ascending = query.sort_ascending.unwrap_or(false); + + let include_user_hidden = is_mod; + + let mut weights_category: Option = None; + let mut weights_type: Option = None; + + if let Some(weights_category_string) = query.weights_category.as_ref() { + let result = WeightsCategory::from_str(&weights_category_string); + match result { + Ok(category) => { + weights_category = Some(category) + }, + Err(e) => { + warn!("invalid weights_category: {:?}", e); + weights_category = None + } + } + } + + if let Some(weights_type_string) = query.weights_type.as_ref() { + let result = WeightsType::from_str(&weights_type_string); + match result { + Ok(wtype) => { + weights_type = Some(wtype) + }, + Err(e) => { + warn!("invalid weights_type: {:?}", e); + weights_type = None + } + } + } + + let mut query_builder = ListWeightsQueryBuilder::new() + .sort_ascending(sort_ascending) + .weights_category(weights_category) + .weights_type(weights_type) + .scope_creator_username(None) + .include_user_hidden(include_user_hidden) + .include_user_deleted_results(is_mod) + .include_mod_deleted_results(is_mod) + .limit(limit) + .offset(Some((query.page_index as u64) * (query.page_size as u64))); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool).await; + + let weights_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListWeightError::ServerError); + } + }; + + let cursor_next = if let Some(id) = weights_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = weights_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ListWeightError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let mut rng = rand::thread_rng(); + let random_float = rng.gen_range(0.0..1.0); + let random_bool = random_float >= 0.5; + + // generate parse a response + let response = ListAvailableWeightsSuccessResponse { + success: true, + weights: weights_page.weights.into_iter() + .map(|weights| ModelWeightForList { + weight_token: weights.token, + title: weights.title, + weights_type: weights.weights_type, + weights_category: weights.weights_category, + + maybe_thumbnail_token:weights.maybe_thumbnail_token, + description_markdown: weights.description_markdown, + description_rendered_html: weights.description_rendered_html, + + creator_user_token: weights.creator_user_token, + creator_set_visibility: weights.creator_set_visibility, + + file_size_bytes:weights.file_size_bytes, + file_checksum_sha2: weights.file_checksum_sha2, + + cached_user_ratings_total_count: weights.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weights.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weights.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weights.maybe_cached_user_ratings_ratio, + + version: weights.version, + + created_at: weights.created_at, + updated_at: weights.updated_at, + + creator_username: weights.creator_username, + creator_display_name: weights.creator_display_name, + creator_email_gravatar_hash: weights.creator_email_gravatar_hash, + + // TODO: FIX THIS when we align again. + bookmarks: random_bool, + likes: rng.gen_range(0..1000), + }).collect::>(), + cursor_next, + cursor_previous, + page_index: query.page_index, + page_size: query.page_size + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs new file mode 100644 index 0000000000..364a14fdde --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_weights_by_user.rs @@ -0,0 +1,185 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::model_weights::list_weights_by_user::{list_weights_by_creator_username, WeightsJoinUserRecord}; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; +use utoipa::ToSchema; + +#[derive(Serialize, Clone,ToSchema)] +pub struct Weight { + weight_token: ModelWeightToken, + title: String, + + creator: UserDetailsLight, + creator_set_visibility: Visibility, + + maybe_thumbnail_token: Option, + + description_markdown: String, + description_rendered_html: String, + + file_size_bytes: i32, + file_checksum_sha2: String, + cached_user_ratings_total_count: u32, + cached_user_ratings_positive_count: u32, + cached_user_ratings_negative_count: u32, + maybe_cached_user_ratings_ratio: Option, + cached_user_ratings_last_updated_at: DateTime, + + created_at: DateTime, + updated_at: DateTime, +} + + +#[derive(Serialize,ToSchema)] +pub struct ListWeightsByUserSuccessResponse { + pub success: bool, + pub weights: Vec, +} + +#[derive(Deserialize,ToSchema)] +pub struct ListWeightsByUserPathInfo { + username: String, +} + +#[derive(Debug,ToSchema)] +pub enum ListWeightsByUserError { + NotAuthorized, + ServerError, +} + +impl fmt::Display for ListWeightsByUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListWeightsByUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListWeightsByUserError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListWeightsByUserError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/weights/by_user/{username}", + responses( + (status = 200, description = "List Weights by user", body = ListWeightsByUserSuccessResponse), + (status = 401, description = "Not authorized", body = ListWeightsByUserError), + (status = 500, description = "Server error", body = ListWeightsByUserError), + ), + params( + ("path" = ListWeightsByUserPathInfo, description = "Payload for Request"), + ) +)] +pub async fn list_weights_by_user_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListWeightsByUserError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListWeightsByUserError::NotAuthorized); + } + }; + + let username = path.username.as_ref(); + let creator_user_token = user_session.user_token.clone(); + let is_mod = user_session.can_ban_users; + + let query_results = list_weights_by_creator_username( + &server_state.mysql_pool, + &username, + is_mod, + ).await.map_err(|e| { + warn!("Error querying for weights: {:?}", e); + ListWeightsByUserError::ServerError + }); + + let weights = match query_results { + Ok(weights) => weights, + Err(e) => { + warn!("Error querying for weights: {:?}", e); + return Err(ListWeightsByUserError::ServerError); + } + }; + + let weights:Vec = weights.into_iter().map(|weight| { + Weight { + weight_token: weight.token, + title: weight.title, + creator: UserDetailsLight::from_db_fields( + &weight.creator_user_token, + &weight.creator_username, + &weight.creator_display_name, + &weight.creator_email_gravatar_hash, + ), + maybe_thumbnail_token: weight.maybe_thumbnail_token, + description_markdown: weight.description_markdown, + description_rendered_html: weight.description_rendered_html, + file_size_bytes: weight.file_size_bytes, + file_checksum_sha2: weight.file_checksum_sha2, + cached_user_ratings_total_count: weight.cached_user_ratings_total_count, + cached_user_ratings_positive_count: weight.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: weight.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at, + creator_set_visibility: weight.creator_set_visibility, + created_at: weight.created_at, + updated_at: weight.updated_at, + } + }).collect(); + + let final_weights:Vec; + + // if it's not the user ... then only show public weights else show private and public + if (creator_user_token != user_session.user_token) { + final_weights = weights.into_iter().filter(|weight| { + weight.creator_set_visibility == Visibility::Public + }).collect(); + + } + else { + final_weights = weights; + } + + + let response: ListWeightsByUserSuccessResponse = ListWeightsByUserSuccessResponse { + success: true, + weights: final_weights, + }; + + + let body = serde_json::to_string(&response) + .map_err(|e| ListWeightsByUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs new file mode 100644 index 0000000000..3fbd1cb09d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -0,0 +1,7 @@ +pub mod get_weight; +pub mod delete_weight; +pub mod update_weight; +pub mod list_weights_by_user; +pub mod list_available_weights; +// pub mod create_weight; this is a private api call internally +// pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs new file mode 100644 index 0000000000..8f69470054 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/update_weight.rs @@ -0,0 +1,163 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{ HttpRequest, HttpResponse, web }; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{ error, warn }; + +use enums::common::visibility::Visibility; +use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; +use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs }; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::server_state::ServerState; +use utoipa::ToSchema; +/// TODO will eventually be polymorphic +#[derive(Deserialize, ToSchema)] +pub struct UpdateWeightRequest { + pub title: Option, + pub thumbnail_token: Option, + pub description_markdown: Option, + pub description_rendered_html: Option, + pub weight_type: Option, + pub weight_category: Option, + pub visibility: Option, +} + +#[derive(Serialize, ToSchema)] +pub struct UpdateWeightResponse { + pub success: bool, +} + +/// For the URL PathInfo +#[derive(Deserialize,ToSchema)] +pub struct UpdateWeightPathInfo { + weight_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize, ToSchema)] +pub enum UpdateWeightError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for UpdateWeightError { + fn status_code(&self) -> StatusCode { + match *self { + UpdateWeightError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateWeightError::NotFound => StatusCode::NOT_FOUND, + UpdateWeightError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateWeightError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UpdateWeightError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== +#[utoipa::path( + post, + path = "/v1/weights/weight/{weight_token}", + responses( + (status = 200, description = "Success Update", body = SimpleGenericJsonSuccess), + (status = 400, description = "Bad input", body = UpdateWeightError), + (status = 401, description = "Not authorized", body = UpdateWeightError), + (status = 500, description = "Server error", body = UpdateWeightError), + ), + params( + ("request" = UpdateWeightRequest, description = "Payload for Request"), + ("path" = UpdateWeightPathInfo, description = "Path for Request") + ) + )] +pub async fn update_weight_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + let my_sql_pool = &server_state.mysql_pool; + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UpdateWeightError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateWeightError::NotAuthorized); + } + }; + + let weight_token = path.weight_token.clone(); + + // TODO wouldn't we want to instead use a function that will query the DB for the user and determine if they are a mod? + let is_mod = user_session.can_ban_users; + + let weight_lookup_result = get_weight_by_token( + &ModelWeightToken::new(weight_token.clone()), + is_mod, + &server_state.mysql_pool + ).await; + + let weight = match weight_lookup_result { + Ok(Some(weight)) => weight, + Ok(None) => { + warn!("Weight not found: {:?}", weight_token); + return Err(UpdateWeightError::NotFound); + } + Err(err) => { + warn!("Error looking up weight: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + }; + + let is_creator = weight.creator_user_token.to_string() == user_session.user_token; + + if !is_creator && !is_mod { + warn!("user is not allowed to edit this weight: {}", user_session.user_token); + return Err(UpdateWeightError::NotAuthorized); + } + + let query_result = update_weights(UpdateWeightArgs { + weight_token: &ModelWeightToken::new(path.weight_token.clone()), + mysql_pool: &server_state.mysql_pool, + title: request.title.as_deref(), + maybe_thumbnail_token: request.thumbnail_token.as_deref(), + description_markdown: request.description_markdown.as_deref(), + description_rendered_html: request.description_rendered_html.as_deref(), + creator_set_visibility: request.visibility.as_ref(), + weights_type: request.weight_type.as_deref().map(|s| s.to_string()), + weights_category: request.weight_category.as_deref().map(|s| s.to_string()), + }).await; + + match query_result { + Ok(_) => {} + Err(err) => { + warn!("Update Weight DB error: {:?}", err); + return Err(UpdateWeightError::ServerError); + } + } + + Ok(simple_json_success()) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs new file mode 100644 index 0000000000..8d2aa5fc5c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/upload_weight.rs @@ -0,0 +1,192 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{ HttpRequest, HttpResponse, web }; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use mysql_queries::queries::model_weights::create_weight::{ self, create_weight }; +use once_cell::sync::Lazy; + +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; +use crate::server_state::ServerState; + +use log::{ error, info, warn }; + +use buckets::public::weight_uploads::original_file::WeightUploadOriginalFilePath; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; + +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::get_media_upload_by_uuid::get_media_upload_by_uuid_with_connection; +use mysql_queries::queries::media_uploads::insert_media_upload::{ Args, insert_media_upload }; + +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{ + drain_multipart_request, + MediaSource, +}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use utoipa::ToSchema; + +use config::bad_urls::is_bad_model_weights_download_url; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::common::visibility::Visibility; +use utoipa::ToSchema; + +#[derive(Deserialize, ToSchema)] +pub struct UploadModelWeightsRequest { + idempotency_token: String, + title: String, + download_url: String, + model_weights_type: WeightsType, + model_weights_category: WeightsCategory, + creator_set_visibility: Option, +} + +/// Tell the frontend how to deal with the download queue. +#[derive(Serialize, ToSchema)] +pub enum DownloadJobType { + /// Modern shared download type (everything will use this in the + /// future, and this endpoint will die.) + #[serde(rename = "generic")] + Generic, +} + +#[derive(Serialize, ToSchema)] +pub struct UploadModelWeightsSuccessResponse { + pub success: bool, + /// This is how frontend clients can request the job execution status. + pub job_token: String, + /// This is a transitional field to tell the frontend how to process the job checking. + pub job_type: DownloadJobType, +} + +#[derive(Debug)] +pub enum UploadModelWeightsError { + BadInput(String), + MustBeLoggedIn, + ServerError, + RateLimited, +} + +impl ResponseError for UploadModelWeightsError { + fn status_code(&self) -> StatusCode { + match *self { + UploadModelWeightsError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadModelWeightsError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadModelWeightsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadModelWeightsError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + UploadModelWeightsError::BadInput(reason) => reason.to_string(), + UploadModelWeightsError::MustBeLoggedIn => "user must be logged in".to_string(), + UploadModelWeightsError::ServerError => "server error".to_string(), + UploadModelWeightsError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn upload_weights_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart +) -> Result { + // Rate limiter check + if + let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request( + &http_request + ) + { + return Err(UploadModelWeightsError::RateLimited); + } + + let maybe_user_session = server_state.session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool).await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UploadModelWeightsError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UploadModelWeightsError::MustBeLoggedIn); + } + }; + + if let Err(reason) = validate_idempotency_token(&request.idempotency_token) { + return Err(UploadModelWeightsError::BadInput(reason)); + } + + if let Err(reason) = validate_model_title(&request.title) { + return Err(UploadModelWeightsError::BadInput(reason)); + } + + match is_bad_model_weights_download_url(&download_url) { + Ok(false) => {} // Ok case + Ok(true) => { + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } + Err(err) => { + warn!("Error parsing url: {:?}", err); + return Err(UploadModelWeightsError::BadInput("Bad model download URL".to_string())); + } + } + + let supported_model_weights_type = request.model_weights_type; + + let job_token; + let download_job_type; + + let (download_job_token, _record_id) = insert_generic_download_job( + InsertGenericDownloadJobArgs { + uuid_idempotency_token: &uuid, + download_type: GenericDownloadType::Vits, + download_url: &download_url, + title: &title, + creator_user_token: &user_session.user_token, + creator_ip_address: &ip_address, + creator_set_visibility: visibility, // TODO: Creator set preference + mysql_pool: &server_state.mysql_pool, + } + ).await.map_err(|err| { + warn!("New generic download creation DB error (for generic model): {:?}", err); + UploadModelWeightsError::ServerError + })?; + + job_token = download_job_token.to_string(); + download_job_type = DownloadJobType::Generic; + + let response = UploadModelWeightsSuccessResponse { + success: true, + job_token: job_token.to_string(), + job_type: download_job_type, + }; + + let body = serde_json::to_string(&response).map_err(|_e| UploadModelWeightsError::ServerError)?; + + Ok(HttpResponse::Ok().content_type("application/json").body(body)) +} + +fn validate_idempotency_token(token: &str) -> Result<(), String> { + if token.len() != 36 { + return Err("idempotency token should be 36 characters".to_string()); + } + + Ok(()) +} diff --git a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs index dae959059f..16908c9ac9 100644 --- a/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs +++ b/crates/service/web/storyteller_web/src/http_server/web_utils/response_success_helpers.rs @@ -1,6 +1,6 @@ use actix_web::HttpResponse; - -#[derive(Serialize)] +use utoipa::ToSchema; +#[derive(Serialize,ToSchema)] pub struct SimpleGenericJsonSuccess { pub success: bool, } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index e77ed85093..57f09a82c8 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -75,6 +75,7 @@ use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStr use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; @@ -173,65 +174,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, ); - info!("Setting up Redis rate limiters..."); - - // Old env vars: - // - // "LIMITER_ENABLED" - // "LIMITER_MAX_REQUESTS" - // "LIMITER_WINDOW_SECONDS" - // - let logged_out_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) - }; - - let logged_in_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) - }; - - let api_high_priority_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) - }; - - let model_upload_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) - }; info!("Connecting to elasticsearch..."); @@ -466,12 +408,7 @@ async fn main() -> AnyhowResult<()> { elasticsearch, redis_pool, redis_ttl_cache, - redis_rate_limiters: RedisRateLimiters { - logged_out: logged_out_redis_rate_limiter, - logged_in: logged_in_redis_rate_limiter, - api_high_priority: api_high_priority_redis_rate_limiter, - model_upload: model_upload_rate_limiter, - }, + redis_rate_limiters: configure_redis_rate_limiters(&common_env)?, firehose_publisher, badge_granter, avt_cookie_manager, diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 96be089846..26ffb503bc 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -11,7 +11,7 @@ use users_component::default_routes::add_suggested_api_v1_account_creation_and_s use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; -use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler; +use crate::http_server::endpoints::animation::enqueue_face_animation::enqueue_face_animation_handler; use crate::http_server::endpoints::animation::enqueue_rerender_animation::enqueue_rerender_animation_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; @@ -29,11 +29,11 @@ use crate::http_server::endpoints::comments::list_comments_handler::list_comment use crate::http_server::endpoints::download_job::enqueue_generic_download::enqueue_generic_download_handler; use crate::http_server::endpoints::download_job::get_generic_upload_job_status::get_generic_download_job_status_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; -use crate::http_server::endpoints::favorites::create_favorite_handler::create_favorite_handler; -use crate::http_server::endpoints::favorites::delete_favorite_handler::delete_favorite_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_entity_handler::list_favorites_for_entity_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_session_handler::list_favorites_for_session_handler; -use crate::http_server::endpoints::favorites::list_favorites_for_user_handler::list_favorites_for_user_handler; +use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler; use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; @@ -153,6 +153,13 @@ use crate::http_server::endpoints::w2l::list_user_w2l_templates::list_user_w2l_t use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::weights::get_weight::get_weight_handler; +use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; +use crate::http_server::endpoints::weights::update_weight::update_weight_handler; +use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; +use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; + + pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where B: MessageBody, @@ -185,6 +192,9 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> app = add_subscription_routes(app); /* /v1/subscriptions/... */ app = add_voice_designer_routes(app); /* /v1/voice_designer */ + if server_environment == ServerEnvironment::Development { + app = add_weights_routes(app); /* /v1/stubs/... */ + } // ==================== Comments ==================== let mut app = RouteBuilder::from_app(app) @@ -193,20 +203,20 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> .add_post("/v1/comments/delete/{comment_token}", delete_comment_handler) .into_app(); - // ==================== Favorites ==================== + // ==================== User Bookmarks ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/favorites/create", create_favorite_handler) - .add_post("/v1/favorites/delete/{favorite_token}", delete_favorite_handler) - .add_get("/v1/favorites/list/session", list_favorites_for_session_handler) - .add_get("/v1/favorites/list/user/{username}", list_favorites_for_user_handler) - .add_get("/v1/favorites/list/entity/{entity_type}/{entity_token}", list_favorites_for_entity_handler) + .add_post("/v1/user_bookmarks/create", create_user_bookmark_handler) + .add_post("/v1/user_bookmarks/delete/{user_bookmark_token}", delete_user_bookmark_handler) + .add_get("/v1/user_bookmarks/list/session", list_user_bookmarks_for_session_handler) + .add_get("/v1/user_bookmarks/list/user/{username}", list_user_bookmarks_for_user_handler) + .add_get("/v1/user_bookmarks/list/entity/{entity_type}/{entity_token}", list_user_bookmarks_for_entity_handler) .into_app(); // ==================== Animations ==================== let mut app = RouteBuilder::from_app(app) - .add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler) + .add_post("/v1/animation/face_animation/create", enqueue_face_animation_handler) .add_post("/v1/animation/rerender/create", enqueue_rerender_animation_handler) .into_app(); @@ -1233,3 +1243,33 @@ fn add_voice_designer_routes (app:App)-> App ) } +// ==================== Weights ROUTES ==================== +fn add_weights_routes(app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = () + > +{ + app.service( + web + ::scope("/v1/weights") + //.route("/upload", web::post().to(upload_weights_handler)) + .service( + web + ::resource("/weight/{weight_token}") + .route(web::get().to(get_weight_handler)) + .route(web::post().to(update_weight_handler)) + .route(web::delete().to(delete_weight_handler)) + ) + .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) + .route( + "/list", + web::get().to(list_available_weights_handler) + ) + ) +} diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index 41ddadf1d3..c7ba601993 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -20,6 +20,7 @@ use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; +use crate::configs::app_startup::username_set::UsernameSet; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo; @@ -140,8 +141,20 @@ pub struct RedisRateLimiters { /// constructions) pub api_high_priority: RedisRateLimiter, + /// API rate limit for AI streamers + pub api_ai_streamers: RedisRateLimiter, + + /// Usernames of AI streamers + pub api_ai_streamer_username_set: UsernameSet, + /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_out: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_in: RedisRateLimiter, } /// In-memory caches of several types. diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 2d29dcc45b..20c89b006b 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" } redis_common = { path = "../../../lib/redis_common" } twitch_common = { path = "../../../lib/api_clients/twitch_common" } +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + # External anyhow = "1.0.75" base64 = "0.21.0" @@ -40,7 +43,7 @@ uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros", "time"] } # Futures futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool @@ -89,9 +92,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" -# Temporary (just to ask for oauth credentials during dev) -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } - [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "2.1.3", features = ["mysql"] } diff --git a/crates/vendor/limitation/Cargo.toml b/crates/vendor/limitation/Cargo.toml index 86fb33b371..d13d4d9d2d 100644 --- a/crates/vendor/limitation/Cargo.toml +++ b/crates/vendor/limitation/Cargo.toml @@ -23,6 +23,9 @@ futures = "0.1.29" redis = "0.13.0" time = "0.1.42" +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + [dev-dependencies] -tokio = "0.1.22" -version-sync = "0.8.1" +tokio = "1.34.0" +version-sync = "0.9.5" diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs index 649b30eb46..b40cf6b1fe 100644 --- a/crates/vendor/limitation/src/lib.rs +++ b/crates/vendor/limitation/src/lib.rs @@ -42,9 +42,12 @@ //! // The key to count and track //! let key = "10.0.0.5"; //! +//! // TODO(bt,2023-12-12): This doctest is so far behind in `futures` versions and needs to be updated. +//! // Tokio should be executing the future and evaluating the result. //! // Start and run a Tokio runtime to drive the Future to completion -//! tokio::run( -//! limiter +//! //let mut rt = tokio::runtime::Runtime::new().unwrap(); +//! //rt.block_on(...)//! +//! let _r = limiter //! // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` //! // containing a Status if the limit has been exceeded //! .count(key) @@ -57,8 +60,9 @@ //! .and_then(|status| { //! println!("ok: {:?}", status); //! Ok(()) -//! }), -//! ); +//! }) +//! .wait(); +//! //); //! # Ok::<(), limitation::Error>(()) //! ``` //! diff --git a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index d0e3422320..34a9823ef0 100644 --- a/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -16,11 +16,13 @@ serde = { version = "1.0.57", features = ["derive"] } serde_json = "1.0.108" hyper = { version = "0.14", features = ["http1"], optional = true } -hyper-tls = { version = "0.5.0", optional = true } +hyper-tls = { version = "0.6.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } -tokio = { version = "1.1.0", features = ["macros"], optional = true } +tokio = { version = "1.34.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } + +# Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } [dev-dependencies] @@ -28,4 +30,4 @@ rand = "0.8.5" env_logger = "0.10.1" futures = { version = "0.3.1" } hyper = { version = "0.14", features = ["http1", "server"] } -tokio = { version = "1.1.0", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } diff --git a/execute_tests.sh b/execute_tests.sh index 319503eb31..ad5219c463 100755 --- a/execute_tests.sh +++ b/execute_tests.sh @@ -13,6 +13,7 @@ cargo test \ --exclude billing_component \ --exclude fakeyou_client \ --exclude inference-job \ + --exclude mysql_queries \ --exclude sqlite_queries # Single broken tests @@ -22,4 +23,9 @@ cargo test \ --skip util::audiowmark::tests::test_audiowmark \ --skip util::common_commands::audiowmark::tests::test_audiowmark +cargo test \ + --package mysql_queries \ + -- \ + --skip queries::model_weights::tests + echo 'Tests passed.'