From da15968c4ee7354f455f7956b7110313cf19118f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:00:56 +0000 Subject: [PATCH 01/40] Bump openssl in /old_code/twitch-ingestion-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.32 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.32...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- old_code/twitch-ingestion-client/Cargo.lock | 37 +++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/old_code/twitch-ingestion-client/Cargo.lock b/old_code/twitch-ingestion-client/Cargo.lock index d0940e310..2f7105417 100644 --- a/old_code/twitch-ingestion-client/Cargo.lock +++ b/old_code/twitch-ingestion-client/Cargo.lock @@ -203,6 +203,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "blocking" version = "1.0.2" @@ -420,7 +426,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" dependencies = [ - "bitflags", + "bitflags 1.2.1", "fuchsia-zircon-sys", ] @@ -867,18 +873,30 @@ checksum = "4ad167a2f54e832b82dbe003a046280dceffe5227b5f79e08e363a29638cfddd" [[package]] name = "openssl" -version = "0.10.32" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if 1.0.0", "foreign-types", - "lazy_static", "libc", + "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.2" @@ -887,11 +905,10 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.60" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1122,7 +1139,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" dependencies = [ - "bitflags", + "bitflags 1.2.1", ] [[package]] @@ -1217,7 +1234,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6af1b6204f89cf0069736daf8b852573e3bc34898eee600e95d3dd855c12e81" dependencies = [ - "bitflags", + "bitflags 1.2.1", "core-foundation", "core-foundation-sys", "libc", From 0128134d3d86056a57a713ddf1c5ba18879c044e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:02:17 +0000 Subject: [PATCH 02/40] Bump openssl from 0.10.33 to 0.10.60 in /old_code/pubsub-enricher Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.33 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.33...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- old_code/pubsub-enricher/Cargo.lock | 35 +++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/old_code/pubsub-enricher/Cargo.lock b/old_code/pubsub-enricher/Cargo.lock index 9abc97549..f9b9184af 100644 --- a/old_code/pubsub-enricher/Cargo.lock +++ b/old_code/pubsub-enricher/Cargo.lock @@ -205,6 +205,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "blocking" version = "1.0.2" @@ -464,7 +470,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" dependencies = [ - "bitflags", + "bitflags 1.2.1", "fuchsia-zircon-sys", ] @@ -1031,18 +1037,30 @@ checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "openssl" -version = "0.10.33" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.2" @@ -1051,11 +1069,10 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.61" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1337,7 +1354,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" dependencies = [ - "bitflags", + "bitflags 1.2.1", ] [[package]] @@ -1481,7 +1498,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ - "bitflags", + "bitflags 1.2.1", "core-foundation", "core-foundation-sys", "libc", From 93ebe101f34c510a29d59e9d3e74a9c0ef18a3d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:03:53 +0000 Subject: [PATCH 03/40] Bump openssl in /old_code/twitter-ingestion-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.34 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.34...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- old_code/twitter-ingestion-client/Cargo.lock | 35 +++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/old_code/twitter-ingestion-client/Cargo.lock b/old_code/twitter-ingestion-client/Cargo.lock index 7c8b31c91..ca62dba89 100644 --- a/old_code/twitter-ingestion-client/Cargo.lock +++ b/old_code/twitter-ingestion-client/Cargo.lock @@ -193,6 +193,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.9.0" @@ -505,7 +511,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" dependencies = [ - "bitflags", + "bitflags 1.2.1", "fuchsia-zircon-sys", ] @@ -1063,18 +1069,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.34" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.2" @@ -1083,11 +1101,10 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.63" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1379,7 +1396,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ - "bitflags", + "bitflags 1.2.1", ] [[package]] @@ -1468,7 +1485,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ - "bitflags", + "bitflags 1.2.1", "core-foundation", "core-foundation-sys", "libc", From a3ab6347255c1bbf993329be90087a506ca5fdb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:11:08 +0000 Subject: [PATCH 04/40] Bump openssl in /old_code/twitch-tts-desktop-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.36 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.36...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- old_code/twitch-tts-desktop-client/Cargo.lock | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/old_code/twitch-tts-desktop-client/Cargo.lock b/old_code/twitch-tts-desktop-client/Cargo.lock index 959c7d072..1e8538ee1 100644 --- a/old_code/twitch-tts-desktop-client/Cargo.lock +++ b/old_code/twitch-tts-desktop-client/Cargo.lock @@ -77,6 +77,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.9.0" @@ -645,18 +651,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.36" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.4" @@ -665,11 +683,10 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.67" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -808,7 +825,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -975,7 +992,7 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", From 133ccff348ba127ca52f7300b4107f105148722f Mon Sep 17 00:00:00 2001 From: kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:12:22 +0530 Subject: [PATCH 05/40] update media file visibilty handler --- .../http_server/endpoints/media_files/mod.rs | 1 + .../media_files/update_media_file.rs | 154 ++++++++++++++++++ .../service/web/storyteller_web/src/routes.rs | 5 + 3 files changed, 160 insertions(+) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/update_media_file.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 014fa6885..4f010c38f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,3 +1,4 @@ pub mod drain_multipart_request; pub mod get_media_file; pub mod delete_media_file; +pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/update_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/update_media_file.rs new file mode 100644 index 000000000..4d099b8bb --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/update_media_file.rs @@ -0,0 +1,154 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use log::{error, log, warn}; +use enums::common::visibility::Visibility; + +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_files::update_media_file::{update_media_file, UpdateMediaFileArgs}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct UpdateMediaFileRequest { + pub creator_set_visibility: Option, +} + +#[derive(Serialize)] +pub struct UpdateMediaFileResponse { + pub success: bool, +} + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct UpdateMediaFilePathInfo { + token: MediaFileToken, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum UpdateMediaFileError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for UpdateMediaFileError { + fn status_code(&self) -> StatusCode { + match *self { + UpdateMediaFileError::BadInput(_) => StatusCode::BAD_REQUEST, + UpdateMediaFileError::NotFound => StatusCode::NOT_FOUND, + UpdateMediaFileError::NotAuthorized => StatusCode::UNAUTHORIZED, + UpdateMediaFileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UpdateMediaFileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn update_media_file_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UpdateMediaFileError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UpdateMediaFileError::NotAuthorized); + } + }; + + let media_file_token = path.token.clone(); + let is_mod = user_session.can_ban_users; + + let media_file_lookup_result = get_media_file( + &path.token, + is_mod, + &server_state.mysql_pool, + ).await; + + let media_file = match media_file_lookup_result { + Ok(Some(media_file)) => media_file, + Ok(None) => { + warn!("MediaFile not found: {:?}", media_file_token); + return Err(UpdateMediaFileError::NotFound); + }, + Err(err) => { + warn!("Error looking up media_file: {:?}", err); + return Err(UpdateMediaFileError::ServerError); + } + }; + + let is_creator = media_file.maybe_creator_user_token + .is_some_and(|t| t.as_str() == &user_session.user_token); + + if !is_creator && !is_mod { + warn!("user is not allowed to edit this media_file: {}", user_session.user_token); + return Err(UpdateMediaFileError::NotAuthorized); + } + + let mut creator_set_visibility = Visibility::Public; + + + if let Some(visibility) = request.creator_set_visibility.as_deref() { + creator_set_visibility = Visibility::from_str(visibility) + .map_err(|_| UpdateMediaFileError::BadInput("bad record visibility".to_string()))?; + } + + let ip_address = get_request_ip(&http_request); + let mut maybe_mod_user_token = None; + + if is_mod { + maybe_mod_user_token = Some(user_session.user_token.clone()); + } + let query_result = update_media_file( + UpdateMediaFileArgs { + media_file_token: &media_file_token.clone(), + creator_set_visibility: &creator_set_visibility, + maybe_mod_user_token: maybe_mod_user_token.as_deref(), + mysql_pool: &server_state.mysql_pool + } + ).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update MediaFile DB error: {:?}", err); + return Err(UpdateMediaFileError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9b6f92006..c1cdc543e 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -43,6 +43,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::update_media_file::update_media_file_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -1000,6 +1001,10 @@ fn add_media_file_routes (app: App) -> App .route(web::delete().to(delete_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/file/{token}/update") + .route(web::post().to(update_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } From 8bb82c0fe5c9b598211bc10065593760765008d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:59:57 +0000 Subject: [PATCH 06/40] Bump async-stripe from 0.25.2 to 0.26.0 Bumps [async-stripe](https://github.com/arlyon/async-stripe) from 0.25.2 to 0.26.0. - [Release notes](https://github.com/arlyon/async-stripe/releases) - [Changelog](https://github.com/arlyon/async-stripe/blob/master/CHANGELOG.md) - [Commits](https://github.com/arlyon/async-stripe/compare/v0.25.2...v0.26.0) --- updated-dependencies: - dependency-name: async-stripe dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25191d742..9d10ab3e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -605,9 +605,9 @@ dependencies = [ [[package]] name = "async-stripe" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b39347f478f57a40cbaf82cbaebb4a350acc757cda30fc7ff88707a11d1d81" +checksum = "19a0b12cb3b2df9abfaa381632e4d55b2aede9063034daaa0d0145413719a7e1" dependencies = [ "chrono", "futures-util", diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 869eee779..72b2ded75 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -45,7 +45,7 @@ serde_json = "1.0.108" toml = "0.5.8" # Money -async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] } +async-stripe = { version = "0.26.0", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] } # Async traits # NB: note: `async` trait functions are not currently supported diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index c8d81cef9..357c79c99 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -139,7 +139,7 @@ jwt = "0.13.0" hyper = { version = "0.14.24" } # Money -async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] } +async-stripe = { version = "0.26.0", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] } # Crypto bcrypt = "0.8" # password hashes From 2d59c5c573ff4dae82705268517e41b7a88b4648 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 13:47:12 -0800 Subject: [PATCH 07/40] initial commit --- .env | 78 ----- Cargo.lock | 2 + .../inference_category.rs | 4 + .../inference_model_type.rs | 2 + .../generic_inference_args.rs | 10 + .../payloads/generic_inference_args/mod.rs | 3 +- .../videofilter_payload.rs | 36 ++ crates/service/job/inference_job/Cargo.toml | 2 + .../inference_job/src/job/job_types/mod.rs | 3 +- .../src/job/job_types/videofilter/mod.rs | 1 + .../rerender_a_video/categorize_error.rs | 5 + .../rerender_a_video/download_video_file.rs | 83 +++++ .../videofilter/rerender_a_video/mod.rs | 6 + .../rerender_a_video/process_jobs.rs | 316 ++++++++++++++++++ .../rerender_a_video/rerender_dependencies.rs | 15 + .../rerender_inference_command.rs | 233 +++++++++++++ .../rerender_a_video/validate_job.rs | 85 +++++ .../src/job_specific_dependencies.rs | 9 + .../service/web/storyteller_web/src/routes.rs | 1 + 19 files changed, 814 insertions(+), 80 deletions(-) delete mode 100644 .env create mode 100644 crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs diff --git a/.env b/.env deleted file mode 100644 index b205c3a6b..000000000 --- a/.env +++ /dev/null @@ -1,78 +0,0 @@ -SERVER_ENVIRONMENT=development - -DATABASE_URL=mysql://storyteller:password@localhost/storyteller -#DATABASE_URL=mysql://root:root@localhost/storyteller -#DATABASE_URL=mysql://root@localhost/storyteller - -# Diesel-cli reads this for the migration directory to use. -# We've renamed it so the migrations are sorted near the top of the project directory tree -# (since they kind of serve as documentation) -MIGRATION_DIRECTORY=_sql/migrations/ - -# SQLX normally requires a live database to reflect upon at build time. -# This setting turns that on and requires a `sql-data.json` cache file instead. -# See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme -#SQLX_OFFLINE=true - -# NB(bt, 2023-03-20): -# These are used in storyteller-web, image-generation-service, and websocket-gateway. -# I'm moving them to env configs that live under storyteller-web/.env. -# This will break the other services for now. -# # We can't serve HTTPS locally without self-signed certificate nonsense. -# COOKIE_SECURE=false -# COOKIE_DOMAIN=.jungle.horse -# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse - -# Bucket config (non-secret) -REGION_NAME=us-east1 - -TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads -TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models - -W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads -W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public - -PRIVATE_BUCKET_NAME=dev-vocodes-private-uploads -PUBLIC_BUCKET_NAME=dev-vocodes-public - -# Bucket roots -AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio -W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates -W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output - -W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip -W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py -W2L_MODEL_FILENAME=wav2lip_gan.pth - -#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth -W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth -W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 - -W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py - -TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py - -TACOTRON_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TACOTRON_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" -TACOTRON_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_tacotron.py" - -TALKNET_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -TALKNET_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_talknet.py" - -HIFIGAN_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts -HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate" -HIFIGAN_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_hifigan.py" - -TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 - -#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID -#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! -TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt - -TTS_HIFIGAN_VOCODER_MODEL_FILENAME=hifigan_universal_g_02500000.pt -TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt - -# Development key. Permissible to check in. -NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index bec222b29..d8fddc06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3063,6 +3063,7 @@ dependencies = [ "log", "media", "memory_caching", + "mime", "mimetypes", "my-workspace-hack", "mysql_queries", @@ -3077,6 +3078,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", + "time 0.3.30", "tokens", "tokio 1.34.0", "tts_common", diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index b38f18146..b96b6fb26 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -25,6 +25,9 @@ pub enum InferenceCategory { #[serde(rename = "voice_conversion")] VoiceConversion, + + #[serde(rename = "rerender_a_video")] + RerenderAVideo, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -38,6 +41,7 @@ impl InferenceCategory { Self::LipsyncAnimation => "lipsync_animation", Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", + Self::RerenderAVideo => "rerender_a_video", } } diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 2a939038d..5401430bd 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -31,6 +31,8 @@ pub enum InferenceModelType { Vits, #[serde(rename = "vall_e_x")] VallEX, + #[serde(rename = "rerender_a_video")] + RerenderAVideo, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 541de77aa..7ba7b0b3e 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -3,6 +3,7 @@ use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; use crate::payloads::generic_inference_args::tts_payload::TTSArgs; +use crate::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; /// Used to encode extra state for the `generic_inference_jobs` table. @@ -35,6 +36,10 @@ pub enum InferenceCategoryAbbreviated { #[serde(rename = "vc")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(alias = "voice_conversion")] VoiceConversion, + + #[serde(rename = "rr")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "rerender_a_video")] + RerenderAVideo, } #[derive(Serialize, Deserialize, Clone, Copy, Debug)] @@ -84,6 +89,9 @@ pub enum PolymorphicInferenceArgs { #[serde(skip_serializing_if = "Option::is_none")] transpose: Option, }, + + /// Rerender a video. (Short name to save space when serializing.) + Rr(RerenderArgs) } @@ -104,6 +112,7 @@ impl InferenceCategoryAbbreviated { InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, + InferenceCategory::RerenderAVideo => Self::RerenderAVideo, } } @@ -112,6 +121,7 @@ impl InferenceCategoryAbbreviated { Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, + Self::RerenderAVideo => InferenceCategory::RerenderAVideo, } } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs index 37980d38d..e68fc5a71 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs @@ -1,3 +1,4 @@ pub mod generic_inference_args; pub mod lipsync_payload; -pub mod tts_payload; \ No newline at end of file +pub mod tts_payload; +pub mod videofilter_payload; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs new file mode 100644 index 000000000..85335cde7 --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -0,0 +1,36 @@ +/// Video sources can be one of several: +/// - F: media_files (todo) +/// - U: media_uploads (legacy) +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum VideofilterVideoSource { + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media File Token (media_files table) + /// Serde cannot yet rename enum variants. + F(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media Upload Token (media_uploads table) + /// Serde cannot yet rename enum variants. + U(String), +} + + +impl VideofilterVideoSource { + pub fn media_file_token(token: &str) -> Self { + VideofilterVideoSource::F(token.to_string()) + } + pub fn media_upload_token(token: &str) -> Self { + VideofilterVideoSource::U(token.to_string()) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct RerenderArgs { + #[serde(rename = "vs")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_video_source: Option, + + #[serde(rename = "sd")] + #[serde(skip_serializing_if = "Option::is_none")] + pub sd_model: String, +} diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index d0008b9ea..65713e40b 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -79,3 +79,5 @@ serde_json = "1.0.108" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +time = "0.3.30" +mime = "0.3.17" diff --git a/crates/service/job/inference_job/src/job/job_types/mod.rs b/crates/service/job/inference_job/src/job/job_types/mod.rs index f21a93a49..a688e281b 100644 --- a/crates/service/job/inference_job/src/job/job_types/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/mod.rs @@ -1,3 +1,4 @@ pub mod lipsync; pub mod tts; -pub mod vc; \ No newline at end of file +pub mod vc; +pub(crate) mod videofilter; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs new file mode 100644 index 000000000..b8f301c73 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -0,0 +1 @@ +pub(crate) mod rerender_a_video; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs new file mode 100644 index 000000000..a1effe90c --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/categorize_error.rs @@ -0,0 +1,5 @@ +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +pub fn categorize_error(stderr_contents: &str) -> Option { + None +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs new file mode 100644 index 000000000..344592bd0 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -0,0 +1,83 @@ +use std::path::PathBuf; + +use anyhow::anyhow; +use log::{error, warn}; +use sqlx::MySqlPool; +use tempdir::TempDir; + +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use cloud_storage::bucket_client::BucketClient; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; + +pub struct VideoFile { + pub filesystem_path: PathBuf, +} + +pub async fn download_video_file( + video_source: &VideofilterVideoSource, + public_bucket_client: &BucketClient, + job_progress_reporter: &mut Box, + job: &AvailableInferenceJob, + temp_dir_creator: &ScopedTempDirCreator, + work_temp_dir: &TempDir, + mysql_pool: &MySqlPool, +) -> Result { + + let bucket_object_path; + + match video_source { + VideofilterVideoSource::F(_media_file_token) => { + // TODO(bt, 2023-09-08): Implement + return Err(ProcessSingleJobError::NotYetImplemented) + } + VideofilterVideoSource::U(media_upload_token) => { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + } + } + + let downloaded_filesystem_path = work_temp_dir.path().join("video.mp4"); + + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "video file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, + job_progress_reporter, + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; + + Ok(VideoFile { + filesystem_path: downloaded_filesystem_path, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs new file mode 100644 index 000000000..152c2840d --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -0,0 +1,6 @@ +mod process_jobs; +mod validate_job; +mod download_video_file; +mod rerender_inference_command; +pub(crate) mod rerender_dependencies; +mod categorize_error; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs new file mode 100644 index 000000000..f26de327c --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs @@ -0,0 +1,316 @@ +use std::fs::read_to_string; +use std::time::Instant; + +use anyhow::anyhow; +use log::{error, info, warn}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; +use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; +use tokens::tokens::users::UserToken; + +use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; +// use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; +use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; +use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; +use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; +use crate::job_dependencies::JobDependencies; +use crate::util::common_commands::ffmpeg_logo_watermark_command; + +/// The maximum that either width or height can be +const MAX_DIMENSION : u32 = 1500; + +const BUCKET_FILE_PREFIX: &str = "fakeyou_"; +const BUCKET_FILE_EXTENSION: &str = ".mp4"; + +pub struct RerenderProcessJobArgs<'a> { + pub job_dependencies: &'a JobDependencies, + pub job: &'a AvailableInferenceJob, +} + +pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { + let job = args.job; + let deps = args.job_dependencies; + + let mut job_progress_reporter = args.job_dependencies + .clients + .job_progress_reporter + .new_generic_inference(job.inference_job_token.as_str()) + .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; + + let model_dependencies = args + .job_dependencies + .job + .job_specific_dependencies + .maybe_rerender_dependencies + .as_ref() + .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing Rerender dependencies".to_string())))?; + + // ==================== UNPACK + VALIDATE INFERENCE ARGS ==================== // + + let job_args = validate_job(job)?; + + // ==================== TEMP DIR ==================== // + + let work_temp_dir = format!("temp_rerender_inference_{}", job.id.0); + + // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem. + let work_temp_dir = args.job_dependencies + .fs + .scoped_temp_dir_creator_for_work + .new_tempdir(&work_temp_dir) + .map_err(|e| ProcessSingleJobError::from_io_error(e))?; + + + // ==================== QUERY AND DOWNLOAD FILES ==================== // + + let video_path = download_video_file( + &job_args.video_source, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir, + &deps.db.mysql_pool + ).await?; + + info!("Downloaded video file: {:?}", image_path.filesystem_path); + + // ==================== TRANSCODE MEDIA (IF NECESSARY) ==================== // + + let mut usable_video_path = video_path.filesystem_path.clone(); + // + // if let Some((width, height)) = job_args.width.zip(job_args.height) { + // info!("Requested image resize to {}x{} ...", width, height); + // + // let mut width = width.min(MAX_DIMENSION); + // let mut height = height.min(MAX_DIMENSION); + // + // if job_args.enhancer.is_some() { + // // The enhancer will double the size of the image, so we'll downsize the image by half + // // if it's enabled. + // width = width / 2; + // height = height / 2; + // info!("Enhancer will be doubling image size, so cutting frame size to {}x{}.", width, height) + // } + // + // info!("Resizing image to {}x{} ...", width, height); + // + // let result = resize_image( + // &image_path.filesystem_path, + // &work_temp_dir, + // width, + // height + // ); + // + // match result { + // Err(err) => { + // error!("Image resize failure: {:?}", err); + // safe_delete_temp_file(&audio_path.filesystem_path); + // safe_delete_temp_file(&image_path.filesystem_path); + // safe_delete_temp_directory(&work_temp_dir); + // return Err(ProcessSingleJobError::Other(anyhow!("image resize failure: {:?}", err))); + // } + // Ok(resized_image_path) => { + // usable_image_path = resized_image_path; + // } + // } + // } + + info!("Used video file: {:?}", usable_video_path); + + // ==================== SETUP FOR INFERENCE ==================== // + + info!("Ready for Rerender A Video inference..."); + + job_progress_reporter.log_status("running inference") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let output_video_fs_path = work_temp_dir.path().join("blend.mp4"); + + info!("Running Rerender A Video inference..."); + + info!("Expected output video filename: {:?}", &output_video_fs_path); + + // TODO: Limit output length for non-premium (???) + + let maybe_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + // ==================== WRITE CONFIG TO DISK ==================== // + + let config_path = work_temp_dir.path().join("config.json"); + + // TODO + + // ==================== RUN INFERENCE SCRIPT ==================== // + + let workdir = work_temp_dir.path().to_path_buf(); + let stderr_output_file = work_temp_dir.path().join("stderr.txt"); + let inference_start_time = Instant::now(); + + let command_exit_status = model_dependencies + .inference_command + .execute_inference(InferenceArgs { + config_file: &config_path, + stderr_output_file: &stderr_output_file, + }); + + let inference_duration = Instant::now().duration_since(inference_start_time); + + info!("Inference took duration to complete: {:?}", &inference_duration); + + if !command_exit_status.is_success() { + error!("Inference failed: {:?}", command_exit_status); + + let mut error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + + // match categorize_error(&contents) { + // Some(ProcessSingleJobError::FaceDetectionFailure) => { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + // } + } + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&config_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + return Err(error); + } + + // ==================== CHECK NON-WATERMARKED RESULT ==================== // + + info!("Checking that output file exists: {:?} ...", output_video_fs_path); + + check_file_exists(&output_video_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== OPTIONAL WATERMARK ==================== // + + let mut finished_file = output_video_fs_path.clone(); + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that output watermark file exists: {:?} ...", finished_file); + check_file_exists(&finished_file).map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Interrogating result file size ..."); + + let file_size_bytes = file_size(&finished_file) + .map_err(|err| ProcessSingleJobError::Other(err))?; + + info!("Interrogating result mimetype ..."); + + let mimetype = get_mimetype_for_file(&finished_file) + .map_err(|err| ProcessSingleJobError::from_io_error(err))? + .map(|mime| mime.to_string()) + .ok_or(ProcessSingleJobError::Other(anyhow!("Mimetype could not be determined")))?; + + info!("Calculating sha256..."); + + let file_checksum = sha256_hash_file(&finished_file) + .map_err(|err| { + ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err)) + })?; + + // ==================== UPLOAD AUDIO TO BUCKET ==================== // + + job_progress_reporter.log_status("uploading result") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let result_bucket_location = MediaFileBucketPath::generate_new( + Some(BUCKET_FILE_PREFIX), + Some(BUCKET_FILE_EXTENSION)); + + let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf(); + + info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf); + + info!("Uploading media ..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &result_bucket_object_pathbuf, + &finished_file, + &mimetype) // TODO: We should check the mimetype to make sure bad payloads can't get uploaded + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== DELETE TEMP FILES ==================== // + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&config_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&work_temp_dir); + + // ==================== SAVE RECORDS ==================== // + + info!("Saving SadTalker result (media_files table record) ..."); + + let (media_file_token, id) = insert_media_file_from_face_animation(InsertArgs { + pool: &args.job_dependencies.db.mysql_pool, + job: &job, + maybe_mime_type: Some(&mimetype), + file_size_bytes, + sha256_checksum: &file_checksum, + public_bucket_directory_hash: result_bucket_location.get_object_hash(), + maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX), + maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION), + is_on_prem: args.job_dependencies.job.info.container.is_on_prem, + worker_hostname: &args.job_dependencies.job.info.container.hostname, + worker_cluster: &args.job_dependencies.job.info.container.cluster_name, + }) + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("SadTalker Done."); + + // TODO: Update upstream to be strongly typed + let maybe_user_token = job.maybe_creator_user_token.as_deref() + .map(|token| UserToken::new_from_str(token)); + + args.job_dependencies.clients.firehose_publisher.lipsync_animation_finished( + maybe_user_token.as_ref(), + &job.inference_job_token, + media_file_token.as_str()) + .await + .map_err(|e| { + error!("error publishing event: {:?}", e); + ProcessSingleJobError::Other(anyhow!("error publishing event")) + })?; + + job_progress_reporter.log_status("done") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", + job.id, id, &media_file_token); + + Ok(JobSuccessResult { + maybe_result_entity: Some(ResultEntity { + entity_type: InferenceResultType::MediaFile, + entity_token: media_file_token.to_string(), + }), + inference_duration, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs new file mode 100644 index 000000000..c50fe62ec --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_dependencies.rs @@ -0,0 +1,15 @@ +use errors::AnyhowResult; + +use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::RerenderInferenceCommand; + +pub struct RerenderDependencies { + pub inference_command: RerenderInferenceCommand, +} + +impl RerenderDependencies { + pub fn setup() -> AnyhowResult { + Ok(Self { + inference_command: RerenderInferenceCommand::from_env()?, + }) + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs new file mode 100644 index 000000000..2c221e2b2 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -0,0 +1,233 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use log::info; +use once_cell::sync::Lazy; +use subprocess::{Popen, PopenConfig, Redirection}; + +use errors::AnyhowResult; +use filesys::path_to_string::path_to_string; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; + +use crate::job::job_loop::command_exit_status::CommandExitStatus; + +// These environment vars are not copied over to the subprocess +// TODO/FIXME(bt, 2023-05-28): This is horrific security! +static IGNORED_ENVIRONMENT_VARS : Lazy> = Lazy::new(|| { + let env_var_names= [ + "MYSQL_URL", + "ACCESS_KEY", + "SECRET_KEY", + "NEWRELIC_API_KEY", + ]; + + env_var_names.iter() + .map(|value| value.to_string()) + .collect::>() +}); + +#[derive(Clone)] +pub struct RerenderInferenceCommand { + /// Where the code lives + rerender_root_code_directory: PathBuf, + + /// A single executable script or a much larger bash command. + executable_or_command: ExecutableOrCommand, + + /// Config file to use + config_path: PathBuf, + + /// eg. `source python/bin/activate` + maybe_virtual_env_activation_command: Option, + + /// If this is run under Docker (eg. in development), these are the options. + maybe_docker_options: Option, + + /// If the execution should be ended after a certain point. + maybe_execution_timeout: Option, + +} + +#[derive(Clone)] +pub enum ExecutableOrCommand { + /// Eg. `inference.py` + Executable(PathBuf), + + /// Eg. `python3 inference.py` + Command(String), +} + +pub struct InferenceArgs<'s, P: AsRef> { + pub config_file: P, + pub stderr_output_file: &'s Path, +} + +impl RerenderInferenceCommand { + pub fn new>( + rerender_root_code_directory: PathBuf, + executable_or_command: ExecutableOrCommand, + config_path: PathBuf, + maybe_virtual_env_activation_command: Option, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + ) -> Self { + Self { + rerender_root_code_directory: rerender_root_code_directory.as_ref().to_path_buf(), + executable_or_command, + config_path: config_path.as_ref().to_path_buf(), + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_docker_options, + maybe_execution_timeout, + } + } + + pub fn from_env() -> AnyhowResult { + let rerender_root_code_directory = easyenv::get_env_pathbuf_required( + "RERENDER_INFERENCE_ROOT_DIRECTORY")?; + + let config_path = easyenv::get_env_pathbuf_required( + "RERENDER_INFERENCE_CONFIG_PATH")?; + + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( + "RERENDER_INFERENCE_MAYBE_VENV_COMMAND"); + + let maybe_execution_timeout = + easyenv::get_env_duration_seconds_optional("RERENDER_TIMEOUT_SECONDS"); + + let maybe_docker_options = easyenv::get_env_string_optional( + "RERENDER_INFERENCE_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_environment_variables: None, + maybe_gpu: Some(DockerGpu::All), + } + }); + + Ok(Self { + rerender_root_code_directory, + executable_or_command, + config_path, + maybe_virtual_env_activation_command, + maybe_docker_options, + maybe_execution_timeout, + }) + } + + pub fn execute_inference>( + &self, + args: InferenceArgs

, + ) -> CommandExitStatus { + match self.do_execute_inference(args) { + Ok(exit_status) => exit_status, + Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) }, + } + } + + fn do_execute_inference>( + &self, + args: InferenceArgs

, + ) -> AnyhowResult { + + let mut command = String::new(); + command.push_str(&format!("cd {}", path_to_string(&self.rerender_root_code_directory))); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + + command.push_str(" && "); + + match self.executable_or_command { + ExecutableOrCommand::Executable(ref executable) => { + command.push_str(&path_to_string(executable)); + command.push_str(" "); + } + ExecutableOrCommand::Command(ref cmd) => { + command.push_str(cmd); + command.push_str(" "); + } + } + + // ===== Begin Python Args ===== + + command.push_str(" --cfg "); + command.push_str(&path_to_string(args.config_file.as_ref())); + + // ===== End Python Args ===== + + if let Some(docker_options) = self.maybe_docker_options.as_ref() { + command = docker_options.to_command_string(&command); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + let mut env_vars = Vec::new(); + + // Copy all environment variables from the parent process. + // This is necessary to send all the kubernetes settings for Nvidia / CUDA. + for (env_key, env_value) in env::vars() { + if IGNORED_ENVIRONMENT_VARS.contains(&env_key) { + continue; + } + env_vars.push(( + OsString::from(env_key), + OsString::from(env_value), + )); + } + + let mut config = PopenConfig::default(); + + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + + if !env_vars.is_empty() { + config.env = Some(env_vars); + } + + let mut p = Popen::create(&command_parts, config)?; + + info!("Subprocess PID: {:?}", p.pid()); + + match self.maybe_execution_timeout { + None => { + let exit_status = p.wait()?; + info!("Subprocess exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + Some(timeout) => { + info!("Executing with timeout: {:?}", &timeout); + let exit_status = p.wait_timeout(timeout.clone())?; + + match exit_status { + None => { + // NB: If the program didn't successfully terminate, kill it. + info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout); + let _r = p.terminate()?; + Ok(CommandExitStatus::Timeout) + } + Some(exit_status) => { + info!("Subprocess timed wait exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + } + } + } + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs new file mode 100644 index 000000000..127a601f3 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs @@ -0,0 +1,85 @@ +use anyhow::anyhow; + +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::{VideofilterVideoSource}; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +pub struct JobArgs<'a> { + pub video_source: &'a VideofilterVideoSource, + pub sd_model: &'a str, +} + +pub fn validate_job(job: &AvailableInferenceJob) -> Result { + let inference_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + let inference_category = job.maybe_inference_args + .as_ref() + .map(|args| args.inference_category) + .flatten(); + + match inference_category { + Some(InferenceCategoryAbbreviated::RerenderAVideo) => {}, // Valid + Some(category) => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category))); + }, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference category for job!"))); + } + }; + + let inference_args = match inference_args { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!"))); + } + }; + + let inference_args = match inference_args { + PolymorphicInferenceArgs::Rr(inference_args) => inference_args, + _ => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!"))); + } + }; + + let video_source = match &inference_args.maybe_video_source { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no video source!"))); + } + }; + + let sd_model = match &inference_args.sd_model { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); + } + }; + + + // let preprocess = match inference_args.maybe_preprocess { + // None => Some("full".to_string()), + // Some(Preprocess::F) => Some("full".to_string()), + // Some(Preprocess::EF) => Some("extfull".to_string()), + // Some(Preprocess::C) => Some("crop".to_string()), + // Some(Preprocess::EC) => Some("extcrop".to_string()), + // }; + // + // let enhancer = match inference_args.maybe_face_enhancer { + // None => None, + // Some(FaceEnhancer::G) => Some("gfpgan".to_string()), + // Some(FaceEnhancer::R) => Some("RestoreFormer".to_string()), + // }; + // + // let mut width = inference_args.maybe_resize_width; + // let mut height = inference_args.maybe_resize_height; + + Ok(JobArgs { + video_source, + sd_model, + }) +} diff --git a/crates/service/job/inference_job/src/job_specific_dependencies.rs b/crates/service/job/inference_job/src/job_specific_dependencies.rs index ddd313a46..a863b3872 100644 --- a/crates/service/job/inference_job/src/job_specific_dependencies.rs +++ b/crates/service/job/inference_job/src/job_specific_dependencies.rs @@ -7,6 +7,7 @@ use crate::job::job_types::tts::vall_e_x::vall_e_x_dependencies::VallExDependenc use crate::job::job_types::tts::vits::vits_dependencies::VitsDependencies; use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies; use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies; +use crate::job::job_types::videofilter::rerender_a_video::rerender_dependencies::RerenderDependencies; use crate::util::scoped_execution::ScopedExecution; pub struct JobSpecificDependencies { @@ -16,6 +17,7 @@ pub struct JobSpecificDependencies { pub maybe_tacotron2_dependencies: Option, pub maybe_vall_e_x_dependencies: Option, pub maybe_vits_dependencies: Option, + pub maybe_rerender_dependencies: Option, } impl JobSpecificDependencies { @@ -27,6 +29,7 @@ impl JobSpecificDependencies { let mut maybe_tacotron2_dependencies = None; let mut maybe_vall_e_x_dependencies = None; let mut maybe_vits_dependencies = None; + let mut maybe_rerender_dependencies = None; if scoped_execution.can_run_job(InferenceModelType::RvcV2) { print_with_space("Setting RVCv2 dependencies..."); @@ -58,6 +61,11 @@ impl JobSpecificDependencies { maybe_vits_dependencies = Some(VitsDependencies::setup()?); } + if scoped_execution.can_run_job(InferenceModelType::RerenderAVideo) { + print_with_space("Setting Rerender dependencies..."); + maybe_rerender_dependencies = Some(RerenderDependencies::setup()?); + } + Ok(JobSpecificDependencies { maybe_rvc_v2_dependencies, maybe_sad_talker_dependencies, @@ -65,6 +73,7 @@ impl JobSpecificDependencies { maybe_tacotron2_dependencies, maybe_vall_e_x_dependencies, maybe_vits_dependencies, + maybe_rerender_dependencies, }) } } diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9bc777545..f612d034f 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -206,6 +206,7 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> let mut app = RouteBuilder::from_app(app) .add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler) + .add_post("/v1/animation/rerender/create", enqueue_rerender_animation_handler) .into_app(); // ==================== "Generic" Inference ==================== From 43030402ca20f41b72ff73b9f22b2c33e3b371fb Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 14:50:50 -0800 Subject: [PATCH 08/40] update --- Cargo.lock | 1 + .../inference_model_type.rs | 3 +- .../generic_inference_args.rs | 2 +- .../videofilter_payload.rs | 2 +- .../web/get_inference_job_status.rs | 2 + .../src/job/job_loop/process_single_job.rs | 3 + .../job/job_types/vc/rvc_v2/process_job.rs | 2 + .../job_types/vc/so_vits_svc/process_job.rs | 2 + .../src/job/job_types/videofilter/mod.rs | 3 +- .../videofilter/process_single_vf_job.rs | 63 +++++ .../videofilter/rerender_a_video/mod.rs | 2 +- .../rerender_a_video/process_jobs.rs | 2 +- .../rerender_a_video/validate_job.rs | 2 +- .../traits/internal_session_cache_purge.rs | 1 + crates/service/web/storyteller_web/Cargo.toml | 1 + .../storyteller_web/src/configs/plans/plan.rs | 19 ++ .../animation/enqueue_rerender_animation.rs | 237 ++++++++++++++++++ .../http_server/endpoints/animation/mod.rs | 3 +- .../inference_job/get_inference_job_status.rs | 8 + .../service/web/storyteller_web/src/routes.rs | 1 + 20 files changed, 351 insertions(+), 8 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs diff --git a/Cargo.lock b/Cargo.lock index d8fddc06d..215225d64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6025,6 +6025,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http", "http_server_common", "hyper", "infer 0.5.0", diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 5401430bd..0a3015653 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -48,7 +48,8 @@ impl InferenceModelType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", - Self::VallEX => "vall_e_x" + Self::VallEX => "vall_e_x", + Self::RerenderAVideo => "rerender_a_video", } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 7ba7b0b3e..f34a4469c 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -3,7 +3,7 @@ use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; use crate::payloads::generic_inference_args::tts_payload::TTSArgs; -use crate::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; +use crate::payloads::generic_inference_args::videofilter_payload::{RerenderArgs}; /// Used to encode extra state for the `generic_inference_jobs` table. diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs index 85335cde7..12d316933 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -32,5 +32,5 @@ pub struct RerenderArgs { #[serde(rename = "sd")] #[serde(skip_serializing_if = "Option::is_none")] - pub sd_model: String, + pub maybe_sd_model: Option, } diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index a59fd2e5d..247eb2b9c 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -177,6 +177,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), + InferenceCategory::RerenderAVideo => Some("rerender animation"), }; // NB: A bit of a hack. We store TTS results with a full path. @@ -185,6 +186,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), + InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), }; // NB: We've moved voice conversion out of their own table and into media_files diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 2d8f04efa..6dc5a4f7d 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -182,6 +182,9 @@ async fn do_process_single_job( InferenceCategory::VoiceConversion => { process_single_vc_job(job_dependencies, job).await? } + InferenceCategory::RerenderAVideo => { + process_single_rerender_job(job_dependencies, job).await? + } }; let maybe_entity_type = job_success_result.maybe_result_entity diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 74496b6ab..082dd95ea 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -239,6 +239,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { override_f0_method, .. } => *override_f0_method, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); @@ -247,6 +248,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index f5d7fa1b8..69337d467 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -168,6 +168,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten() .unwrap_or(false); @@ -177,6 +178,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, + PolymorphicInferenceArgs::Rr(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs index b8f301c73..7f70f4119 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -1 +1,2 @@ -pub(crate) mod rerender_a_video; \ No newline at end of file +pub(crate) mod rerender_a_video; +mod process_single_vf_job; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs new file mode 100644 index 000000000..eee4f2cd8 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -0,0 +1,63 @@ +use std::time::Duration; + +use anyhow::anyhow; +use log::{error, info}; + +use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::job::job_loop::job_success_result::JobSuccessResult; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; +use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; +use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job_dependencies::JobDependencies; + +pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { + let maybe_media_upload_token = job.maybe_input_source_token + .as_deref() + .map(|token| MediaUploadToken::new_from_str(token)); + + let media_upload_token = match maybe_media_upload_token { + None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media upload for vc job: {:?}", job.inference_job_token))), + Some(token) => token, + }; + + let maybe_media_upload_result = + get_media_upload_for_inference(&media_upload_token, &job_dependencies.db.mysql_pool).await; + + let media_upload = match maybe_media_upload_result { + Ok(Some(media_upload)) => media_upload, + Ok(None) => { + error!("no media upload record found for token: {:?}", media_upload_token); + return Err(ProcessSingleJobError::Other(anyhow!("no media upload record found for token: {:?}", media_upload_token))); + }, + Err(err) => { + error!("error fetching media upload record from db: {:?}", err); + return Err(ProcessSingleJobError::Other(err)); + }, + }; + + info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); + info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); + info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); + + + + let job_success_result = match vc_model.model_type { + VoiceConversionModelType::RvcV2 => { + rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + job_dependencies, + job, + vc_model: &vc_model, + media_upload_token: &media_upload_token, + media_upload: &media_upload, + }).await? + } + }; + + Ok(job_success_result) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 152c2840d..711397cfb 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -3,4 +3,4 @@ mod validate_job; mod download_video_file; mod rerender_inference_command; pub(crate) mod rerender_dependencies; -mod categorize_error; \ No newline at end of file +mod categorize_error; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs index f26de327c..affec1aee 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs @@ -83,7 +83,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result Result args, None => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index d125c53d7..69abd86b9 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,4 +1,5 @@ use actix_web::HttpRequest; +use mockall::automock; /// Allows us to purge session caches that might keep stale premium plan/billing data around #[cfg_attr(test, automock)] diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index df7cab336..f015262c1 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -164,6 +164,7 @@ pulldown-cmark = "0.9.3" async-trait = "0.1.58" utoipa = { version = "4", features = ["actix_extras"] } utoipa-swagger-ui = { version = "5", features = ["actix-web"] } +http = "0.2.11" [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 67dd178cc..80d100a9f 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -49,6 +49,9 @@ pub struct Plan { can_remove_visual_watermarks: bool, + // ========== Features for Videofilters ========== + rerender_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -91,6 +94,7 @@ impl Plan { is_development_plan: builder.is_development_plan, is_synthetic_plan: builder.is_synthetic_plan, lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive, + rerender_requires_frontend_keepalive: builder.rerender_requires_frontend_keepalive, can_remove_visual_watermarks: builder.can_remove_visual_watermarks, tts_base_priority_level: builder.tts_base_priority_level, tts_max_duration: builder.tts_max_duration, @@ -146,6 +150,10 @@ impl Plan { self.lipsync_requires_frontend_keepalive } + pub fn rerender_requires_frontent_keepalive(&self) -> bool { + self.rerender_requires_frontend_keepalive + } + pub fn can_remove_visual_watermarks(&self) -> bool { self.can_remove_visual_watermarks } @@ -192,6 +200,9 @@ pub struct PlanBuilder { can_remove_visual_watermarks: bool, + // ========== Features for Videofilters ========== + rerender_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -239,6 +250,9 @@ impl PlanBuilder { lipsync_requires_frontend_keepalive: true, can_remove_visual_watermarks: false, + // Videofilters + rerender_requires_frontend_keepalive: true, + // TTS tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL, tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS), @@ -304,6 +318,11 @@ impl PlanBuilder { self } + pub fn rerender_requires_frontend_keepalive(mut self, value: bool) -> Self { + self.rerender_requires_frontend_keepalive = value; + self + } + pub fn can_remove_visual_watermarks(mut self, value: bool) -> Self { self.can_remove_visual_watermarks = value; self diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs new file mode 100644 index 000000000..1519e5332 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -0,0 +1,237 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use log::{info, warn}; + +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use enums::common::visibility::Visibility; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::{RerenderArgs, VideofilterVideoSource}; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::media_uploads::MediaUploadToken; +use tokens::tokens::users::UserToken; + +use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// Debug requests can get routed to special "debug-only" workers, which can +/// be used to trial new code, run debugging, etc. +const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; + +/// The routing tag header can send workloads to particular k8s hosts. +/// This is useful for catching the live logs or intercepting the job. +const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; + +#[derive(Deserialize)] +pub struct EnqueueRerenderAnimationRequest { + uuid_idempotency_token: String, + + video_source: VideoSource, + sd_model: String, + creator_set_visibility: Option, +} + +/// Treated as an enum. Only one of these may be set. +#[derive(Deserialize)] +pub struct VideoSource { + maybe_media_file_token: Option, + maybe_media_upload_token: Option, +} + +#[derive(Serialize)] +pub struct EnqueueRerenderAnimationSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueRerenderAnimationError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueRerenderAnimationError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueRerenderAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueRerenderAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueRerenderAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueRerenderAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueRerenderAnimationError::BadInput(reason) => reason.to_string(), + EnqueueRerenderAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueRerenderAnimationError::ServerError => "server error".to_string(), + EnqueueRerenderAnimationError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl std::fmt::Display for EnqueueRerenderAnimationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn enqueue_rerender_animation_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let mut maybe_user_token : Option = None; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueRerenderAnimationError::ServerError + })?; + + let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); + + // ==================== USER SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EnqueueRerenderAnimationError::ServerError + })?; + + if let Some(user_session) = maybe_user_session.as_ref() { + maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token)); + } + + // TODO: Plan should handle "first anonymous use" and "investor" cases. + let plan = get_correct_plan_for_session( + server_state.server_environment, + maybe_user_session.as_ref()); + + // TODO: Separate priority for animation. + let priority_level = plan.web_vc_base_priority_level(); + + // ==================== DEBUG MODE + ROUTING TAG ==================== // + + let is_debug_request = + get_request_header_optional(&http_request, DEBUG_HEADER_NAME) + .is_some(); + + let maybe_routing_tag= + get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) + .map(|routing_tag| routing_tag.trim().to_string()); + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueRerenderAnimationError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueRerenderAnimationError::RateLimited); + } + + // ==================== LOOK UP MODEL INFO ==================== // + + // TODO(bt): CHECK DATABASE FOR TOKENS! + + let video_source: VideofilterVideoSource = { + if let Some(ref token) = request.video_source.maybe_media_file_token { + VideofilterVideoSource::media_file_token(token.as_str()) + } else if let Some(ref token) = request.video_source.maybe_media_upload_token { + VideofilterVideoSource::media_upload_token(token.as_str()) + } else { + return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); + } + }; + + let ip_address = get_request_ip(&http_request); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session: &users_component::utils::user_session_extended::UserSessionExtended| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(Visibility::Public); + + let inference_args = RerenderArgs { + maybe_video_source: Some(video_source), + maybe_sd_model: Some(request.sd_model.clone()), + }; + + info!("Creating rerender animation job record..."); + + let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_category: InferenceCategory::RerenderAVideo, + maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference + maybe_model_token: None, // NB: Model is static during inference + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? + maybe_raw_inference_text: None, // No text + maybe_max_duration_seconds: None, + maybe_inference_args: Some(GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo), + args: Some(PolymorphicInferenceArgs::Rr(inference_args)), + }), + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + requires_keepalive: plan.rerender_requires_frontent_keepalive(), + is_debug_request, + maybe_routing_tag: maybe_routing_tag.as_deref(), + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match query_result { + Ok((job_token, _id)) => job_token, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueRerenderAnimationError::ServerError); + } + }; + + let response: EnqueueRerenderAnimationSuccessResponse = EnqueueRerenderAnimationSuccessResponse { + success: true, + inference_job_token: job_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| EnqueueRerenderAnimationError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs index 1102d9357..5a9c5c080 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -1 +1,2 @@ -pub mod enqueue_lipsync_animation; \ No newline at end of file +pub mod enqueue_lipsync_animation; +pub mod enqueue_rerender_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 96be5ed7d..d227585c2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -256,6 +256,14 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } + InferenceCategory::RerenderAVideo => { + MediaFileBucketPath::from_object_hash( + &result_details.public_bucket_location_or_hash, + result_details.maybe_media_file_public_bucket_prefix.as_deref(), + result_details.maybe_media_file_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string() + } InferenceCategory::TextToSpeech => { match result_details.entity_type.as_str() { "media_file" => { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index f612d034f..96be08984 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -12,6 +12,7 @@ use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler; +use crate::http_server::endpoints::animation::enqueue_rerender_animation::enqueue_rerender_animation_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; From 14432cce83a8dde04613d0a8fc45a8b7bbf0e6fd Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 15:12:34 -0800 Subject: [PATCH 09/40] fix errors --- .../src/job/job_loop/process_single_job.rs | 3 +- .../src/job/job_types/videofilter/mod.rs | 2 +- .../videofilter/process_single_vf_job.rs | 35 ++++++++++--------- .../videofilter/rerender_a_video/mod.rs | 2 +- .../{process_jobs.rs => process_job.rs} | 0 .../rerender_inference_command.rs | 20 +++++++++-- .../traits/internal_session_cache_purge.rs | 4 +-- 7 files changed, 41 insertions(+), 25 deletions(-) rename crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/{process_jobs.rs => process_job.rs} (100%) diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 6dc5a4f7d..c7edcdbb7 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -16,6 +16,7 @@ use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSucce use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job; +use crate::job::job_types::videofilter::process_single_vf_job::process_single_rr_job; use crate::job_dependencies::JobDependencies; pub async fn process_single_job( @@ -183,7 +184,7 @@ async fn do_process_single_job( process_single_vc_job(job_dependencies, job).await? } InferenceCategory::RerenderAVideo => { - process_single_rerender_job(job_dependencies, job).await? + process_single_rr_job(job_dependencies, job).await? } }; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs index 7f70f4119..305ceb8a2 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/mod.rs @@ -1,2 +1,2 @@ pub(crate) mod rerender_a_video; -mod process_single_vf_job; +pub(crate) mod process_single_vf_job; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index eee4f2cd8..6a35427d0 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,19 +1,13 @@ -use std::time::Duration; - use anyhow::anyhow; use log::{error, info}; -use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; -use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; -use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; -use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job::job_types::videofilter::rerender_a_video; use crate::job_dependencies::JobDependencies; pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { @@ -47,17 +41,24 @@ pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &Ava - let job_success_result = match vc_model.model_type { - VoiceConversionModelType::RvcV2 => { - rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { - job_dependencies, - job, - vc_model: &vc_model, - media_upload_token: &media_upload_token, - media_upload: &media_upload, - }).await? + // let job_success_result = match vc_model.model_type { + // VoiceConversionModelType::RvcV2 => { + // rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + // job_dependencies, + // job, + // vc_model: &vc_model, + // media_upload_token: &media_upload_token, + // media_upload: &media_upload, + // }).await? + // } + // }; + + let job_success_result = rerender_a_video::process_job::process_job( + rerender_a_video::process_job::RerenderProcessJobArgs { + job_dependencies, + job } - }; + ).await?; Ok(job_success_result) } diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 711397cfb..28c9dea48 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -1,4 +1,4 @@ -mod process_jobs; +pub(crate) mod process_job; mod validate_job; mod download_video_file; mod rerender_inference_command; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs similarity index 100% rename from crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_jobs.rs rename to crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs index 2c221e2b2..8b0b01189 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -77,9 +77,9 @@ impl RerenderInferenceCommand { maybe_execution_timeout: Option, ) -> Self { Self { - rerender_root_code_directory: rerender_root_code_directory.as_ref().to_path_buf(), + rerender_root_code_directory: rerender_root_code_directory.clone(), executable_or_command, - config_path: config_path.as_ref().to_path_buf(), + config_path: config_path.clone(), maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), maybe_docker_options, maybe_execution_timeout, @@ -93,6 +93,20 @@ impl RerenderInferenceCommand { let config_path = easyenv::get_env_pathbuf_required( "RERENDER_INFERENCE_CONFIG_PATH")?; + let executable_or_command = match easyenv::get_env_string_optional( + "RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND") { + None => { + return Err(anyhow!("RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND is required")); + } + Some(executable_or_command) => { + if executable_or_command.contains(" ") { + ExecutableOrCommand::Command(executable_or_command) + } else { + ExecutableOrCommand::Executable(PathBuf::from(executable_or_command)) + } + } + }; + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( "RERENDER_INFERENCE_MAYBE_VENV_COMMAND"); @@ -192,7 +206,7 @@ impl RerenderInferenceCommand { let mut config = PopenConfig::default(); - info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_os_str()); let stderr_file = File::create(&args.stderr_output_file)?; config.stderr = Redirection::File(stderr_file); diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index 69abd86b9..cc4df2852 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,8 +1,8 @@ use actix_web::HttpRequest; -use mockall::automock; +// use mockall::automock; /// Allows us to purge session caches that might keep stale premium plan/billing data around -#[cfg_attr(test, automock)] +// #[cfg_attr(test, automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. From e5a85c752821c75f610f0dc2bd794dbc4974edfd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 18:56:02 -0500 Subject: [PATCH 10/40] fix failing doctest --- crates/vendor/limitation/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/vendor/limitation/src/lib.rs b/crates/vendor/limitation/src/lib.rs index 649b30eb4..b40cf6b1f 100644 --- a/crates/vendor/limitation/src/lib.rs +++ b/crates/vendor/limitation/src/lib.rs @@ -42,9 +42,12 @@ //! // The key to count and track //! let key = "10.0.0.5"; //! +//! // TODO(bt,2023-12-12): This doctest is so far behind in `futures` versions and needs to be updated. +//! // Tokio should be executing the future and evaluating the result. //! // Start and run a Tokio runtime to drive the Future to completion -//! tokio::run( -//! limiter +//! //let mut rt = tokio::runtime::Runtime::new().unwrap(); +//! //rt.block_on(...)//! +//! let _r = limiter //! // Count returns a Status if the key is under the limit and an `Error::LimitExceeded` //! // containing a Status if the limit has been exceeded //! .count(key) @@ -57,8 +60,9 @@ //! .and_then(|status| { //! println!("ok: {:?}", status); //! Ok(()) -//! }), -//! ); +//! }) +//! .wait(); +//! //); //! # Ok::<(), limitation::Error>(()) //! ``` //! From e375c8090c658bd52ee6d8cd73b0078bdf457343 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 12 Dec 2023 19:21:42 -0500 Subject: [PATCH 11/40] fix IDE handling (code compiled and tested, but IDE unhappy) --- .../billing/src/stripe/traits/internal_session_cache_purge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs index d125c53d7..df5bae077 100644 --- a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs +++ b/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; /// Allows us to purge session caches that might keep stale premium plan/billing data around -#[cfg_attr(test, automock)] +#[cfg_attr(test, mockall::automock)] pub trait InternalSessionCachePurge { /// Best effort attempt to delete /// Fails silently. From d51ee67d4c066b111da06352dde0835777d85c0d Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 17:35:36 -0800 Subject: [PATCH 12/40] complete failure --- .../inference_category.rs | 8 ++++++- .../inference_model_type.rs | 6 +++-- .../config/inference-job.development.env | 9 +++++++ .../rerender_inference_command.rs | 24 +++---------------- .../animation/enqueue_rerender_animation.rs | 12 ++-------- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index b96b6fb26..d1a420544 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -50,6 +50,7 @@ impl InferenceCategory { "lipsync_animation" => Ok(Self::LipsyncAnimation), "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), + "rerender_a_video" => Ok(Self::RerenderAVideo), _ => Err(format!("invalid value: {:?}", value)), } } @@ -61,6 +62,7 @@ impl InferenceCategory { Self::LipsyncAnimation, Self::TextToSpeech, Self::VoiceConversion, + Self::RerenderAVideo, ]) } } @@ -75,6 +77,7 @@ mod tests { assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation"); assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); + assert_serialization(InferenceCategory::RerenderAVideo, "rerender_a_video"); } #[test] @@ -82,6 +85,7 @@ mod tests { assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation"); assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); + assert_eq!(InferenceCategory::RerenderAVideo.to_str(), "rerender_a_video"); } #[test] @@ -89,16 +93,18 @@ mod tests { assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation); assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); + assert_eq!(InferenceCategory::from_str("rerender_a_video").unwrap(), InferenceCategory::RerenderAVideo); } #[test] fn all_variants() { // Static check let mut variants = InferenceCategory::all_variants(); - assert_eq!(variants.len(), 3); + assert_eq!(variants.len(), 4); assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::RerenderAVideo)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 0a3015653..c5c312b30 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -49,7 +49,7 @@ impl InferenceModelType { Self::Tacotron2 => "tacotron2", Self::Vits => "vits", Self::VallEX => "vall_e_x", - Self::RerenderAVideo => "rerender_a_video", + Self::RerenderAVideo => "rerender_a_video", } } @@ -61,6 +61,7 @@ impl InferenceModelType { "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), "vall_e_x" => Ok(Self::VallEX), + "rerender_a_video" => Ok(Self::RerenderAVideo), _ => Err(format!("invalid value: {:?}", value)), } } @@ -74,7 +75,8 @@ impl InferenceModelType { InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, - InferenceModelType::VallEX + InferenceModelType::VallEX, + InferenceModelType::RerenderAVideo ]) } } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 0df22ea66..359c8edf3 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -94,6 +94,15 @@ VITS_INFERENCE_SCRIPT="infer_ts_job.py" VITS_INFERENCE_MAYBE_PYTHON_INTERPRETER="python3" VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b" + +# ================================= # +# # +# Rerender # +# # +# ================================= # +RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/" +RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="conda activate rerender && python rerender.py" + # ================================= # # # # CACHE SETTINGS # diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs index 8b0b01189..a3ba8953a 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs @@ -40,7 +40,7 @@ pub struct RerenderInferenceCommand { executable_or_command: ExecutableOrCommand, /// Config file to use - config_path: PathBuf, + config_path: Option, /// eg. `source python/bin/activate` maybe_virtual_env_activation_command: Option, @@ -68,30 +68,12 @@ pub struct InferenceArgs<'s, P: AsRef> { } impl RerenderInferenceCommand { - pub fn new>( - rerender_root_code_directory: PathBuf, - executable_or_command: ExecutableOrCommand, - config_path: PathBuf, - maybe_virtual_env_activation_command: Option, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - ) -> Self { - Self { - rerender_root_code_directory: rerender_root_code_directory.clone(), - executable_or_command, - config_path: config_path.clone(), - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_docker_options, - maybe_execution_timeout, - } - } - pub fn from_env() -> AnyhowResult { let rerender_root_code_directory = easyenv::get_env_pathbuf_required( "RERENDER_INFERENCE_ROOT_DIRECTORY")?; - let config_path = easyenv::get_env_pathbuf_required( - "RERENDER_INFERENCE_CONFIG_PATH")?; + let config_path = easyenv::get_env_pathbuf_optional( + "RERENDER_INFERENCE_CONFIG_PATH"); let executable_or_command = match easyenv::get_env_string_optional( "RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND") { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs index 1519e5332..d90caa3b8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -20,7 +20,6 @@ use mysql_queries::payloads::generic_inference_args::videofilter_payload::{Reren use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; -use tokens::tokens::media_uploads::MediaUploadToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; @@ -39,17 +38,12 @@ const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; pub struct EnqueueRerenderAnimationRequest { uuid_idempotency_token: String, - video_source: VideoSource, + video_source: Option, sd_model: String, creator_set_visibility: Option, } /// Treated as an enum. Only one of these may be set. -#[derive(Deserialize)] -pub struct VideoSource { - maybe_media_file_token: Option, - maybe_media_upload_token: Option, -} #[derive(Serialize)] pub struct EnqueueRerenderAnimationSuccessResponse { @@ -165,10 +159,8 @@ pub async fn enqueue_rerender_animation_handler( // TODO(bt): CHECK DATABASE FOR TOKENS! let video_source: VideofilterVideoSource = { - if let Some(ref token) = request.video_source.maybe_media_file_token { + if let Some(ref token) = request.video_source { VideofilterVideoSource::media_file_token(token.as_str()) - } else if let Some(ref token) = request.video_source.maybe_media_upload_token { - VideofilterVideoSource::media_upload_token(token.as_str()) } else { return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); } From 292b831d1697d4912632a02cba080620fba55854 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 17:36:53 -0800 Subject: [PATCH 13/40] update tests --- .../by_table/generic_inference_jobs/inference_model_type.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index c5c312b30..a3ea0bcd2 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -94,6 +94,7 @@ mod tests { assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); assert_serialization(InferenceModelType::VallEX, "vall_e_x"); + assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video"); } #[test] @@ -104,6 +105,7 @@ mod tests { assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x"); + assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video"); } #[test] @@ -114,19 +116,21 @@ mod tests { assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX); + assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo); } #[test] fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 6); + assert_eq!(variants.len(), 7); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo)); assert_eq!(variants.pop_first(), None); // Generated check From 57e2f70707a2ace6f5ead0407b0ca3c66bc1fb46 Mon Sep 17 00:00:00 2001 From: salt Date: Tue, 12 Dec 2023 23:07:27 -0800 Subject: [PATCH 14/40] WORKING! --- Cargo.lock | 1 + .../generic_synthetic_ids/id_category.rs | 6 + .../media_file_origin_model_type.rs | 7 +- .../media_file_origin_product_category.rs | 6 + .../insert_media_file_from_rerender.rs | 158 ++++++++++++ .../src/queries/media_files/mod.rs | 1 + crates/service/job/inference_job/Cargo.toml | 1 + .../config/inference-job.development.env | 2 +- .../videofilter/process_single_vf_job.rs | 76 +++--- .../rerender_a_video/download_video_file.rs | 32 ++- .../rerender_a_video/process_job.rs | 229 ++++++++++++------ 11 files changed, 415 insertions(+), 104 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs diff --git a/Cargo.lock b/Cargo.lock index 215225d64..4193767e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3070,6 +3070,7 @@ dependencies = [ "newrelic-telemetry", "once_cell", "r2d2_redis", + "rand 0.8.5", "redis_common", "serde", "serde_derive", diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index eccff8cf4..01bddb94d 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,6 +21,10 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimationResult, + // Results from rerender a video + #[serde(rename = "rerender")] + RerenderResult, + /// Results from tacotron2 /// Applies for RVC and SVC #[serde(rename = "tts_result")] @@ -62,6 +66,7 @@ impl IdCategory { Self::ZeroShotVoiceDataset => "zs_dataset", Self::ZeroShotVoiceEmbedding => "zs_voice", Self::ZeroShotTtsResult => "zs_tts_result", + Self::RerenderResult => "rerender", Self::ModelWeights => "model_weights", } } @@ -75,6 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), + "rerender" => Ok(Self::RerenderResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index 8cb9685bc..a79930d8c 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -30,6 +30,9 @@ pub enum MediaFileOriginModelType { #[serde(rename = "vall_e_x")] VallEX, + + #[serde(rename = "rerender")] + Rerender, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -45,7 +48,8 @@ impl MediaFileOriginModelType { Self::SadTalker => "sad_talker", Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", - Self::VallEX => "vall_e_x" + Self::VallEX => "vall_e_x", + Self::Rerender => "rerender", } } @@ -56,6 +60,7 @@ impl MediaFileOriginModelType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 7b6f2a029..5965f9b3a 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -38,6 +38,10 @@ pub enum MediaFileOriginProductCategory { /// Media files created by (or uploaded for) the Zero Shot voice product. #[serde(rename = "zs_voice")] ZeroShotVoice, + + // Media files for Rerender + #[serde(rename = "rerender")] + Rerender, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -54,6 +58,7 @@ impl MediaFileOriginProductCategory { Self::Unknown => "unknown", Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", + Self::Rerender => "rerender", } } @@ -64,6 +69,7 @@ impl MediaFileOriginProductCategory { "unknown" => Ok(Self::Unknown), "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs new file mode 100644 index 000000000..c599a4fad --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs @@ -0,0 +1,158 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub struct InsertArgs<'a> { + pub pool: &'a MySqlPool, + pub job: &'a AvailableInferenceJob, + + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub sha256_checksum: &'a str, + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, + + pub is_on_prem: bool, + pub worker_hostname: &'a str, + pub worker_cluster: &'a str, +} + +pub async fn insert_media_file_from_rerender( + args: InsertArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let result_token = MediaFileToken::generate(); + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() { + let user_token = UserToken::new_from_str(creator_user_token); + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_rerender_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::RerenderResult, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_rerender_id); + } + + let creator_ip_address = args.job.creator_ip_address.as_str(); + let creator_set_visibility = args.job.creator_set_visibility.clone(); + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Rerender; + const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::Rerender; + const MEDIA_TYPE : MediaFileType = MediaFileType::Video; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + maybe_origin_model_type = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ?, + + is_generated_on_prem = ?, + generated_by_worker = ?, + generated_by_cluster = ? + + "#, + result_token.as_str(), + + ORIGIN_CATEGORY.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + ORIGIN_MODEL_TYPE.to_str(), + + MEDIA_TYPE.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.job.maybe_creator_user_token, + args.job.maybe_creator_anonymous_visitor_token, + args.job.creator_ip_address, + + args.job.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + + args.is_on_prem, + args.worker_hostname, + args.worker_cluster + ) + .execute(&mut *transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((result_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 8930a3ae7..ff431d1cb 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -8,3 +8,4 @@ pub mod insert_media_file_from_zero_shot_tts; pub mod list_media_files; pub mod list_media_files_for_user; pub mod update_media_file; +pub mod insert_media_file_from_rerender; diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 65713e40b..fb451aa8a 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -81,3 +81,4 @@ async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } time = "0.3.30" mime = "0.3.17" +rand = { version = "0.8.5", features = [] } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 359c8edf3..15ab7f649 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -101,7 +101,7 @@ VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b" # # # ================================= # RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/" -RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="conda activate rerender && python rerender.py" +RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py" # ================================= # # # diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index 6a35427d0..9d7cc0163 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,43 +1,56 @@ use anyhow::anyhow; -use log::{error, info}; +use log::info; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; +use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; -use tokens::tokens::media_uploads::MediaUploadToken; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use tokens::tokens::media_files::MediaFileToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::videofilter::rerender_a_video; +use crate::job::job_types::videofilter::rerender_a_video::process_job::RerenderProcessJobArgs; use crate::job_dependencies::JobDependencies; pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { - let maybe_media_upload_token = job.maybe_input_source_token - .as_deref() - .map(|token| MediaUploadToken::new_from_str(token)); + // let maybe_inference_args = job.maybe_inference_args.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Inference args not found")))?; + // + // let maybe_rerender_args = maybe_inference_args.args.as_ref().map(|args| match args { + // Rr(args) => Some(args), + // _ => None, + // }).flatten(); + // + // let rerender_args = match maybe_rerender_args { + // None => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + // Some(args) => args, + // }; + // + // let videofilter_source = rerender_args.maybe_video_source.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Video source not found")))?; + // + // let media_file_token = match videofilter_source { + // VideofilterVideoSource::F(token) => token, + // _ => return Err(ProcessSingleJobError::Other(anyhow!("Video source not found"))), + // }; + // + // let media_file_token = MediaFileToken::new_from_str(media_file_token); + // + // info!("media_file_token: {:?}", media_file_token); + // + // + // let media_file = match media_file_result { + // Ok(Some(result)) => result, + // Ok(None) => { + // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); + // } + // Err(e) => { + // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); + // } + // }; - let media_upload_token = match maybe_media_upload_token { - None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media upload for vc job: {:?}", job.inference_job_token))), - Some(token) => token, - }; - - let maybe_media_upload_result = - get_media_upload_for_inference(&media_upload_token, &job_dependencies.db.mysql_pool).await; - - let media_upload = match maybe_media_upload_result { - Ok(Some(media_upload)) => media_upload, - Ok(None) => { - error!("no media upload record found for token: {:?}", media_upload_token); - return Err(ProcessSingleJobError::Other(anyhow!("no media upload record found for token: {:?}", media_upload_token))); - }, - Err(err) => { - error!("error fetching media upload record from db: {:?}", err); - return Err(ProcessSingleJobError::Other(err)); - }, - }; - - info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); - info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); - info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); + // info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); + // info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); + // info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); @@ -54,9 +67,10 @@ pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &Ava // }; let job_success_result = rerender_a_video::process_job::process_job( - rerender_a_video::process_job::RerenderProcessJobArgs { + RerenderProcessJobArgs { job_dependencies, - job + job, + // media_file } ).await?; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs index 344592bd0..e05d69584 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -4,13 +4,16 @@ use anyhow::anyhow; use log::{error, warn}; use sqlx::MySqlPool; use tempdir::TempDir; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use cloud_storage::bucket_client::BucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -34,9 +37,32 @@ pub async fn download_video_file( let bucket_object_path; match video_source { - VideofilterVideoSource::F(_media_file_token) => { - // TODO(bt, 2023-09-08): Implement - return Err(ProcessSingleJobError::NotYetImplemented) + VideofilterVideoSource::F(media_file_token) => { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + false, + mysql_pool + ).await; + match media_file_result { + Ok(Some(result)) => { + let media_file_bucket_path = + MediaFileBucketPath::from_object_hash( + &result.public_bucket_directory_hash, + result.maybe_public_bucket_prefix.as_deref(), + result.maybe_public_bucket_extension.as_deref(), + ); + bucket_object_path = media_file_bucket_path.to_full_object_pathbuf(); + } + Ok(None) => { + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file: {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + } } VideofilterVideoSource::U(media_upload_token) => { let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index affec1aee..0df0d8032 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -1,8 +1,9 @@ use std::fs::read_to_string; use std::time::Instant; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use log::{error, info, warn}; +use serde::{Deserialize, Serialize}; use buckets::public::media_files::original_file::MediaFileBucketPath; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; @@ -12,19 +13,16 @@ use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; -use tokens::tokens::users::UserToken; +use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs}; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; -// use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error; use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; use crate::job_dependencies::JobDependencies; -use crate::util::common_commands::ffmpeg_logo_watermark_command; /// The maximum that either width or height can be const MAX_DIMENSION : u32 = 1500; @@ -35,6 +33,7 @@ const BUCKET_FILE_EXTENSION: &str = ".mp4"; pub struct RerenderProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, + // pub media_file: MediaFile, } pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { @@ -88,43 +87,8 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { - // error!("Image resize failure: {:?}", err); - // safe_delete_temp_file(&audio_path.filesystem_path); - // safe_delete_temp_file(&image_path.filesystem_path); - // safe_delete_temp_directory(&work_temp_dir); - // return Err(ProcessSingleJobError::Other(anyhow!("image resize failure: {:?}", err))); - // } - // Ok(resized_image_path) => { - // usable_image_path = resized_image_path; - // } - // } - // } + + //TODO: re encode with ffmpeg info!("Used video file: {:?}", usable_video_path); @@ -148,15 +112,43 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + Some(args) => args, + }; + + let rr_args = match poly_args { + Rr(args) => args, + _ => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), + }; + + let sd_model = rr_args.maybe_sd_model.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + // ============== CREATE AND WRITE CONFIG TO DISK ============== // + + // create seed if not exists + // let seed = rr_args.seed.unwrap_or_else(|| rand::random::()); + let seed = rand::random::(); + + let config = RerenderConfig { + input: Some(usable_video_path.to_str().unwrap().to_string()), + output: Some(output_video_fs_path.to_str().unwrap().to_string()), + work_dir: Some(work_temp_dir.path().to_str().unwrap().to_string()), + sd_model: Some(sd_model.to_string()), + prompt: Some("1girl, anime style, red hair".to_string()), + a_prompt: Some("1girl, 3d, realistic, read hair".to_string()), + n_prompt: Some("Bad quality, low quality, photo, artifacts, watermark, signature, faceless".to_string()), + seed: Some(seed as i64), + ..RerenderConfig::new() + }; + + config.validate().map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; let config_path = work_temp_dir.path().join("config.json"); - - // TODO + let config_json = serde_json::to_string(&config).unwrap(); + std::fs::write(&config_path, config_json).unwrap(); // ==================== RUN INFERENCE SCRIPT ==================== // - - let workdir = work_temp_dir.path().to_path_buf(); let stderr_output_file = work_temp_dir.path().join("stderr.txt"); let inference_start_time = Instant::now(); @@ -174,7 +166,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, + + #[serde(rename = "output")] + pub output: Option, + + #[serde(rename = "work_dir")] + pub work_dir: Option, + + #[serde(rename = "key_subdir")] + pub key_subdir: Option, + + #[serde(rename = "sd_model")] + pub sd_model: Option, + + #[serde(rename = "lora_path")] + pub lora_path: Option, + + #[serde(rename = "interval")] + pub interval: Option, + + #[serde(rename = "prompt")] + pub prompt: Option, + + #[serde(rename = "a_prompt")] + pub a_prompt: Option, + + #[serde(rename = "n_prompt")] + pub n_prompt: Option, + + #[serde(rename = "x0_strength")] + pub x0_strength: Option, + + #[serde(rename = "control_type")] + pub control_type: Option, + + #[serde(rename = "canny_low")] + pub canny_low: Option, + + #[serde(rename = "canny_high")] + pub canny_high: Option, + + #[serde(rename = "control_strength")] + pub control_strength: Option, + + #[serde(rename = "seed")] + pub seed: Option, + + #[serde(rename = "warp_period")] + pub warp_period: Option>, + + #[serde(rename = "ada_period")] + pub ada_period: Option>, +} + +impl RerenderConfig { + pub fn new() -> Self { + Self { + input: None, + output: None, + work_dir: None, + key_subdir: Some("keys".to_string()), + sd_model: None, + lora_path: None, + interval: Some(8), + prompt: None, + a_prompt: None, + n_prompt: None, + x0_strength: Some(0.95), + control_type: Some("canny".to_string()), + canny_low: Some(50), + canny_high: Some(100), + control_strength: Some(0.7), + seed: None, + warp_period: Some(vec![0., 0.1]), + ada_period: Some(vec![0.8, 1.]), + } + } + + pub fn validate(&self) -> Result<(), Error> { + let fields = &[ + (self.input.is_some(), "input"), + (self.output.is_some(), "output"), + (self.work_dir.is_some(), "work_dir"), + (self.key_subdir.is_some(), "key_subdir"), + (self.sd_model.is_some(), "sd_model"), + (self.interval.is_some(), "interval"), + (self.prompt.is_some(), "prompt"), + (self.a_prompt.is_some(), "a_prompt"), + (self.n_prompt.is_some(), "n_prompt"), + (self.x0_strength.is_some(), "x0_strength"), + (self.control_type.is_some(), "control_type"), + (self.canny_low.is_some(), "canny_low"), + (self.canny_high.is_some(), "canny_high"), + (self.control_strength.is_some(), "control_strength"), + (self.seed.is_some(), "seed"), + (self.warp_period.is_some(), "warp_period"), + (self.ada_period.is_some(), "ada_period"), + ]; + + let missing_fields: Vec<&str> = fields + .iter() + .filter_map(|(is_some, field_name)| if *is_some { None } else { Some(*field_name) }) + .collect(); + + if missing_fields.is_empty() { + Ok(()) + } else { + Err(anyhow!("Missing fields: {}", missing_fields.join(", "))) + } + } +} From 196be25fb5a8ccbb660b3f11c85d14d724d7cf59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 11:04:13 -0500 Subject: [PATCH 15/40] voice takedown requests --- _sql/actions/voice_conversion_models.sql | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 8520aa609..9fd7b4ee2 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -344,6 +344,36 @@ where token IN ( 'vcm_zmxv2k1q9jd8' ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'takedown request' +where token IN ( + 'vcm_3mzcjkf78x7m', + 'vcm_7k2t1wnyqxsc', + 'vcm_w2edpe6q2nya', + 'vcm_vz9cg294xa29', + 'vcm_0z1ys58589k7', + 'vcm_4x8a30h1r26z', + 'vcm_0vf9prxsww60', + 'vcm_yxbadkvqhg16', + 'vcm_p0ezrc6wvzak', + 'vcm_6r5pbtexyvtp', + 'vcm_pb0ewrf7ych5', + 'vcm_7frb2xk0cxk3', + 'vcm_x129c9s2af06', + 'vcm_e0snndfd59b6', + 'vcm_nb8zb0djqwqw', + 'vcm_2erz5hbh8tpf', + 'vcm_e9wn2ccqe3bk', + 'vcm_f8jbxdbjgcjb', + 'vcm_x02h7z3jqxb0', + 'vcm_xy97xcgnk9y6', + 'vcm_a47wtfbsjrfc', + 'vcm_kygenhbg5mgj', + 'vcm_s8gndv19yt7p' +); + From 3766758cfb33ac0cedf93ced058c117130419883 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 13:33:48 -0500 Subject: [PATCH 16/40] Rate limiter support for AI streamers --- .../config/storyteller-web.development.env | 3 + .../src/configs/app_startup/mod.rs | 2 + .../app_startup/redis_rate_limiters.rs | 102 ++++++++++++++++++ .../src/configs/app_startup/username_set.rs | 102 ++++++++++++++++++ .../web/storyteller_web/src/configs/mod.rs | 1 + .../enqueue_infer_tts_handler.rs | 19 ++++ .../service/web/storyteller_web/src/main.rs | 67 +----------- .../web/storyteller_web/src/server_state.rs | 7 ++ 8 files changed, 238 insertions(+), 65 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/mod.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs create mode 100644 crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs diff --git a/crates/service/web/storyteller_web/config/storyteller-web.development.env b/crates/service/web/storyteller_web/config/storyteller-web.development.env index 01ffcc593..c890b5ef3 100644 --- a/crates/service/web/storyteller_web/config/storyteller-web.development.env +++ b/crates/service/web/storyteller_web/config/storyteller-web.development.env @@ -5,6 +5,9 @@ COOKIE_SECURE=false COOKIE_DOMAIN=.jungle.horse WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse +# Test the rate limiter bypass (obviously these are dummy values) +AI_STREAMER_USERNAMES="PewDiePie,xQc" + # Stripe configs # All secrets are development and okay to check in STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs new file mode 100644 index 000000000..824be656d --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/mod.rs @@ -0,0 +1,2 @@ +pub mod redis_rate_limiters; +pub mod username_set; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs new file mode 100644 index 000000000..1dc2b4d5c --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -0,0 +1,102 @@ +use std::time::Duration; + +use log::info; + +use config::common_env::CommonEnv; +use errors::AnyhowResult; +use limitation::Limiter; +use crate::configs::app_startup::username_set::UsernameSet; + +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::server_state::RedisRateLimiters; + + +/// Build the various rate limiters +pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult { + // TODO(bt,2023-12-13): Kill CommonEnv. + + info!("Setting up Redis rate limiters..."); + + let logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) + }; + + let logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) + }; + + let api_high_priority_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) + }; + + // This is for the following users: + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let api_ai_streamers_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_AI_STREAMERS_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_MAX_REQUESTS", 30)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_AI_STREAMERS_WINDOW_SECONDS", 30)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "api_ai_streamers", limiter_enabled) + }; + + let ai_streamer_usernames = + UsernameSet::from_comma_separated(&easyenv::get_env_string_or_default("AI_STREAMER_USERNAMES", "")); + + info!("AI Streamers that can bypass the normal rate limiter ({}): {:?}", + ai_streamer_usernames.len(), + ai_streamer_usernames.list_names()); + + let model_upload_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) + }; + + Ok(RedisRateLimiters { + logged_out: logged_out_redis_rate_limiter, + logged_in: logged_in_redis_rate_limiter, + api_high_priority: api_high_priority_redis_rate_limiter, + api_ai_streamers: api_ai_streamers_redis_rate_limiter, + api_ai_streamer_username_set: ai_streamer_usernames, + model_upload: model_upload_rate_limiter, + }) +} diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs new file mode 100644 index 000000000..f9858a201 --- /dev/null +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -0,0 +1,102 @@ +use std::collections::{BTreeSet, HashSet}; + +#[derive(Clone)] +pub struct UsernameSet { + /// "usernames" in our service are always lowercase. + /// "display names" are case sensitive. + usernames: HashSet, +} + +impl UsernameSet { + pub fn from_comma_separated(comma_separated_list: &str) -> Self { + let usernames = comma_separated_list.split(",") + .into_iter() + .map(|item| item.trim()) + .map(|item| item.to_lowercase()) + .filter(|item| !item.is_empty()) + .collect::>(); + Self { + usernames + } + } + + pub fn len(&self) -> usize { + self.usernames.len() + } + + pub fn list_names(&self) -> BTreeSet<&str> { + // NB: BTreeSet for sorting, which is good for logging at app startup + self.usernames.iter().map(|item| item.as_str()).collect::<_>() + } + + pub fn username_is_in_set(&self, username: &str) -> bool { + // We lowercase the input username just in case a display name is passed + let username = username.to_lowercase(); + self.usernames.contains(&username) + } +} + +#[cfg(test)] +mod tests { + use crate::configs::app_startup::username_set::UsernameSet; + fn build_set() -> UsernameSet { + UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") + } + #[test] + fn test_deduplication() { + let set = build_set(); + assert_eq!(set.len(), 3); + } + + #[test] + fn test_in_set() { + let set = build_set(); + assert!(set.username_is_in_set("bob")); + assert!(set.username_is_in_set("alice")); + assert!(set.username_is_in_set("john")); + } + + #[test] + fn test_in_set_case_insensitive() { + let set = build_set(); + + assert!(set.username_is_in_set("Bob")); + assert!(set.username_is_in_set("BOB")); + + assert!(set.username_is_in_set("AlIcE")); + assert!(set.username_is_in_set("ALICE")); + + assert!(set.username_is_in_set("jOHN")); + assert!(set.username_is_in_set("JOHN")); + } + + #[test] + fn test_not_in_set() { + let set = build_set(); + + // NB: The build_set() function might have attempted to introduce these, + // but the struct constructor should filter blank spaces out + assert!(!set.username_is_in_set("")); + assert!(!set.username_is_in_set(" ")); + + // Obviously not in set + assert!(!set.username_is_in_set("mary")); + assert!(!set.username_is_in_set("bill")); + + // These usernames are in the set, but not with padding + assert!(!set.username_is_in_set(" alice ")); + assert!(!set.username_is_in_set("bob ")); + assert!(!set.username_is_in_set(" john")); + } + + #[test] + fn test_username_return() { + let set = build_set(); + let usernames = set.list_names(); + assert_eq!(usernames.len(), 3); + assert!(usernames.contains(&"alice")); + assert!(usernames.contains(&"bob")); + assert!(usernames.contains(&"john")); + assert!(!usernames.contains(&"mary")); + } +} diff --git a/crates/service/web/storyteller_web/src/configs/mod.rs b/crates/service/web/storyteller_web/src/configs/mod.rs index 0a735a5b5..806e1e320 100644 --- a/crates/service/web/storyteller_web/src/configs/mod.rs +++ b/crates/service/web/storyteller_web/src/configs/mod.rs @@ -1,2 +1,3 @@ +pub mod app_startup; pub mod plans; pub mod static_api_tokens; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index b01dd37a5..cb417f590 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -242,7 +242,12 @@ pub async fn enqueue_infer_tts_handler( // ==================== RATE LIMIT ==================== // + // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { @@ -250,6 +255,15 @@ pub async fn enqueue_infer_tts_handler( warn!("User is not authorized to use TTS model because they are banned."); return Err(InferTtsError::NotAuthorized); } + + if server_state + .redis_rate_limiters + .api_ai_streamer_username_set + .username_is_in_set(&user.user.username) + { + is_ai_streamer = true; + } + &server_state.redis_rate_limiters.logged_in }, }; @@ -264,6 +278,11 @@ pub async fn enqueue_infer_tts_handler( rate_limiter = &server_state.redis_rate_limiters.api_high_priority; } + if is_ai_streamer { + info!("Using AI streamer rate limiter"); + rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; + } + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index e77ed8509..57f09a82c 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -75,6 +75,7 @@ use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStr use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; @@ -173,65 +174,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_num("REDIS_CACHE_TTL_SECONDS", 60)?, ); - info!("Setting up Redis rate limiters..."); - - // Old env vars: - // - // "LIMITER_ENABLED" - // "LIMITER_MAX_REQUESTS" - // "LIMITER_WINDOW_SECONDS" - // - let logged_out_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) - }; - - let logged_in_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) - }; - - let api_high_priority_redis_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_API_HIGH_PRIORITY_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_MAX_REQUESTS", 30)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_API_HIGH_PRIORITY_WINDOW_SECONDS", 30)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "api_high_priority", limiter_enabled) - }; - - let model_upload_rate_limiter = { - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; - - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; - - RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) - }; info!("Connecting to elasticsearch..."); @@ -466,12 +408,7 @@ async fn main() -> AnyhowResult<()> { elasticsearch, redis_pool, redis_ttl_cache, - redis_rate_limiters: RedisRateLimiters { - logged_out: logged_out_redis_rate_limiter, - logged_in: logged_in_redis_rate_limiter, - api_high_priority: api_high_priority_redis_rate_limiter, - model_upload: model_upload_rate_limiter, - }, + redis_rate_limiters: configure_redis_rate_limiters(&common_env)?, firehose_publisher, badge_granter, avt_cookie_manager, diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index 41ddadf1d..a8f86133e 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -20,6 +20,7 @@ use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; +use crate::configs::app_startup::username_set::UsernameSet; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo; @@ -140,6 +141,12 @@ pub struct RedisRateLimiters { /// constructions) pub api_high_priority: RedisRateLimiter, + /// API rate limit for AI streamers + pub api_ai_streamers: RedisRateLimiter, + + /// Usernames of AI streamers + pub api_ai_streamer_username_set: UsernameSet, + /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, } From 81dd377599b08f62d0f7a6d483ebb1238d6c512c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 15:27:34 -0500 Subject: [PATCH 17/40] clean up and test rate limiter choice code --- .../src/configs/app_startup/username_set.rs | 1 + .../enqueue_infer_tts_handler.rs | 148 +++++++++++++----- 2 files changed, 114 insertions(+), 35 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs index f9858a201..0f48cb535 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/username_set.rs @@ -42,6 +42,7 @@ mod tests { fn build_set() -> UsernameSet { UsernameSet::from_comma_separated("bob,john,ALICE, BoB,jOhn, ,,, , aLiCe, joHn, bob,Alice") } + #[test] fn test_deduplication() { let set = build_set(); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index cb417f590..200156203 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -32,6 +32,7 @@ use tts_common::priority::{FAKEYOU_DEFAULT_VALID_API_TOKEN_PRIORITY_LEVEL, FAKEY use user_input_common::check_for_slurs::contains_slurs; use users_component::utils::user_session_extended::UserSessionExtended; +use crate::configs::app_startup::username_set::UsernameSet; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; use crate::configs::plans::plan::Plan; use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; @@ -240,49 +241,44 @@ pub async fn enqueue_infer_tts_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + warn!("User is not authorized to use TTS model because they are banned."); + return Err(InferTtsError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // - // TODO(bt,2023-12-13): Unit test this! if !disable_rate_limiter { - // For AI live streamers - // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 - let mut is_ai_streamer = false; + let maybe_username = maybe_user_session + .as_ref() + .map(|session| session.user.username.as_str()); - let mut rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - warn!("User is not authorized to use TTS model because they are banned."); - return Err(InferTtsError::NotAuthorized); - } + // FIXME(bt,2023-12-13): These boolean flags are a bit dangerous. + let rate_limiter_type = get_rate_limiter_type( + maybe_username, + &server_state.redis_rate_limiters.api_ai_streamer_username_set, + is_investor, + is_from_api, + use_high_priority_rate_limiter + ); - if server_state - .redis_rate_limiters - .api_ai_streamer_username_set - .username_is_in_set(&user.user.username) - { - is_ai_streamer = true; - } - - &server_state.redis_rate_limiters.logged_in + let rate_limiter = match rate_limiter_type { + RateLimiterType::LoggedOut => &server_state.redis_rate_limiters.logged_out, + RateLimiterType::LoggedIn => &server_state.redis_rate_limiters.logged_in, + RateLimiterType::ApiHighPriority => { + info!("Using API high priority rate limiter"); + &server_state.redis_rate_limiters.api_high_priority + }, + RateLimiterType::ApiAiStreamer => { + info!("Using AI streamer rate limiter"); + &server_state.redis_rate_limiters.api_ai_streamers }, }; - // TODO/TEMP - if is_investor || is_from_api { - rate_limiter = &server_state.redis_rate_limiters.logged_in; - } - - // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters - if use_high_priority_rate_limiter { - rate_limiter = &server_state.redis_rate_limiters.api_high_priority; - } - - if is_ai_streamer { - info!("Using AI streamer rate limiter"); - rate_limiter = &server_state.redis_rate_limiters.api_ai_streamers; - } - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } @@ -495,6 +491,54 @@ pub async fn enqueue_infer_tts_handler( .body(body)) } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RateLimiterType { + LoggedOut, + LoggedIn, + ApiHighPriority, + ApiAiStreamer, +} + +// FIXME(bt,2023-12-13): The boolean flags and disjoint decision making are kind of terrible. +fn get_rate_limiter_type( + maybe_username: Option<&str>, + ai_streamer_username_set: &UsernameSet, + is_investor: bool, + is_from_api: bool, + use_high_priority_rate_limiter: bool, +) -> RateLimiterType { + // For AI live streamers + // https://www.notion.so/storytellerai/dd88609558a24196b4ddeeef6079da98 + let mut is_ai_streamer = false; + + let mut rate_limiter = match maybe_username { + None => RateLimiterType::LoggedOut, + Some(username) => { + if ai_streamer_username_set.username_is_in_set(username) { + is_ai_streamer = true; + } + + RateLimiterType::LoggedIn + }, + }; + + // TODO/TEMP + if is_investor || is_from_api { + rate_limiter = RateLimiterType::LoggedIn; + } + + // TODO: This is for VidVoice.ai and should be replaced with per-API consumer rate limiters + if use_high_priority_rate_limiter { + rate_limiter = RateLimiterType::ApiHighPriority; + } + + if is_ai_streamer { + rate_limiter = RateLimiterType::ApiAiStreamer; + } + + rate_limiter +} + pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> { if text.len() < 3 { return Err("text is too short".to_string()); @@ -561,3 +605,37 @@ fn random_troll_text() -> &'static str { let random = texts.choose(&mut rand::thread_rng()); random.map(|r| *r).unwrap_or(texts[0]) } + + +#[cfg(test)] +mod tests { + + mod rate_limiter_tests { + use crate::configs::app_startup::username_set::UsernameSet; + use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::{get_rate_limiter_type, RateLimiterType}; + + fn build_username_set() -> UsernameSet { + UsernameSet::from_comma_separated("ai_streamer") + } + + #[test] + fn test_logged_out() { + let rate_limiter = get_rate_limiter_type(None, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedOut); + } + + #[test] + fn test_logged_in() { + let username = Some("not_ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::LoggedIn); + } + + #[test] + fn test_ai_streamer() { + let username = Some("ai_streamer"); + let rate_limiter = get_rate_limiter_type(username, &build_username_set(), false, false, false); + assert_eq!(rate_limiter, RateLimiterType::ApiAiStreamer); + } + } +} From 9ab8e5221190980666a1510fdf699171149bfcba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 16:58:37 -0500 Subject: [PATCH 18/40] more rate limiter fixes --- .../app_startup/redis_rate_limiters.rs | 28 +++++++++++++++++++ .../animation/enqueue_face_animation.rs | 16 +++++++---- .../media_uploads/common/handle_upload.rs | 21 +++++++------- .../enqueue_voice_conversion_inference.rs | 15 ++++++---- .../inference/enqueue_tts_request.rs | 17 ++++++----- .../voice_dataset_samples/upload_zs_sample.rs | 18 +++++++----- .../voice_designer/voices/create_voice.rs | 16 +++++++---- .../w2l/enqueue_infer_w2l_with_uploads.rs | 15 ++++++---- .../web/storyteller_web/src/server_state.rs | 6 ++++ 9 files changed, 103 insertions(+), 49 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs index 1dc2b4d5c..3aa3df32c 100644 --- a/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs +++ b/crates/service/web/storyteller_web/src/configs/app_startup/redis_rate_limiters.rs @@ -91,6 +91,32 @@ pub fn configure_redis_rate_limiters(common_env: &CommonEnv) -> AnyhowResult AnyhowResult &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueFaceAnimationError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs index 2d79cab2e..7fe2872d8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs @@ -66,26 +66,25 @@ pub async fn handle_upload( UploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(UploadError::RateLimited); } - if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadError::RateLimited); - } - // ==================== READ MULTIPART REQUEST ==================== // let upload_media_request = drain_multipart_request(multipart_payload) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index 22131729c..7be7cec00 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -205,17 +205,20 @@ pub async fn enqueue_voice_conversion_inference_handler( get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) .map(|routing_tag| routing_tag.trim().to_string()); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _session) => &server_state.redis_rate_limiters.logged_in }; // TODO/TEMP diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index 41adb5256..7e09f4078 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,7 +1,6 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] -use utoipa::ToSchema; use std::fmt::Debug; use std::sync::Arc; @@ -12,6 +11,7 @@ use actix_web::http::StatusCode; use log::warn; use serde::Deserialize; use serde::Serialize; +use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; @@ -153,16 +153,19 @@ pub async fn enqueue_tts_request( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueTTSRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueTTSRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs index 004d303f4..e89e18eca 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_zs_sample.rs @@ -26,6 +26,7 @@ use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::server_state::ServerState; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; @@ -111,16 +112,19 @@ pub async fn upload_zs_sample_handler( UploadSampleError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(UploadSampleError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(UploadSampleError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _user) => &server_state.redis_rate_limiters.file_upload_logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs index b47de2c73..665484300 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs @@ -30,6 +30,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -138,16 +139,19 @@ pub async fn create_voice_handler( |routing_tag| routing_tag.trim().to_string() ); + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.role.is_banned { + return Err(EnqueueCreateVoiceRequestError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueCreateVoiceRequestError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - } + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index eee83b0dc..8eea03805 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -102,16 +102,19 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_session { + if user.is_banned { + return Err(InferW2lWithUploadError::NotAuthorized); + } + } + // ==================== RATE LIMIT ==================== // let rate_limiter = match maybe_session { None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.is_banned { - return Err(InferW2lWithUploadError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, + Some(ref _user) => &server_state.redis_rate_limiters.logged_in, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index a8f86133e..c7ba60199 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -149,6 +149,12 @@ pub struct RedisRateLimiters { /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_out: RedisRateLimiter, + + /// For uploading files for voice conversion, face animator, etc. + pub file_upload_logged_in: RedisRateLimiter, } /// In-memory caches of several types. From b30c352d79bad5ad5a4b07bd219412455168ac69 Mon Sep 17 00:00:00 2001 From: saltacc Date: Wed, 13 Dec 2023 16:29:45 -0800 Subject: [PATCH 19/40] pass tests --- .../src/by_table/generic_synthetic_ids/id_category.rs | 9 +++++++-- .../media_files/media_file_origin_model_type.rs | 10 +++++++--- .../media_files/media_file_origin_product_category.rs | 7 ++++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index 01bddb94d..e5b2e9077 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -80,7 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), - "rerender" => Ok(Self::RerenderResult), + "rerender" => Ok(Self::RerenderResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } @@ -92,6 +92,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimationResult, + Self::RerenderResult, Self::TtsResult, Self::VoiceConversionResult, Self::ZeroShotTtsResult, @@ -114,6 +115,7 @@ mod tests { fn test_serialization() { assert_serialization(IdCategory::MediaFile, "media_file"); assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation"); + assert_serialization(IdCategory::RerenderResult, "rerender"); assert_serialization(IdCategory::TtsResult, "tts_result"); assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion"); assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); @@ -126,6 +128,7 @@ mod tests { fn to_str() { assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation"); + assert_eq!(IdCategory::RerenderResult.to_str(), "rerender"); assert_eq!(IdCategory::TtsResult.to_str(), "tts_result"); assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion"); assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset"); @@ -138,6 +141,7 @@ mod tests { fn from_str() { assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult); + assert_eq!(IdCategory::from_str("rerender").unwrap(), IdCategory::RerenderResult); assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult); assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult); assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset); @@ -150,9 +154,10 @@ mod tests { fn all_variants() { // Static check let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 8); + assert_eq!(variants.len(), 9); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); + assert_eq!(variants.pop_first(), Some(IdCategory::RerenderResult)); assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult)); diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index a79930d8c..fa75c7f79 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -60,7 +60,7 @@ impl MediaFileOriginModelType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), - "rerender" => Ok(Self::Rerender), + "rerender" => Ok(Self::Rerender), _ => Err(format!("invalid value: {:?}", value)), } } @@ -73,7 +73,8 @@ impl MediaFileOriginModelType { Self::SadTalker, Self::SoVitsSvc, Self::Tacotron2, - Self::VallEX + Self::VallEX, + Self::Rerender, ]) } } @@ -98,6 +99,7 @@ mod tests { assert_eq!(MediaFileOriginModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(MediaFileOriginModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(MediaFileOriginModelType::VallEX.to_str(), "vall_e_x"); + assert_eq!(MediaFileOriginModelType::Rerender.to_str(), "rerender"); } #[test] @@ -107,18 +109,20 @@ mod tests { assert_eq!(MediaFileOriginModelType::from_str("so_vits_svc").unwrap(), MediaFileOriginModelType::SoVitsSvc); assert_eq!(MediaFileOriginModelType::from_str("tacotron2").unwrap(), MediaFileOriginModelType::Tacotron2); assert_eq!(MediaFileOriginModelType::from_str("vall_e_x").unwrap(), MediaFileOriginModelType::VallEX); + assert_eq!(MediaFileOriginModelType::from_str("rerender").unwrap(), MediaFileOriginModelType::Rerender); assert!(MediaFileOriginModelType::from_str("foo").is_err()); } #[test] fn all_variants() { let mut variants = MediaFileOriginModelType::all_variants(); - assert_eq!(variants.len(), 5); + assert_eq!(variants.len(), 6); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::VallEX)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Rerender)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 5965f9b3a..103bab004 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -80,6 +80,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::TextToSpeech, + Self::Rerender, Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, @@ -102,6 +103,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); + assert_serialization(MediaFileOriginProductCategory::Rerender, "rerender") } #[test] @@ -111,6 +113,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); + assert_eq!(MediaFileOriginProductCategory::Rerender.to_str(), "rerender"); } #[test] @@ -120,17 +123,19 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); + assert_eq!(MediaFileOriginProductCategory::from_str("rerender").unwrap(), MediaFileOriginProductCategory::Rerender); } #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 5); + assert_eq!(variants.len(), 6); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Rerender)); assert_eq!(variants.pop_first(), None); } } From 0ed04839d3a36c0270905aa08a7c4ab8473ff35a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 20:27:04 -0500 Subject: [PATCH 20/40] takedown request --- _sql/actions/voice_conversion_models.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 9fd7b4ee2..1c040d57d 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -371,7 +371,8 @@ where token IN ( 'vcm_xy97xcgnk9y6', 'vcm_a47wtfbsjrfc', 'vcm_kygenhbg5mgj', - 'vcm_s8gndv19yt7p' + 'vcm_s8gndv19yt7p', + 'vcm_ny9pj57w5eyh' ); From aad6b4e30457cd516a5c9668af0673fda8a8da17 Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 13 Dec 2023 23:21:55 -0500 Subject: [PATCH 21/40] Update .env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index c41492d47..a39f1b1d8 100644 --- a/.env +++ b/.env @@ -76,4 +76,4 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt # Development key. Permissible to check in. NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL -STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. +#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting. From cf034a900e55e81c1b6d8ea38a73bc579903a8e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 04:27:36 +0000 Subject: [PATCH 22/40] Bump clap from 2.34.0 to 4.4.8 Bumps [clap](https://github.com/clap-rs/clap) from 2.34.0 to 4.4.8. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v2.34.0...v4.4.8) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Cargo.lock | 91 +++---------------- crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_upload_media_file/Cargo.toml | 2 +- crates/cli/elasticsearch-cli/Cargo.toml | 2 +- crates/cli/lru-delete-models/Cargo.toml | 2 +- crates/cli/scrape_test/Cargo.toml | 2 +- .../service/job/email_sender_job/Cargo.toml | 2 +- .../service/job/tts_inference_job/Cargo.toml | 2 +- 8 files changed, 20 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19da97d55..3874e5e3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,15 +483,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "anstream" version = "0.6.4" @@ -684,17 +675,6 @@ dependencies = [ "url", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi 0.3.9", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -1098,21 +1078,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", -] - [[package]] name = "clap" version = "4.4.8" @@ -1132,7 +1097,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.10.0", + "strsim", ] [[package]] @@ -1529,7 +1494,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn 1.0.109", ] @@ -1543,7 +1508,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn 1.0.109", ] @@ -1557,7 +1522,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn 2.0.39", ] @@ -1737,7 +1702,7 @@ name = "dev-database-seed" version = "0.0.1" dependencies = [ "buckets", - "clap 2.34.0", + "clap", "cloud_storage", "config", "easyenv", @@ -1763,7 +1728,7 @@ name = "dev-upload-media-file" version = "0.0.1" dependencies = [ "buckets", - "clap 2.34.0", + "clap", "cloud_storage", "config", "easyenv", @@ -2046,7 +2011,7 @@ name = "elasticsearch-cli" version = "0.0.1" dependencies = [ "buckets", - "clap 2.34.0", + "clap", "config", "easyenv", "elasticsearch", @@ -2102,7 +2067,7 @@ dependencies = [ "bootstrap", "buckets", "chrono", - "clap 2.34.0", + "clap", "cloud_storage", "concurrency", "config", @@ -2666,15 +2631,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.3" @@ -3077,7 +3033,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi", "rustix", "windows-sys 0.48.0", ] @@ -3380,7 +3336,7 @@ dependencies = [ name = "lru-delete-models" version = "0.0.1" dependencies = [ - "clap 4.4.8", + "clap", "errors", "logging", "my-workspace-hack", @@ -3973,7 +3929,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi", "libc", ] @@ -5125,7 +5081,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" name = "scrape_test" version = "0.0.1" dependencies = [ - "clap 2.34.0", + "clap", "errors", "my-workspace-hack", "tokio", @@ -6006,12 +5962,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.10.0" @@ -6331,15 +6281,6 @@ dependencies = [ "my-workspace-hack", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "thiserror" version = "1.0.50" @@ -6763,7 +6704,7 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap 2.34.0", + "clap", "cloud_storage", "collections", "config", @@ -7184,12 +7125,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version-sync" version = "0.9.5" diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index a6e1bfc85..b065225a7 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -33,7 +33,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External log = "0.4.14" -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated rand = "0.8.5" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 47d58a89c..9d928ea9e 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -29,7 +29,7 @@ tokens = { path = "../../_types/tokens" } my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated log = "0.4.14" # Runtime diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index 7361ad74b..9b0af1cde 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -29,7 +29,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External log = "0.4.14" -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated strum = { version = "0.25", features = ["derive"] } # Runtime diff --git a/crates/cli/lru-delete-models/Cargo.toml b/crates/cli/lru-delete-models/Cargo.toml index eb08a399a..082775d72 100644 --- a/crates/cli/lru-delete-models/Cargo.toml +++ b/crates/cli/lru-delete-models/Cargo.toml @@ -21,7 +21,7 @@ logging = { path = "../../lib/logging" } my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External -clap = { version = "4.4.7", features = ["derive"] } +clap = { version = "4.4.8", features = ["derive"] } walkdir = "2.4.0" [dev-dependencies] diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index 36284d1f3..f68f20dc3 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -22,5 +22,5 @@ web_scrapers = { path = "../../lib/web_scrapers" } my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 733388d42..07db875a2 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -47,7 +47,7 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 2d1f6fa73..02b6a01ed 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -33,7 +33,7 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } -clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" From a0375f10e21cc6bf89902620f3978c58c47e32aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 13 Dec 2023 10:48:58 -0500 Subject: [PATCH 23/40] Featured weights endpoint --- .../model_weights/weights_category.rs | 5 +- .../by_table/model_weights/weights_types.rs | 2 +- .../model_weights/list_weights_by_tokens.rs | 264 ++++++++++++++++++ .../src/queries/model_weights/mod.rs | 3 +- .../endpoints/weights/get_weight.rs | 4 +- .../weights/list_featured_weights.rs | 156 +++++++++++ .../src/http_server/endpoints/weights/mod.rs | 7 +- .../service/web/storyteller_web/src/routes.rs | 7 +- 8 files changed, 434 insertions(+), 14 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs diff --git a/crates/_types/enums/src/by_table/model_weights/weights_category.rs b/crates/_types/enums/src/by_table/model_weights/weights_category.rs index f6c4e6ed3..f2b10e67d 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_category.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_category.rs @@ -4,9 +4,8 @@ use std::collections::BTreeSet; use strum::EnumCount; #[cfg(test)] use strum::EnumIter; - - use utoipa::ToSchema; + #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)] pub enum WeightsCategory { @@ -51,7 +50,7 @@ impl WeightsCategory { } impl_enum_display_and_debug_using_to_str!(WeightsCategory); impl_mysql_enum_coders!(WeightsCategory); - +impl_mysql_from_row!(WeightsCategory); #[cfg(test)] mod tests { diff --git a/crates/_types/enums/src/by_table/model_weights/weights_types.rs b/crates/_types/enums/src/by_table/model_weights/weights_types.rs index 1b6bbcb13..12f00955d 100644 --- a/crates/_types/enums/src/by_table/model_weights/weights_types.rs +++ b/crates/_types/enums/src/by_table/model_weights/weights_types.rs @@ -72,7 +72,7 @@ impl WeightsType { impl_enum_display_and_debug_using_to_str!(WeightsType); impl_mysql_enum_coders!(WeightsType); - +impl_mysql_from_row!(WeightsType); #[cfg(test)] mod tests { diff --git a/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs new file mode 100644 index 000000000..f08a9534a --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/model_weights/list_weights_by_tokens.rs @@ -0,0 +1,264 @@ +use chrono::{DateTime, Utc}; +use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +#[derive(Serialize)] +pub struct WeightsByTokensRecord { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn list_weights_by_tokens( + mysql_pool: &MySqlPool, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let mut connection = mysql_pool.acquire().await?; + + let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + + Ok(map_to_weights(raw_weights)) +} + +async fn get_raw_weights_by_tokens( + connection: &mut MySqlConnection, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let connection = connection.acquire().await?; + + let mut query_builder : QueryBuilder = if can_see_deleted { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.token IN ( + "#, + ) + + } else { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.user_deleted_at IS NULL + AND mw.mod_deleted_at IS NULL + AND mw.token IN ( + "# + ) + }; + + query_builder.push(token_predicate(weight_tokens)); + + query_builder.push(")"); + + let query = query_builder.build_query_as::(); + + let results = query.fetch_all(connection).await?; + + Ok(results) +} + +/// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. +/// Issue: https://github.com/launchbadge/sqlx/issues/875 +fn token_predicate(tokens: &[ModelWeightToken]) -> String { + tokens.iter() + .map(|ty| ty.as_str()) + .map(|ty| format!("\"{}\"", ty)) + .collect::>() + .join(", ") +} + +fn map_to_weights(dataset:Vec) -> Vec { + let weights: Vec = dataset + .into_iter() + .map(|dataset: RawWeightJoinUser| { + WeightsByTokensRecord { + token: dataset.token, + title: dataset.title, + weights_type: dataset.weights_type, + weights_category: dataset.weights_category, + maybe_thumbnail_token: dataset.maybe_thumbnail_token, + + creator_user_token: dataset.creator_user_token, + creator_username:dataset.creator_username, + creator_display_name:dataset.creator_display_name, + creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + + private_bucket_hash: dataset.private_bucket_hash, + maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, + maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + + cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + + maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, + + created_at: dataset.created_at, + updated_at: dataset.updated_at, + user_deleted_at: dataset.user_deleted_at, + mod_deleted_at: dataset.mod_deleted_at, + } + }).collect(); + + weights +} + + #[derive(Serialize)] + pub struct RawWeightJoinUser { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for RawWeightJoinUser { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + token: ModelWeightToken::new(row.try_get("token")?), + weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, + weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, + title: row.try_get("title")?, + maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, + creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), + creator_username: row.try_get("creator_username")?, + creator_display_name: row.try_get("creator_display_name")?, + creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, + private_bucket_hash: row.try_get("private_bucket_hash")?, + maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, + maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, + cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, + cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, + cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, + maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, + cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + user_deleted_at: row.try_get("user_deleted_at")?, + mod_deleted_at: row.try_get("mod_deleted_at")?, + }) + } +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs index 898685caa..06aeec026 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/mod.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/mod.rs @@ -1,7 +1,8 @@ pub mod create_weight; pub mod delete_weights; pub mod get_weight; -pub mod update_weight; +pub mod list_weights_by_tokens; pub mod list_weights_by_user; pub mod list_weights_query_builder; pub mod tests; +pub mod update_weight; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs index 02be96fef..0b3f059aa 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/get_weight.rs @@ -109,12 +109,12 @@ pub async fn get_weight_handler( } }; - let weight_token = path.weight_token.clone(); + let weight_token = ModelWeightToken::new_from_str(&path.weight_token); let creator_user_token = user_session.user_token.clone(); let is_mod = user_session.can_ban_users; let weight_lookup_result = get_weight_by_token( - &ModelWeightToken::new(weight_token.clone()), + &weight_token, is_mod, &server_state.mysql_pool, ).await; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs new file mode 100644 index 000000000..33f921b72 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs @@ -0,0 +1,156 @@ +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{DateTime, Utc}; +use log::{debug, error, warn}; +use r2d2_redis::redis::Commands; +use utoipa::ToSchema; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, ToSchema)] +pub struct ListFeaturedWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, +} + +#[derive(Serialize, ToSchema)] +pub struct ModelWeightForList { + pub weight_token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator: UserDetailsLight, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// The key we store featured weights tokens under +const REDIS_KEY : &str = "featured_weights_list"; + +#[derive(Debug, ToSchema)] +pub enum ListFeaturedWeightsError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListFeaturedWeightsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListFeaturedWeightsError { + fn status_code(&self) -> StatusCode { + match *self { + ListFeaturedWeightsError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListFeaturedWeightsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/weights/list_featured", + responses( + (status = 200, description = "List Weights", body = ListFeaturedWeightsSuccessResponse), + (status = 401, description = "Not authorized", body = ListFeaturedWeightsError), + (status = 500, description = "Server error", body = ListFeaturedWeightsError), + ), +)] +pub async fn list_featured_weights_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let mut redis = server_state.redis_pool.get() + .map_err(|err| { + error!("Could not obtain redis: {err}"); + ListFeaturedWeightsError::ServerError + })?; + + let token_list : Option = redis.get(REDIS_KEY) + .map_err(|err| { + error!("Could not get redis result: {err}"); + ListFeaturedWeightsError::ServerError + })?; + + let weight_tokens = token_list + .unwrap_or_else(|| "".to_string()) + .split(",") + .into_iter() + .map(|item| item.trim()) + .filter(|item| !item.is_empty()) + .map(|item| ModelWeightToken::new_from_str(item)) + .collect::>(); + + debug!("Weight tokens from Redis: {:?}", weight_tokens); + + let mut weights = Vec::new(); + + if !weight_tokens.is_empty() { + let query_results = + list_weights_by_tokens(&server_state.mysql_pool, &weight_tokens, false).await; + + weights = match query_results { + Ok(weights) => weights, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListFeaturedWeightsError::ServerError); + } + }; + } + + // generate parse a response + let response = ListFeaturedWeightsSuccessResponse { + success: true, + weights: weights.into_iter() + .map(|w| ModelWeightForList { + weight_token: w.token, + title: w.title, + weights_type: w.weights_type, + weights_category: w.weights_category, + maybe_thumbnail_token: w.maybe_thumbnail_token, + creator: UserDetailsLight::from_db_fields( + &w.creator_user_token, + &w.creator_username, + &w.creator_display_name, + &w.creator_email_gravatar_hash + ), + cached_user_ratings_total_count: w.cached_user_ratings_total_count, + cached_user_ratings_positive_count: w.cached_user_ratings_positive_count, + cached_user_ratings_negative_count: w.cached_user_ratings_negative_count, + maybe_cached_user_ratings_ratio: w.maybe_cached_user_ratings_ratio, + created_at: w.created_at, + updated_at: w.updated_at, + }).collect::>(), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListFeaturedWeightsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs index 3fbd1cb09..f52604fe3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/mod.rs @@ -1,7 +1,8 @@ -pub mod get_weight; pub mod delete_weight; -pub mod update_weight; -pub mod list_weights_by_user; +pub mod get_weight; pub mod list_available_weights; +pub mod list_featured_weights; +pub mod list_weights_by_user; +pub mod update_weight; // pub mod create_weight; this is a private api call internally // pub mod drain_multipart_weights_request; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9dcdce3e3..0d13225c1 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -156,6 +156,7 @@ use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; use crate::http_server::endpoints::weights::update_weight::update_weight_handler; use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; +use crate::http_server::endpoints::weights::list_featured_weights::list_featured_weights_handler; use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; @@ -1265,9 +1266,7 @@ fn add_weights_routes(app: App) -> App .route(web::delete().to(delete_weight_handler)) ) .route("/by_user/{username}", web::get().to(list_weights_by_user_handler)) - .route( - "/list", - web::get().to(list_available_weights_handler) - ) + .route("/list", web::get().to(list_available_weights_handler)) + .route("/list_featured", web::get().to(list_featured_weights_handler)) ) } From faff80fcc64747cd56d71e9cc9ab380f2341e17b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 03:20:47 -0500 Subject: [PATCH 24/40] deterministic RNGs for tests --- Cargo.lock | 1 + crates/_types/tokens/Cargo.toml | 4 +- crates/_types/tokens/src/deterministic_rng.rs | 43 +++++++++++++++++++ crates/_types/tokens/src/lib.rs | 1 + crates/_types/tokens/src/macros.rs | 35 +++++++++++++++ .../tokens/tests/media_file_token_tests.rs | 24 +++++++++++ .../weights/list_featured_weights.rs | 1 - 7 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 crates/_types/tokens/src/deterministic_rng.rs diff --git a/Cargo.lock b/Cargo.lock index 19da97d55..af9894f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6442,6 +6442,7 @@ name = "tokens" version = "0.0.1" dependencies = [ "my-workspace-hack", + "once_cell", "rand 0.8.5", "serde", "serde_derive", diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index 581f8c65d..94152dee1 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -25,7 +25,9 @@ serde = "1.0.125" serde_derive = "1.0.125" # Random -rand = "0.8.5" +rand = { version = "0.8.5", features = ["std_rng"] } + +once_cell = { version = "1.18.0", features = [] } sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } diff --git a/crates/_types/tokens/src/deterministic_rng.rs b/crates/_types/tokens/src/deterministic_rng.rs new file mode 100644 index 000000000..90ad47f5a --- /dev/null +++ b/crates/_types/tokens/src/deterministic_rng.rs @@ -0,0 +1,43 @@ +use std::sync::{Arc, LockResult, RwLock, RwLockWriteGuard}; + +use once_cell::sync::Lazy; +use rand::rngs::StdRng; +use rand::SeedableRng; + +// FIXME(bt,2023-12-14): Is this threadsafe? Looks like it should be fine. +/// Only use this in tests (and dev seed tools), never in production! +#[derive(Clone)] +pub struct DeterministicRng { + rng : Arc> +} + +impl DeterministicRng { + pub fn get_instance() -> LockResult> { + static DETERMINISTIC_RNG : Lazy>> = Lazy::new(|| { + Arc::new(RwLock::new(DeterministicRng::new())) + }); + + DETERMINISTIC_RNG.write() + } + + fn new() -> Self { + Self { + rng: Arc::new(RwLock::new(StdRng::seed_from_u64(0))) + } + } + + /// Reset the deterministic RNG's seed. + pub fn reset_rng(&self) { + match self.rng.write() { + Err(err) => panic!(format!("test panic due to rng failure {err}")), + Ok(mut lock) => { + *lock = StdRng::seed_from_u64(0); + } + } + } + + /// Grab the deterministic RNG + pub fn get_rng(&self) -> LockResult>{ + self.rng.write() + } +} diff --git a/crates/_types/tokens/src/lib.rs b/crates/_types/tokens/src/lib.rs index df6e81b4f..b220bdb75 100644 --- a/crates/_types/tokens/src/lib.rs +++ b/crates/_types/tokens/src/lib.rs @@ -49,6 +49,7 @@ pub(crate) static MINIMUM_CHARACTER_ENTROPY : usize = 8; #[macro_use] mod macros; +pub (crate) mod deterministic_rng; pub (crate) mod prefixes; pub mod tokens; diff --git a/crates/_types/tokens/src/macros.rs b/crates/_types/tokens/src/macros.rs index 5964b6109..4df28db41 100644 --- a/crates/_types/tokens/src/macros.rs +++ b/crates/_types/tokens/src/macros.rs @@ -60,6 +60,41 @@ macro_rules! impl_crockford_generator { $t(token) } + /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. + /// This will not use a real RNG, so do not use in production code! + /// Once called, it will output the same random numbers in sequence. + pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously() { + let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); + deterministic.reset_rng(); + } + + /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. + /// This will not use a real RNG, so do not use in production code! + /// Once called, it will output the same random numbers in sequence. + pub fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously() -> Self { + use rand::Rng; + + // FIXME(bt,2023-12-14): This is bad error handling. (Maybe panic if lock is poisoned?) + let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); + let mut rng = deterministic.get_rng().expect("tests should not fail due to mutex"); + + let charset = Self::token_character_set(); + let entropy_length = Self::entropic_character_len(); + + let entropy_part: String = (0..entropy_length) + .map(|_| { + let idx = rng.gen_range(0..charset.len()); + charset[idx] as char + }) + .collect(); + + let token_prefix = Self::token_prefix(); + + let token = format!("{}{}", token_prefix, entropy_part); + + $t(token) + } + #[inline] pub fn entropy_suffix(&self) -> &str { use crate::prefixes::PrefixGenerator; diff --git a/crates/_types/tokens/tests/media_file_token_tests.rs b/crates/_types/tokens/tests/media_file_token_tests.rs index dc0969248..9a8704e1b 100644 --- a/crates/_types/tokens/tests/media_file_token_tests.rs +++ b/crates/_types/tokens/tests/media_file_token_tests.rs @@ -21,6 +21,30 @@ mod interface { assert!(token.to_string().starts_with("m_")); } + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_1() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_2() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + + #[test] + fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_3() { + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); + assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); + } + #[test] fn new() { let token = MediaFileToken::new("m_foo".to_string()); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs index 33f921b72..87d75eed0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list_featured_weights.rs @@ -122,7 +122,6 @@ pub async fn list_featured_weights_handler( }; } - // generate parse a response let response = ListFeaturedWeightsSuccessResponse { success: true, weights: weights.into_iter() From d447ad7d8676f1ea2f3e5304b9c3fed77278acd8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 03:49:58 -0500 Subject: [PATCH 25/40] deterministic test tokens it'd be good to use the same token values in tests, dev seed tools, etc. this should be a threadsafe, resettable, deterministic RNG --- crates/_types/tokens/src/deterministic_rng.rs | 4 +- crates/_types/tokens/src/macros.rs | 4 +- .../tokens/tests/media_file_token_tests.rs | 17 +++---- .../src/seeding/model_weights.rs | 44 ++++++++++--------- .../src/seeding/seed_model_weights.rs | 44 +++++++++---------- 5 files changed, 54 insertions(+), 59 deletions(-) diff --git a/crates/_types/tokens/src/deterministic_rng.rs b/crates/_types/tokens/src/deterministic_rng.rs index 90ad47f5a..affd9fda0 100644 --- a/crates/_types/tokens/src/deterministic_rng.rs +++ b/crates/_types/tokens/src/deterministic_rng.rs @@ -27,11 +27,11 @@ impl DeterministicRng { } /// Reset the deterministic RNG's seed. - pub fn reset_rng(&self) { + pub fn reset_rng(&self, state: u64) { match self.rng.write() { Err(err) => panic!(format!("test panic due to rng failure {err}")), Ok(mut lock) => { - *lock = StdRng::seed_from_u64(0); + *lock = StdRng::seed_from_u64(state); } } } diff --git a/crates/_types/tokens/src/macros.rs b/crates/_types/tokens/src/macros.rs index 4df28db41..44fc14d6d 100644 --- a/crates/_types/tokens/src/macros.rs +++ b/crates/_types/tokens/src/macros.rs @@ -63,9 +63,9 @@ macro_rules! impl_crockford_generator { /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. /// This will not use a real RNG, so do not use in production code! /// Once called, it will output the same random numbers in sequence. - pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously() { + pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(state: u64) { let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex"); - deterministic.reset_rng(); + deterministic.reset_rng(state); } /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding. diff --git a/crates/_types/tokens/tests/media_file_token_tests.rs b/crates/_types/tokens/tests/media_file_token_tests.rs index 9a8704e1b..722f89c0a 100644 --- a/crates/_types/tokens/tests/media_file_token_tests.rs +++ b/crates/_types/tokens/tests/media_file_token_tests.rs @@ -23,23 +23,18 @@ mod interface { #[test] fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_1() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // NB: Using the same reset seed will produce the same results each time + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); - } - - #[test] - fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_2() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // NB: Same seed -> same tokens generated + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); - } - - #[test] - fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_3() { - MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(); + // Once more... + MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h"); assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e"); diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev_database_seed/src/seeding/model_weights.rs index 8860d323c..13e23d601 100644 --- a/crates/cli/dev_database_seed/src/seeding/model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/model_weights.rs @@ -132,7 +132,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT - Gawr Gura MOS 4 "#; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura: {}", i); description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); description_rendered_html = rvc_markdown_description; @@ -155,7 +155,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT }, 21..=40 =>{ // TTS - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("dragonfruitGTv1: {}", i); description = format!("Description Number {}:", i); @@ -187,7 +187,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT }, 41..=60 => { // SD 1.5 - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("dragonfruitGTv1: {}", i); description = format!("Description Number {}:", i); @@ -232,7 +232,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT "#; let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg"; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("SDXL_Niji_Special Edition: {}", i); description = format!("Description Number {}:", i); @@ -263,7 +263,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT "#; let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg"; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura LoRA: {}", i); description = format!("Description Number {}:", i); @@ -295,7 +295,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool, user_token: UserT - Gawr Gura MOS 4 "#; - model_weight_token = ModelWeightToken(format!("{}", i)); + model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); title = format!("Gawr Gura: {}", i); description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i); description_rendered_html = rvc_markdown_description; @@ -356,11 +356,13 @@ pub async fn seed_weights_for_user_token( ) -> AnyhowResult<()> { info!("Seeding weights..."); - let model_weight_token1 = ModelWeightToken("11".to_string()); - let model_weight_token2 = ModelWeightToken("22".to_string()); - let model_weight_token3 = ModelWeightToken("33".to_string()); - let model_weight_token4 = ModelWeightToken("44".to_string()); - let model_weight_token5 = ModelWeightToken("55".to_string()); + ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(54321); + + let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); let model_weights_args = vec![ CreateModelWeightsArgs { @@ -503,20 +505,20 @@ pub async fn seed_weights_for_user_token( pub async fn original_seed_weights(mysql_pool: &Pool,user_token: UserToken) -> AnyhowResult<()> { - let model_weight_token1 = ModelWeightToken("1".to_string()); - let creator_token1 = UserToken("creatorToken1".to_string()); + let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token1 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token2 = ModelWeightToken("2".to_string()); - let creator_token2 = UserToken("creatorToken2".to_string()); + let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token2 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token3 = ModelWeightToken("3".to_string()); - let creator_token3 = UserToken("creatorToken3".to_string()); + let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token3 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token4 = ModelWeightToken("4".to_string()); - let creator_token4 = UserToken("creatorToken4".to_string()); + let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token4 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); - let model_weight_token5 = ModelWeightToken("5".to_string()); - let creator_token5 = UserToken("creatorToken5".to_string()); + let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let creator_token5 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); let model_weights_args = vec![ CreateModelWeightsArgs { diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs index 72ab9badd..ac6388b87 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs @@ -1,35 +1,31 @@ use std::path::Path; -use log::{ info, warn }; -use sqlx::{ MySql, Pool }; - -use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self }, private }; +use log::{info, warn}; +use rand::Rng; +use sqlx::{MySql, Pool}; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::{ + by_table::model_weights::{weights_category::WeightsCategory, weights_types::WeightsType}, + common::visibility::Visibility, +}; +use errors::AnyhowResult; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; -use mimetypes::mimetype_for_file::get_mimetype_for_file; - -use storyteller_root::get_seed_tool_data_root; -use tokens::tokens::users::UserToken; -use crate::bucket_clients::BucketClients; -use crate::seeding::users::HANASHI_USER_TOKEN; - -use enums::{ - by_table::model_weights::{ weights_category::WeightsCategory, weights_types::WeightsType }, - common::visibility::Visibility, -}; - -use rand::Rng; -use errors::{ anyhow, AnyhowResult }; use mysql_queries::queries::model_weights::create_weight::{ create_weight, - CreateModelWeightsArgs, - self, + CreateModelWeightsArgs + , }; +use storyteller_root::get_seed_tool_data_root; use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; pub async fn seed_weights_files( mysql_pool: &Pool, @@ -37,6 +33,9 @@ pub async fn seed_weights_files( ) -> AnyhowResult<()> { info!("Seeding weights files..."); + // NB: Stable token generation. + ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(123); + let seeded_weights = [ ( HANASHI_USER_TOKEN, @@ -87,9 +86,8 @@ pub async fn seed_weights_files( let mut rng = rand::thread_rng(); let n: u32 = rng.gen(); - let model_weight_token = ModelWeightToken(format!("{}", n)); // random token - let model_weight_token = model_weight_token; - + let model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously(); + let title = model_name; let description = "This is a description"; let description_rendered_html = "This is a description rendered html"; From eee75f0711b2e76a9b90fed588207f38ce9bc119 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 04:37:44 -0500 Subject: [PATCH 26/40] polymorphic token checks (thanks kasisnu!) + new token types --- .../by_table/comments/comment_entity_type.rs | 18 ++++++++++ .../user_bookmark_entity_type.rs | 9 +++++ .../src/by_table/user_ratings/entity_type.rs | 18 ++++++++++ .../user_ratings/user_rating_entity.rs | 8 +++++ .../queries/comments/comment_entity_token.rs | 8 +++++ .../user_bookmarks/list_user_bookmarks.rs | 4 +++ .../list_user_bookmarks_result.rs | 2 ++ .../user_bookmark_entity_token.rs | 4 +++ .../comments/create_comment_handler.rs | 27 +++++++++++++++ .../create_user_bookmark_handler.rs | 34 ++++++++++++++++++- .../user_ratings/get_user_rating_handler.rs | 7 +++- .../user_ratings/set_user_rating_handler.rs | 32 +++++++++++++++-- 12 files changed, 167 insertions(+), 4 deletions(-) diff --git a/crates/_types/enums/src/by_table/comments/comment_entity_type.rs b/crates/_types/enums/src/by_table/comments/comment_entity_type.rs index 72294e2b2..014e04e17 100644 --- a/crates/_types/enums/src/by_table/comments/comment_entity_type.rs +++ b/crates/_types/enums/src/by_table/comments/comment_entity_type.rs @@ -13,6 +13,14 @@ pub enum CommentEntityType { #[serde(rename = "user")] User, + /// Media files + #[serde(rename = "media_file")] + MediaFile, + + /// Model weights + #[serde(rename = "model_weight")] + ModelWeight, + /// TTS model (architecture does not matter) #[serde(rename = "tts_model")] TtsModel, @@ -39,6 +47,8 @@ impl CommentEntityType { pub fn to_str(&self) -> &'static str { match self { Self::User => "user", + Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::TtsModel => "tts_model", Self::TtsResult => "tts_result", Self::W2lTemplate => "w2l_template", @@ -49,6 +59,8 @@ impl CommentEntityType { pub fn from_str(value: &str) -> Result { match value { "user" => Ok(Self::User), + "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "tts_model" => Ok(Self::TtsModel), "tts_result" => Ok(Self::TtsResult), "w2l_template" => Ok(Self::W2lTemplate), @@ -69,6 +81,8 @@ mod tests { #[test] fn test_serialization() { assert_serialization(CommentEntityType::User, "user"); + assert_serialization(CommentEntityType::MediaFile, "media_file"); + assert_serialization(CommentEntityType::ModelWeight, "model_weight"); assert_serialization(CommentEntityType::TtsModel, "tts_model"); assert_serialization(CommentEntityType::TtsResult, "tts_result"); assert_serialization(CommentEntityType::W2lTemplate, "w2l_template"); @@ -82,6 +96,8 @@ mod tests { #[test] fn test_to_str() { assert_eq!(CommentEntityType::User.to_str(), "user"); + assert_eq!(CommentEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(CommentEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(CommentEntityType::TtsModel.to_str(), "tts_model"); assert_eq!(CommentEntityType::TtsResult.to_str(), "tts_result"); assert_eq!(CommentEntityType::W2lTemplate.to_str(), "w2l_template"); @@ -91,6 +107,8 @@ mod tests { #[test] fn test_from_str() { assert_eq!(CommentEntityType::from_str("user").unwrap(), CommentEntityType::User); + assert_eq!(CommentEntityType::from_str("media_file").unwrap(), CommentEntityType::MediaFile); + assert_eq!(CommentEntityType::from_str("model_weight").unwrap(), CommentEntityType::ModelWeight); assert_eq!(CommentEntityType::from_str("tts_model").unwrap(), CommentEntityType::TtsModel); assert_eq!(CommentEntityType::from_str("tts_result").unwrap(), CommentEntityType::TtsResult); assert_eq!(CommentEntityType::from_str("w2l_template").unwrap(), CommentEntityType::W2lTemplate); diff --git a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs index 69bf47cf5..1e3fa6692 100644 --- a/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs +++ b/crates/_types/enums/src/by_table/user_bookmarks/user_bookmark_entity_type.rs @@ -33,6 +33,10 @@ pub enum UserBookmarkEntityType { #[serde(rename = "media_file")] MediaFile, + /// ModelWeight (the new way to store models) + #[serde(rename = "model_weight")] + ModelWeight, + /// VoiceConversionModel #[serde(rename = "voice_conversion_model")] VoiceConversionModel, @@ -56,6 +60,7 @@ impl UserBookmarkEntityType { Self::W2lTemplate => "w2l_template", Self::W2lResult => "w2l_result", Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::VoiceConversionModel => "voice_conversion_model", Self::ZsVoice => "zs_voice", } @@ -69,6 +74,7 @@ impl UserBookmarkEntityType { "w2l_template" => Ok(Self::W2lTemplate), "w2l_result" => Ok(Self::W2lResult), "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "voice_conversion_model" => Ok(Self::VoiceConversionModel), "zs_voice" => Ok(Self::ZsVoice), _ => Err(format!("invalid value: {:?}", value)), @@ -92,6 +98,7 @@ mod tests { assert_serialization(UserBookmarkEntityType::W2lTemplate, "w2l_template"); assert_serialization(UserBookmarkEntityType::W2lResult, "w2l_result"); assert_serialization(UserBookmarkEntityType::MediaFile, "media_file"); + assert_serialization(UserBookmarkEntityType::ModelWeight, "model_weight"); assert_serialization(UserBookmarkEntityType::VoiceConversionModel, "voice_conversion_model"); assert_serialization(UserBookmarkEntityType::ZsVoice, "zs_voice"); } @@ -108,6 +115,7 @@ mod tests { assert_eq!(UserBookmarkEntityType::W2lTemplate.to_str(), "w2l_template"); assert_eq!(UserBookmarkEntityType::W2lResult.to_str(), "w2l_result"); assert_eq!(UserBookmarkEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserBookmarkEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(UserBookmarkEntityType::VoiceConversionModel.to_str(), "voice_conversion_model"); assert_eq!(UserBookmarkEntityType::ZsVoice.to_str(), "zs_voice"); } @@ -120,6 +128,7 @@ mod tests { assert_eq!(UserBookmarkEntityType::from_str("w2l_template").unwrap(), UserBookmarkEntityType::W2lTemplate); assert_eq!(UserBookmarkEntityType::from_str("w2l_result").unwrap(), UserBookmarkEntityType::W2lResult); assert_eq!(UserBookmarkEntityType::from_str("media_file").unwrap(), UserBookmarkEntityType::MediaFile); + assert_eq!(UserBookmarkEntityType::from_str("model_weight").unwrap(), UserBookmarkEntityType::ModelWeight); assert_eq!(UserBookmarkEntityType::from_str("voice_conversion_model").unwrap(), UserBookmarkEntityType::VoiceConversionModel); assert_eq!(UserBookmarkEntityType::from_str("zs_voice").unwrap(), UserBookmarkEntityType::ZsVoice); assert!(UserBookmarkEntityType::from_str("foo").is_err()); diff --git a/crates/_types/enums/src/by_table/user_ratings/entity_type.rs b/crates/_types/enums/src/by_table/user_ratings/entity_type.rs index f463471a9..025a8e4d6 100644 --- a/crates/_types/enums/src/by_table/user_ratings/entity_type.rs +++ b/crates/_types/enums/src/by_table/user_ratings/entity_type.rs @@ -9,6 +9,14 @@ use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] pub enum UserRatingEntityType { + /// Media files (inference results, uploads, etc.) + #[serde(rename = "media_file")] + MediaFile, + + /// Model weights (modern, polymorphic, type agnostic) + #[serde(rename = "model_weight")] + ModelWeight, + /// TTS model (architecture does not matter) #[serde(rename = "tts_model")] TtsModel, @@ -34,6 +42,8 @@ impl_mysql_enum_coders!(UserRatingEntityType); impl UserRatingEntityType { pub fn to_str(&self) -> &'static str { match self { + Self::MediaFile => "media_file", + Self::ModelWeight => "model_weight", Self::TtsModel => "tts_model", Self::TtsResult => "tts_result", Self::W2lTemplate => "w2l_template", @@ -43,6 +53,8 @@ impl UserRatingEntityType { pub fn from_str(value: &str) -> Result { match value { + "media_file" => Ok(Self::MediaFile), + "model_weight" => Ok(Self::ModelWeight), "tts_model" => Ok(Self::TtsModel), "tts_result" => Ok(Self::TtsResult), "w2l_template" => Ok(Self::W2lTemplate), @@ -62,6 +74,8 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(UserRatingEntityType::MediaFile, "media_file"); + assert_serialization(UserRatingEntityType::ModelWeight, "model_weight"); assert_serialization(UserRatingEntityType::TtsModel, "tts_model"); assert_serialization(UserRatingEntityType::TtsResult, "tts_result"); assert_serialization(UserRatingEntityType::W2lTemplate, "w2l_template"); @@ -74,6 +88,8 @@ mod tests { #[test] fn test_to_str() { + assert_eq!(UserRatingEntityType::MediaFile.to_str(), "media_file"); + assert_eq!(UserRatingEntityType::ModelWeight.to_str(), "model_weight"); assert_eq!(UserRatingEntityType::TtsModel.to_str(), "tts_model"); assert_eq!(UserRatingEntityType::TtsResult.to_str(), "tts_result"); assert_eq!(UserRatingEntityType::W2lTemplate.to_str(), "w2l_template"); @@ -82,6 +98,8 @@ mod tests { #[test] fn test_from_str() { + assert_eq!(UserRatingEntityType::from_str("media_file").unwrap(), UserRatingEntityType::MediaFile); + assert_eq!(UserRatingEntityType::from_str("model_weight").unwrap(), UserRatingEntityType::ModelWeight); assert_eq!(UserRatingEntityType::from_str("tts_model").unwrap(), UserRatingEntityType::TtsModel); assert_eq!(UserRatingEntityType::from_str("tts_result").unwrap(), UserRatingEntityType::TtsResult); assert_eq!(UserRatingEntityType::from_str("w2l_template").unwrap(), UserRatingEntityType::W2lTemplate); diff --git a/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs b/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs index a5fe4d14b..a100f7df3 100644 --- a/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs +++ b/crates/lib/mysql_queries/src/composite_keys/by_table/user_ratings/user_rating_entity.rs @@ -1,4 +1,6 @@ use enums::by_table::user_ratings::entity_type::UserRatingEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; @@ -12,6 +14,8 @@ use tokens::tokens::w2l_templates::W2lTemplateToken; /// NB: We probably don't want to expose this type of interface to web clients because it's /// monotonous as a JSON interface. pub enum UserRatingEntity { + MediaFile(MediaFileToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), W2lTemplate(W2lTemplateToken), } @@ -20,6 +24,8 @@ pub enum UserRatingEntity { impl UserRatingEntity { pub fn get_entity_type(&self) -> UserRatingEntityType { match self { + UserRatingEntity::MediaFile(_) => UserRatingEntityType::MediaFile, + UserRatingEntity::ModelWeight(_) => UserRatingEntityType::ModelWeight, UserRatingEntity::TtsModel(_) => UserRatingEntityType::TtsModel, UserRatingEntity::W2lTemplate(_) => UserRatingEntityType::W2lTemplate, } @@ -27,6 +33,8 @@ impl UserRatingEntity { pub fn get_entity_token_str(&self) -> &str { match self { + UserRatingEntity::MediaFile(token) => token.as_str(), + UserRatingEntity::ModelWeight(token) => token.as_str(), UserRatingEntity::TtsModel(token) => token.as_str(), UserRatingEntity::W2lTemplate(token) => token.as_str(), } diff --git a/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs b/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs index 47f918941..885ceac73 100644 --- a/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs +++ b/crates/lib/mysql_queries/src/queries/comments/comment_entity_token.rs @@ -1,10 +1,14 @@ use enums::by_table::comments::comment_entity_type::CommentEntityType; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; use tokens::tokens::users::UserToken; pub enum CommentEntityToken { User(UserToken), + MediaFile(MediaFileToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), TtsResult(String), // TODO: Strong type W2lTemplate(W2lTemplateToken), @@ -20,6 +24,8 @@ impl CommentEntityToken { CommentEntityType::TtsResult => Self::TtsResult(token.to_string()), CommentEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), CommentEntityType::W2lResult => Self::W2lResult(token.to_string()), + CommentEntityType::MediaFile => Self::MediaFile(MediaFileToken::new_from_str(token)), + CommentEntityType::ModelWeight => Self::ModelWeight(ModelWeightToken::new_from_str(token)), } } @@ -30,6 +36,8 @@ impl CommentEntityToken { CommentEntityToken::TtsResult(tts_result_token) => (CommentEntityType::TtsResult, tts_result_token.as_str()), CommentEntityToken::W2lTemplate(w2l_template_token) => (CommentEntityType::W2lTemplate, w2l_template_token.as_str()), CommentEntityToken::W2lResult(w2l_result_token) => (CommentEntityType::W2lResult, w2l_result_token.as_str()), + CommentEntityToken::MediaFile(media_file_token) => (CommentEntityType::MediaFile, media_file_token.as_str()), + CommentEntityToken::ModelWeight(model_weight_token) => (CommentEntityType::ModelWeight, model_weight_token.as_str()), }; (entity_type, entity_token) } diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs index 232c7344a..c28ab1612 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks.rs @@ -33,6 +33,7 @@ SELECT media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`, media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + model_weights.title as maybe_descriptive_text_model_weight_title, tts_models.title as maybe_descriptive_text_tts_model_title, tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text, users.display_name as maybe_descriptive_text_user_display_name, @@ -44,6 +45,7 @@ FROM JOIN users AS u ON f.user_token = u.token +LEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token LEFT OUTER JOIN media_files ON media_files.token = f.entity_token LEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token LEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token @@ -100,6 +102,7 @@ SELECT media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`, media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + model_weights.title as maybe_descriptive_text_model_weight_title, tts_models.title as maybe_descriptive_text_tts_model_title, tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text, users.display_name as maybe_descriptive_text_user_display_name, @@ -111,6 +114,7 @@ FROM JOIN users AS u ON f.user_token = u.token +LEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token LEFT OUTER JOIN media_files ON media_files.token = f.entity_token LEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token LEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index e56d1de9d..f384fbcd7 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -44,6 +44,7 @@ pub struct RawUserBookmarkRecord { pub (crate) maybe_media_file_type: Option, pub (crate) maybe_media_file_origin_category: Option, + pub (crate) maybe_descriptive_text_model_weight_title: Option, pub (crate) maybe_descriptive_text_tts_model_title: Option, pub (crate) maybe_descriptive_text_tts_result_inference_text: Option, pub (crate) maybe_descriptive_text_user_display_name: Option, @@ -59,6 +60,7 @@ impl RawUserBookmarkRecord { entity_token: self.entity_token, maybe_entity_descriptive_text: match self.entity_type { UserBookmarkEntityType::User => self.maybe_descriptive_text_user_display_name, + UserBookmarkEntityType::ModelWeight => self.maybe_descriptive_text_model_weight_title, UserBookmarkEntityType::TtsModel => self.maybe_descriptive_text_tts_model_title, UserBookmarkEntityType::TtsResult => self.maybe_descriptive_text_tts_result_inference_text, UserBookmarkEntityType::W2lTemplate => None, diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs index 5fbf8224a..aee34d7de 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/user_bookmark_entity_token.rs @@ -1,5 +1,6 @@ use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::tts_results::TtsResultToken; use tokens::tokens::users::UserToken; @@ -10,6 +11,7 @@ use tokens::tokens::zs_voices::ZsVoiceToken; pub enum UserBookmarkEntityToken { User(UserToken), + ModelWeight(ModelWeightToken), TtsModel(TtsModelToken), TtsResult(TtsResultToken), W2lTemplate(W2lTemplateToken), @@ -23,6 +25,7 @@ impl UserBookmarkEntityToken { pub fn from_entity_type_and_token(entity_type: UserBookmarkEntityType, token: &str) -> Self { match entity_type { UserBookmarkEntityType::User => Self::User(UserToken::new_from_str(token)), + UserBookmarkEntityType::ModelWeight => Self::ModelWeight(ModelWeightToken::new_from_str(token)), UserBookmarkEntityType::TtsModel => Self::TtsModel(TtsModelToken::new_from_str(token)), UserBookmarkEntityType::TtsResult => Self::TtsResult(TtsResultToken::new_from_str(token)), UserBookmarkEntityType::W2lTemplate => Self::W2lTemplate(W2lTemplateToken::new_from_str(token)), @@ -36,6 +39,7 @@ impl UserBookmarkEntityToken { pub fn get_composite_keys(&self) -> (UserBookmarkEntityType, &str) { match self { UserBookmarkEntityToken::User(user_token) => (UserBookmarkEntityType::User, user_token.as_str()), + UserBookmarkEntityToken::ModelWeight(model_weight_token) => (UserBookmarkEntityType::ModelWeight, model_weight_token.as_str()), UserBookmarkEntityToken::TtsModel(tts_model_token) => (UserBookmarkEntityType::TtsModel, tts_model_token.as_str()), UserBookmarkEntityToken::TtsResult(tts_result_token) => (UserBookmarkEntityType::TtsResult, tts_result_token.as_str()), UserBookmarkEntityToken::W2lTemplate(w2l_template_token) => (UserBookmarkEntityType::W2lTemplate, w2l_template_token.as_str()), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs index 09acffdb8..8e6f98b2e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/comments/create_comment_handler.rs @@ -16,6 +16,13 @@ use http_server_common::request::get_request_ip::get_request_ip; use mysql_queries::queries::comments::comment_entity_token::CommentEntityToken; use mysql_queries::queries::comments::insert_comment::{insert_comment, InsertCommentArgs}; use tokens::tokens::comments::CommentToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; use user_input_common::check_for_slurs::contains_slurs; use user_input_common::markdown_to_html::markdown_to_html; @@ -76,6 +83,26 @@ pub async fn create_comment_handler( server_state: web::Data>, ) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + CommentEntityType::User => token.starts_with(UserToken::token_prefix()) || token.starts_with("U:"), + CommentEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + CommentEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + CommentEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + CommentEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + CommentEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + CommentEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(CreateCommentError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool .acquire() .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs index b62fe2fee..43f0fe6ac 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_bookmarks/create_user_bookmark_handler.rs @@ -12,9 +12,19 @@ use actix_web::http::StatusCode; use log::warn; use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType; -use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs}; +use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; use tokens::tokens::user_bookmarks::UserBookmarkToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::voice_conversion_models::VoiceConversionModelToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use tokens::tokens::zs_voices::ZsVoiceToken; + use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -70,6 +80,28 @@ pub async fn create_user_bookmark_handler( server_state: web::Data>, ) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + UserBookmarkEntityType::User => token.starts_with(UserToken::token_prefix()) || token.starts_with("U:"), + UserBookmarkEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + UserBookmarkEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + UserBookmarkEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + UserBookmarkEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + UserBookmarkEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + UserBookmarkEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + UserBookmarkEntityType::VoiceConversionModel => token.starts_with(VoiceConversionModelToken::token_prefix()), + UserBookmarkEntityType::ZsVoice => token.starts_with(ZsVoiceToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(CreateUserBookmarkError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool .acquire() .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs index 228b053af..5c7f07791 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs @@ -9,6 +9,8 @@ use enums::by_table::user_ratings::rating_value::UserRatingValue; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use mysql_queries::composite_keys::by_table::user_ratings::user_rating_entity::UserRatingEntity; use mysql_queries::queries::users::user_ratings::get_user_rating::{Args, get_user_rating}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::w2l_templates::W2lTemplateToken; @@ -105,9 +107,12 @@ pub async fn get_user_rating_handler( UserRatingEntityType::W2lTemplate => UserRatingEntity::W2lTemplate( W2lTemplateToken::new_from_str(&path.entity_token)), + UserRatingEntityType::MediaFile => UserRatingEntity::MediaFile(MediaFileToken::new_from_str(&path.entity_token)), + UserRatingEntityType::ModelWeight => UserRatingEntity::ModelWeight(ModelWeightToken::new_from_str(&path.entity_token)), + // TODO: We'll handle ratings of more types in the future. UserRatingEntityType::W2lResult | UserRatingEntityType::TtsResult => - return Err(GetUserRatingError::BadInput("type not yet supported".to_string())) + return Err(GetUserRatingError::BadInput("type not yet supported".to_string())), }; let maybe_rating = get_user_rating(Args { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs index 8d5af4069..e25a3d6e8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/set_user_rating_handler.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use actix_web::http::StatusCode; -use log::{error, info}; +use log::{error, info, warn}; use enums::by_table::user_ratings::entity_type::UserRatingEntityType; use enums::by_table::user_ratings::rating_value::UserRatingValue; @@ -11,7 +11,11 @@ use http_server_common::response::serialize_as_json_error::serialize_as_json_err use mysql_queries::composite_keys::by_table::user_ratings::user_rating_entity::UserRatingEntity; use mysql_queries::queries::users::user_ratings::update_tts_model_ratings::update_tts_model_ratings; use mysql_queries::queries::users::user_ratings::upsert_user_rating::{Args, upsert_user_rating}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::w2l_results::W2lResultToken; use tokens::tokens::w2l_templates::W2lTemplateToken; use crate::server_state::ServerState; @@ -73,6 +77,27 @@ pub async fn set_user_rating_handler( request: web::Json, server_state: web::Data>) -> Result { + // NB(bt,2023-12-14): Kasisnu found that we're getting entity type mismatches in production. Apart from + // querying the database for entity existence, this is the next best way to prevent incorrect comment + // attachment. This is a bit of a bad process, though, since the token types are supposed to be opaque. + let token = request.entity_token.as_str(); + let token_prefix_matches = match request.entity_type { + // NB: Users had an older prefix (U:) that got replaced with the new prefix (user_) + UserRatingEntityType::MediaFile => token.starts_with(MediaFileToken::token_prefix()), + UserRatingEntityType::ModelWeight => token.starts_with(ModelWeightToken::token_prefix()), + UserRatingEntityType::TtsModel => token.starts_with(TtsModelToken::token_prefix()), + UserRatingEntityType::TtsResult => token.starts_with(TtsResultToken::token_prefix()), + UserRatingEntityType::W2lTemplate => token.starts_with(W2lTemplateToken::token_prefix()), + UserRatingEntityType::W2lResult => token.starts_with(W2lResultToken::token_prefix()), + //UserRatingEntityType::VoiceConversionModel => token.starts_with(VoiceConversionModelToken::token_prefix()), + //UserRatingEntityType::ZsVoice => token.starts_with(ZsVoiceToken::token_prefix()), + }; + + if !token_prefix_matches { + warn!("invalid token prefix: {:?} for {:?}", request.entity_token, request.entity_type); + return Err(SetUserRatingError::BadInput("invalid token prefix".to_string())); + } + let mut mysql_connection = server_state.mysql_pool.acquire() .await .map_err(|e| { @@ -100,6 +125,9 @@ pub async fn set_user_rating_handler( let ip_address = get_request_ip(&http_request); let entity= match request.entity_type { + UserRatingEntityType::MediaFile => UserRatingEntity::MediaFile(MediaFileToken::new_from_str(&request.entity_token)), + UserRatingEntityType::ModelWeight => UserRatingEntity::ModelWeight(ModelWeightToken::new_from_str(&request.entity_token)), + UserRatingEntityType::TtsModel => UserRatingEntity::TtsModel( TtsModelToken::new_from_str(&request.entity_token)), @@ -108,7 +136,7 @@ pub async fn set_user_rating_handler( // TODO: We'll handle ratings of more types in the future. UserRatingEntityType::W2lResult | UserRatingEntityType::TtsResult => - return Err(SetUserRatingError::BadInput("type not yet supported".to_string())) + return Err(SetUserRatingError::BadInput("type not yet supported".to_string())), }; let _r = upsert_user_rating(Args { From ff7ea2a4dbb3176ac3683d367a3e5e5ffcc4d335 Mon Sep 17 00:00:00 2001 From: kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:25:51 +0530 Subject: [PATCH 27/40] make media endpoint for profiles follow public pagination --- ...b1efd0cfa6b20899151f9e7f1a122c1f7a70e.json | 201 ------------------ ...9449362c074fc2397e23ecd175b6a0425b16b.json | 201 ------------------ .../media_files/list_media_files_for_user.rs | 131 +++++++----- .../common_responses/pagination_page.rs | 4 +- .../media_files/list_media_files_for_user.rs | 53 ++--- 5 files changed, 96 insertions(+), 494 deletions(-) delete mode 100644 .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json delete mode 100644 .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json diff --git a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json b/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json deleted file mode 100644 index 33334d176..000000000 --- a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "entity_token", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "user_token: tokens::tokens::users::UserToken", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "username", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "user_display_name", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "user_gravatar_hash", - "type_info": { - "type": "String", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "deleted_at", - "type_info": { - "type": "Timestamp", - "flags": "MULTIPLE_KEY | BINARY", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 10, - "name": "maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType", - "type_info": { - "type": "VarString", - "flags": "MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 11, - "name": "maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory", - "type_info": { - "type": "VarString", - "flags": "MULTIPLE_KEY | BINARY", - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "maybe_descriptive_text_tts_result_inference_text", - "type_info": { - "type": "Blob", - "flags": "BLOB | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 14, - "name": "maybe_descriptive_text_user_display_name", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 15, - "name": "maybe_descriptive_text_voice_conversion_model_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 16, - "name": "maybe_descriptive_text_zs_voice_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" -} diff --git a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json b/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json deleted file mode 100644 index 55d497bee..000000000 --- a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token: tokens::tokens::user_bookmarks::UserBookmarkToken", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "entity_token", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "user_token: tokens::tokens::users::UserToken", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "username", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "user_display_name", - "type_info": { - "type": "VarString", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "user_gravatar_hash", - "type_info": { - "type": "String", - "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "deleted_at", - "type_info": { - "type": "Timestamp", - "flags": "MULTIPLE_KEY | BINARY", - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 10, - "name": "maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType", - "type_info": { - "type": "VarString", - "flags": "MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 11, - "name": "maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory", - "type_info": { - "type": "VarString", - "flags": "MULTIPLE_KEY | BINARY", - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "maybe_descriptive_text_tts_result_inference_text", - "type_info": { - "type": "Blob", - "flags": "BLOB | BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 14, - "name": "maybe_descriptive_text_user_display_name", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 15, - "name": "maybe_descriptive_text_voice_conversion_model_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 16, - "name": "maybe_descriptive_text_zs_voice_title", - "type_info": { - "type": "VarString", - "flags": "BINARY | NO_DEFAULT_VALUE", - "char_set": 224, - "max_size": 1020 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" -} diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index 87bcc96ee..d7115cd5a 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use log::warn; use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; @@ -11,16 +12,15 @@ use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; + + pub struct MediaFileListPage { pub records: Vec, pub sort_ascending: bool, - /// ID of the first record. - pub first_id: Option, - - /// ID of the last record. - pub last_id: Option, + pub current_page: usize, + pub total_page_count: usize, } pub struct MediaFileListItem { @@ -51,9 +51,9 @@ pub enum ViewAs { pub struct ListMediaFileForUserArgs<'a> { pub username: &'a str, - pub limit: usize, pub maybe_filter_media_type: Option, - pub maybe_offset: Option, + pub page_size: usize, + pub page_index: usize, pub cursor_is_reversed: bool, pub view_as: ViewAs, pub mysql_pool: &'a MySqlPool, @@ -61,25 +61,42 @@ pub struct ListMediaFileForUserArgs<'a> { pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult { + /// Let's figure out how many results we could have returned total + let count_fields = select_total_count_field(); + let mut count_query_builder = query_builder( + args.maybe_filter_media_type, + args.username, + false, + 0, + 0, + args.cursor_is_reversed, + args.view_as, + count_fields.as_str(), + ); + + let row_count_query = count_query_builder.build_query_scalar::(); + let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?; + /// Figure out limit start and end based on page size and indexes as requested + let limit_start = args.page_size * args.page_index; + let limit_end = limit_start + args.page_size; + + /// Now fetch the actual results with all the fields + let result_fields = select_result_fields(); let mut query = query_builder( args.maybe_filter_media_type, args.username, - args.limit, - args.maybe_offset, + true, + limit_start, + limit_end, args.cursor_is_reversed, args.view_as, + result_fields.as_str(), ); let query = query.build_query_as::(); - let results = query.fetch_all(args.mysql_pool).await?; - let first_id = results.first() - .map(|raw_result| raw_result.id); - - let last_id = results.last() - .map(|raw_result| raw_result.id); - + let number_of_pages = (row_count_result / args.page_size as i64) as usize; let results = results.into_iter() .map(|record| { MediaFileListItem { @@ -102,43 +119,59 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An Ok(MediaFileListPage { records: results, sort_ascending: !args.cursor_is_reversed, - first_id, - last_id, + current_page: args.page_index, + total_page_count: number_of_pages, }) } + +fn select_result_fields() -> String { + r#" + m.id, + m.token, + + m.origin_category, + m.origin_product_category, + + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + m.media_type, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.creator_set_visibility, + m.created_at, + m.updated_at + "# + .to_string() +} + +fn select_total_count_field() -> String { + r#" + COUNT(m.id) AS total_count + "# + .to_string() +} + fn query_builder<'a>( maybe_filter_media_type: Option, username: &'a str, - limit: usize, - maybe_offset: Option, + enforce_limits: bool, + limit_start: usize, + limit_end: usize, cursor_is_reversed: bool, view_as: ViewAs, + select_fields: &'a str, ) -> QueryBuilder<'a, MySql> { // NB: Query cannot be statically checked by sqlx let mut query_builder: QueryBuilder = QueryBuilder::new( - r#" + format!(r#" SELECT - m.id, - m.token, - - m.origin_category, - m.origin_product_category, - - m.maybe_origin_model_type, - m.maybe_origin_model_token, - - m.media_type, - - m.public_bucket_directory_hash, - m.maybe_public_bucket_prefix, - m.maybe_public_bucket_extension, - - m.creator_set_visibility, - m.created_at, - m.updated_at - + {select_fields} FROM media_files AS m LEFT OUTER JOIN users AS u ON m.maybe_creator_user_token = u.token @@ -146,7 +179,7 @@ LEFT OUTER JOIN users AS u WHERE m.user_deleted_at IS NULL AND m.mod_deleted_at IS NULL "# - ); + )); query_builder.push(" AND u.username = "); query_builder.push_bind(username); @@ -171,21 +204,19 @@ WHERE m.user_deleted_at IS NULL } } - if let Some(offset) = maybe_offset { - if cursor_is_reversed { - query_builder.push(format!(" AND m.id < {offset} ")); - } else { - query_builder.push(format!(" AND m.id > {offset} ")); - } - } - if cursor_is_reversed { query_builder.push(" ORDER BY m.id ASC "); } else { query_builder.push(" ORDER BY m.id DESC "); } - query_builder.push(format!(" LIMIT {limit} ")); + if (enforce_limits) { + if cursor_is_reversed { + query_builder.push(format!(" LIMIT {limit_start}, {limit_end} ")); + } else { + query_builder.push(format!(" LIMIT {limit_end}, {limit_start} ")); + } + } query_builder } diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs index 3e64af962..50534944e 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs @@ -8,10 +8,10 @@ #[derive(Serialize)] pub struct PaginationPage { /// The current page number - pub current: u64, + pub current: usize, /// How many pages there are in total - pub total_page_count: u64, + pub total_page_count: usize, } #[cfg(test)] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs index bd3eda2b7..ef03b3dd4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -16,6 +16,7 @@ use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_ use tokens::tokens::media_files::MediaFileToken; use crate::http_server::common_responses::pagination_cursors::PaginationCursors; +use crate::http_server::common_responses::pagination_page::PaginationPage; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -27,7 +28,8 @@ pub struct PathInfo { #[derive(Deserialize)] pub struct QueryParams { pub sort_ascending: Option, - pub limit: Option, + pub page_size: Option, + pub page_index: Option, pub cursor: Option, pub cursor_is_reversed: Option, pub filter_media_type: Option, @@ -37,7 +39,7 @@ pub struct QueryParams { pub struct SuccessResponse { pub success: bool, pub results: Vec, - pub pagination: PaginationCursors, + pub pagination: PaginationPage, } #[derive(Serialize)] @@ -115,21 +117,13 @@ pub async fn list_media_files_for_user_handler( }; // TODO(bt,2023-12-04): Enforce real maximums and defaults - let limit = query.limit.unwrap_or(25); + let limit = query.page_size.unwrap_or(25); let sort_ascending = query.sort_ascending.unwrap_or(false); let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); - let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ErrorResponse::ServerError - })?; - Some(cursor as usize) - } else { - None - }; + let page_size = query.page_size.unwrap_or_else(|| 25); + let page_index = query.page_index.unwrap_or_else(|| 0); let view_as = if is_author { ViewAs::Author @@ -139,11 +133,12 @@ pub async fn list_media_files_for_user_handler( ViewAs::AnotherUser }; + let query_results = list_media_files_for_user(ListMediaFileForUserArgs { username: &path.username, - limit, maybe_filter_media_type: query.filter_media_type, - maybe_offset: cursor, + page_size, + page_index, cursor_is_reversed, view_as, mysql_pool: &server_state.mysql_pool, @@ -158,27 +153,6 @@ pub async fn list_media_files_for_user_handler( } }; - let cursor_next = if let Some(id) = results_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ErrorResponse::ServerError - })?; - Some(cursor) - } else { - None - }; - - let cursor_previous = if let Some(id) = results_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ErrorResponse::ServerError - })?; - Some(cursor) - } else { - None - }; let results = results_page.records.into_iter() .map(|record| MediaFileListItem { @@ -200,10 +174,9 @@ pub async fn list_media_files_for_user_handler( let response = SuccessResponse { success: true, results, - pagination: PaginationCursors { - maybe_next: cursor_next, - maybe_previous: cursor_previous, - cursor_is_reversed, + pagination: PaginationPage{ + current: results_page.current_page, + total_page_count: results_page.total_page_count, } }; From 8618e774e7bfa7af95f531eb1d737f94c0140646 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 13:37:25 -0500 Subject: [PATCH 28/40] codegen db --- ...663d0eb3d969365f6b2fd048ca41c5c455b2.json} | 23 +- ...44666e6c7c0f24b5da79dde828ffdb2c8302.json} | 23 +- .../media_files/list_media_files_by_tokens.rs | 264 ++++++++++++++++++ 3 files changed, 298 insertions(+), 12 deletions(-) rename .sqlx/{query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json => query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json} (78%) rename .sqlx/{query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json => query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json} (79%) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs diff --git a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json b/.sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json similarity index 78% rename from .sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json rename to .sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json index 33334d176..921e62dbe 100644 --- a/.sqlx/query-d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e.json +++ b/.sqlx/query-3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2.json @@ -1,6 +1,6 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n model_weights.title as maybe_descriptive_text_model_weight_title,\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.entity_type = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { @@ -125,7 +125,7 @@ }, { "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", + "name": "maybe_descriptive_text_model_weight_title", "type_info": { "type": "VarString", "flags": "BINARY | NO_DEFAULT_VALUE", @@ -135,6 +135,16 @@ }, { "ordinal": 13, + "name": "maybe_descriptive_text_tts_model_title", + "type_info": { + "type": "VarString", + "flags": "BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, "name": "maybe_descriptive_text_tts_result_inference_text", "type_info": { "type": "Blob", @@ -144,7 +154,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "maybe_descriptive_text_user_display_name", "type_info": { "type": "VarString", @@ -154,7 +164,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "maybe_descriptive_text_voice_conversion_model_title", "type_info": { "type": "VarString", @@ -164,7 +174,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "maybe_descriptive_text_zs_voice_title", "type_info": { "type": "VarString", @@ -194,8 +204,9 @@ true, true, true, + true, true ] }, - "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e" + "hash": "3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2" } diff --git a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json b/.sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json similarity index 79% rename from .sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json rename to .sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json index 55d497bee..0b00758da 100644 --- a/.sqlx/query-eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b.json +++ b/.sqlx/query-707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302.json @@ -1,6 +1,6 @@ { "db_name": "MySQL", - "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", + "query": "\nSELECT\n f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n f.entity_token,\n f.user_token as `user_token: tokens::tokens::users::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n f.created_at,\n f.updated_at,\n f.deleted_at,\n\n media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n model_weights.title as maybe_descriptive_text_model_weight_title,\n tts_models.title as maybe_descriptive_text_tts_model_title,\n tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n users.display_name as maybe_descriptive_text_user_display_name,\n voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n user_bookmarks AS f\nJOIN users AS u\n ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n u.username = ?\n AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n ", "describe": { "columns": [ { @@ -125,7 +125,7 @@ }, { "ordinal": 12, - "name": "maybe_descriptive_text_tts_model_title", + "name": "maybe_descriptive_text_model_weight_title", "type_info": { "type": "VarString", "flags": "BINARY | NO_DEFAULT_VALUE", @@ -135,6 +135,16 @@ }, { "ordinal": 13, + "name": "maybe_descriptive_text_tts_model_title", + "type_info": { + "type": "VarString", + "flags": "BINARY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, "name": "maybe_descriptive_text_tts_result_inference_text", "type_info": { "type": "Blob", @@ -144,7 +154,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "maybe_descriptive_text_user_display_name", "type_info": { "type": "VarString", @@ -154,7 +164,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "maybe_descriptive_text_voice_conversion_model_title", "type_info": { "type": "VarString", @@ -164,7 +174,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "maybe_descriptive_text_zs_voice_title", "type_info": { "type": "VarString", @@ -194,8 +204,9 @@ true, true, true, + true, true ] }, - "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b" + "hash": "707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302" } diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs new file mode 100644 index 000000000..f08a9534a --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -0,0 +1,264 @@ +use chrono::{DateTime, Utc}; +use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +#[derive(Serialize)] +pub struct WeightsByTokensRecord { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn list_weights_by_tokens( + mysql_pool: &MySqlPool, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let mut connection = mysql_pool.acquire().await?; + + let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + + Ok(map_to_weights(raw_weights)) +} + +async fn get_raw_weights_by_tokens( + connection: &mut MySqlConnection, + weight_tokens: &[ModelWeightToken], + can_see_deleted: bool +) -> AnyhowResult> { + + let connection = connection.acquire().await?; + + let mut query_builder : QueryBuilder = if can_see_deleted { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.token IN ( + "#, + ) + + } else { + QueryBuilder::new(r#" + SELECT + mw.token, + mw.title, + mw.weights_type, + mw.weights_category, + mw.maybe_thumbnail_token, + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash as creator_email_gravatar_hash, + mw.private_bucket_hash, + mw.maybe_private_bucket_prefix, + mw.maybe_private_bucket_extension, + mw.cached_user_ratings_negative_count, + mw.cached_user_ratings_positive_count, + mw.cached_user_ratings_total_count, + mw.maybe_cached_user_ratings_ratio, + mw.cached_user_ratings_last_updated_at, + mw.created_at, + mw.updated_at, + mw.user_deleted_at, + mw.mod_deleted_at + FROM model_weights as mw + JOIN users + ON users.token = mw.creator_user_token + WHERE + mw.creator_set_visibility = "public" + AND mw.user_deleted_at IS NULL + AND mw.mod_deleted_at IS NULL + AND mw.token IN ( + "# + ) + }; + + query_builder.push(token_predicate(weight_tokens)); + + query_builder.push(")"); + + let query = query_builder.build_query_as::(); + + let results = query.fetch_all(connection).await?; + + Ok(results) +} + +/// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. +/// Issue: https://github.com/launchbadge/sqlx/issues/875 +fn token_predicate(tokens: &[ModelWeightToken]) -> String { + tokens.iter() + .map(|ty| ty.as_str()) + .map(|ty| format!("\"{}\"", ty)) + .collect::>() + .join(", ") +} + +fn map_to_weights(dataset:Vec) -> Vec { + let weights: Vec = dataset + .into_iter() + .map(|dataset: RawWeightJoinUser| { + WeightsByTokensRecord { + token: dataset.token, + title: dataset.title, + weights_type: dataset.weights_type, + weights_category: dataset.weights_category, + maybe_thumbnail_token: dataset.maybe_thumbnail_token, + + creator_user_token: dataset.creator_user_token, + creator_username:dataset.creator_username, + creator_display_name:dataset.creator_display_name, + creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + + private_bucket_hash: dataset.private_bucket_hash, + maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, + maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + + cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, + cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, + cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + + maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, + cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, + + created_at: dataset.created_at, + updated_at: dataset.updated_at, + user_deleted_at: dataset.user_deleted_at, + mod_deleted_at: dataset.mod_deleted_at, + } + }).collect(); + + weights +} + + #[derive(Serialize)] + pub struct RawWeightJoinUser { + pub token: ModelWeightToken, + + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + + pub title: String, + + pub maybe_thumbnail_token: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_email_gravatar_hash: String, + + pub private_bucket_hash: String, + pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_extension: Option, + + pub cached_user_ratings_total_count: u32, + pub cached_user_ratings_positive_count: u32, + pub cached_user_ratings_negative_count: u32, + pub maybe_cached_user_ratings_ratio: Option, + pub cached_user_ratings_last_updated_at: DateTime, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for RawWeightJoinUser { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + token: ModelWeightToken::new(row.try_get("token")?), + weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, + weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, + title: row.try_get("title")?, + maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, + creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), + creator_username: row.try_get("creator_username")?, + creator_display_name: row.try_get("creator_display_name")?, + creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, + private_bucket_hash: row.try_get("private_bucket_hash")?, + maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, + maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, + cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, + cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, + cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, + maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, + cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + user_deleted_at: row.try_get("user_deleted_at")?, + mod_deleted_at: row.try_get("mod_deleted_at")?, + }) + } +} From 32b1786b59c8456ea382c511ab50b9be1d906d35 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 14:53:17 -0500 Subject: [PATCH 29/40] No hateful / antisemitic models on our service (hitler) --- _sql/actions/voice_conversion_models.sql | 9 +++++++++ crates/lib/mysql_queries/src/queries/media_files/mod.rs | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 1c040d57d..7f97f2da6 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -376,6 +376,15 @@ where token IN ( ); +update voice_conversion_models +set + mod_deleted_at = NOW(), + maybe_mod_comments = 'abuse' +where token IN ( + 'vcm_qajyd528p8kw', + 'vcm_f5pcdecmzcc5' +); + diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index ff431d1cb..cedc16583 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -2,10 +2,11 @@ pub mod delete_media_file; pub mod get_media_file; pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_rerender; pub mod insert_media_file_from_tacotron2; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; pub mod list_media_files; +pub mod list_media_files_by_tokens; pub mod list_media_files_for_user; pub mod update_media_file; -pub mod insert_media_file_from_rerender; From 5a4d2a8c065e208f994749ddf421ff715e347b72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 15:15:31 -0500 Subject: [PATCH 30/40] media files query by tokens (WHERE IN clause) --- .../media_files/list_media_files_by_tokens.rs | 296 ++++++++---------- 1 file changed, 132 insertions(+), 164 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs index f08a9534a..a6e06e49f 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -2,142 +2,131 @@ use chrono::{DateTime, Utc}; use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; -use enums::by_table::model_weights::{ - weights_category::WeightsCategory, - weights_types::WeightsType, -}; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; -use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; #[derive(Serialize)] -pub struct WeightsByTokensRecord { - pub token: ModelWeightToken, +pub struct MediaFilesByTokensRecord { + pub token: MediaFileToken, - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, - pub title: String, + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, - pub maybe_thumbnail_token: Option, - - pub creator_user_token: UserToken, - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, - - pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, - pub maybe_private_bucket_extension: Option, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_email_gravatar_hash: Option, pub created_at: DateTime, pub updated_at: DateTime, - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, } -pub async fn list_weights_by_tokens( +pub async fn list_media_files_by_tokens( mysql_pool: &MySqlPool, - weight_tokens: &[ModelWeightToken], + media_file_tokens: &[MediaFileToken], can_see_deleted: bool -) -> AnyhowResult> { +) -> AnyhowResult> { let mut connection = mysql_pool.acquire().await?; - let raw_weights: Vec = get_raw_weights_by_tokens(&mut connection, weight_tokens, can_see_deleted).await?; + let raw_media_files: Vec = get_raw_media_files_by_tokens(&mut connection, media_file_tokens, can_see_deleted).await?; - Ok(map_to_weights(raw_weights)) + Ok(map_to_media_files(raw_media_files)) } -async fn get_raw_weights_by_tokens( +async fn get_raw_media_files_by_tokens( connection: &mut MySqlConnection, - weight_tokens: &[ModelWeightToken], + media_file_tokens: &[MediaFileToken], can_see_deleted: bool -) -> AnyhowResult> { +) -> AnyhowResult> { let connection = connection.acquire().await?; let mut query_builder : QueryBuilder = if can_see_deleted { QueryBuilder::new(r#" SELECT - mw.token, - mw.title, - mw.weights_type, - mw.weights_category, - mw.maybe_thumbnail_token, - users.token as creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - users.email_gravatar_hash as creator_email_gravatar_hash, - mw.private_bucket_hash, - mw.maybe_private_bucket_prefix, - mw.maybe_private_bucket_extension, - mw.cached_user_ratings_negative_count, - mw.cached_user_ratings_positive_count, - mw.cached_user_ratings_total_count, - mw.maybe_cached_user_ratings_ratio, - mw.cached_user_ratings_last_updated_at, - mw.created_at, - mw.updated_at, - mw.user_deleted_at, - mw.mod_deleted_at - FROM model_weights as mw - JOIN users - ON users.token = mw.creator_user_token + m.token, + + m.origin_category, + m.origin_product_category, + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_email_gravatar_hash, + + m.media_type, + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + + FROM media_files as m + LEFT OUTER JOIN users + ON users.token = m.maybe_creator_user_token WHERE - mw.creator_set_visibility = "public" - AND mw.token IN ( + m.creator_set_visibility = "public" + AND m.token IN ( "#, ) } else { QueryBuilder::new(r#" SELECT - mw.token, - mw.title, - mw.weights_type, - mw.weights_category, - mw.maybe_thumbnail_token, - users.token as creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - users.email_gravatar_hash as creator_email_gravatar_hash, - mw.private_bucket_hash, - mw.maybe_private_bucket_prefix, - mw.maybe_private_bucket_extension, - mw.cached_user_ratings_negative_count, - mw.cached_user_ratings_positive_count, - mw.cached_user_ratings_total_count, - mw.maybe_cached_user_ratings_ratio, - mw.cached_user_ratings_last_updated_at, - mw.created_at, - mw.updated_at, - mw.user_deleted_at, - mw.mod_deleted_at - FROM model_weights as mw - JOIN users - ON users.token = mw.creator_user_token + m.token, + + m.origin_category, + m.origin_product_category, + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_email_gravatar_hash, + + m.media_type, + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + + FROM media_files as m + LEFT OUTER JOIN users + ON users.token = m.maybe_creator_user_token WHERE - mw.creator_set_visibility = "public" - AND mw.user_deleted_at IS NULL - AND mw.mod_deleted_at IS NULL - AND mw.token IN ( + m.creator_set_visibility = "public" + AND m.user_deleted_at IS NULL + AND m.mod_deleted_at IS NULL + AND m.token IN ( "# ) }; - query_builder.push(token_predicate(weight_tokens)); + query_builder.push(token_predicate(media_file_tokens)); query_builder.push(")"); - let query = query_builder.build_query_as::(); + let query = query_builder.build_query_as::(); let results = query.fetch_all(connection).await?; @@ -146,7 +135,7 @@ async fn get_raw_weights_by_tokens( /// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec, etc. /// Issue: https://github.com/launchbadge/sqlx/issues/875 -fn token_predicate(tokens: &[ModelWeightToken]) -> String { +fn token_predicate(tokens: &[MediaFileToken]) -> String { tokens.iter() .map(|ty| ty.as_str()) .map(|ty| format!("\"{}\"", ty)) @@ -154,73 +143,57 @@ fn token_predicate(tokens: &[ModelWeightToken]) -> String { .join(", ") } -fn map_to_weights(dataset:Vec) -> Vec { - let weights: Vec = dataset +fn map_to_media_files(dataset:Vec) -> Vec { + let media_files: Vec = dataset .into_iter() - .map(|dataset: RawWeightJoinUser| { - WeightsByTokensRecord { - token: dataset.token, - title: dataset.title, - weights_type: dataset.weights_type, - weights_category: dataset.weights_category, - maybe_thumbnail_token: dataset.maybe_thumbnail_token, + .map(|media_file: RawMediaFileJoinUser| { + MediaFilesByTokensRecord { + token: media_file.token, - creator_user_token: dataset.creator_user_token, - creator_username:dataset.creator_username, - creator_display_name:dataset.creator_display_name, - creator_email_gravatar_hash:dataset.creator_email_gravatar_hash, + origin_category: media_file.origin_category, + origin_product_category: media_file.origin_product_category, + maybe_origin_model_type: media_file.maybe_origin_model_type, + maybe_origin_model_token: media_file.maybe_origin_model_token, - private_bucket_hash: dataset.private_bucket_hash, - maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix, - maybe_private_bucket_extension: dataset.maybe_private_bucket_extension, + maybe_creator_user_token: media_file.maybe_creator_user_token, + maybe_creator_username: media_file.maybe_creator_username, + maybe_creator_display_name: media_file.maybe_creator_display_name, + maybe_creator_email_gravatar_hash: media_file.maybe_creator_email_gravatar_hash, - cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count, - cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count, - cached_user_ratings_total_count: dataset.cached_user_ratings_total_count, + media_type: media_file.media_type, + public_bucket_directory_hash: media_file.public_bucket_directory_hash, + maybe_public_bucket_prefix: media_file.maybe_public_bucket_prefix, + maybe_public_bucket_extension: media_file.maybe_public_bucket_extension, - maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio, - cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at, - - created_at: dataset.created_at, - updated_at: dataset.updated_at, - user_deleted_at: dataset.user_deleted_at, - mod_deleted_at: dataset.mod_deleted_at, + created_at: media_file.created_at, + updated_at: media_file.updated_at, } }).collect(); - weights + media_files } #[derive(Serialize)] - pub struct RawWeightJoinUser { - pub token: ModelWeightToken, + pub struct RawMediaFileJoinUser { + pub token: MediaFileToken, - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, - - pub title: String, - - pub maybe_thumbnail_token: Option, + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, - pub creator_user_token: UserToken, - pub creator_username: String, - pub creator_display_name: String, - pub creator_email_gravatar_hash: String, + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_email_gravatar_hash: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, - pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, - pub maybe_private_bucket_extension: Option, - - pub cached_user_ratings_total_count: u32, - pub cached_user_ratings_positive_count: u32, - pub cached_user_ratings_negative_count: u32, - pub maybe_cached_user_ratings_ratio: Option, - pub cached_user_ratings_last_updated_at: DateTime, - pub created_at: DateTime, pub updated_at: DateTime, - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, } // NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or @@ -235,30 +208,25 @@ fn map_to_weights(dataset:Vec) -> Vec // This results in the automatic mapping not being able to be found by name (for macro derive), and // in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the // full "as" clause). -impl FromRow<'_, MySqlRow> for RawWeightJoinUser { +impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { fn from_row(row: &MySqlRow) -> Result { Ok(Self { - token: ModelWeightToken::new(row.try_get("token")?), - weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?, - weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?, - title: row.try_get("title")?, - maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?, - creator_user_token: UserToken::new_from_str(row.try_get("creator_user_token")?), - creator_username: row.try_get("creator_username")?, - creator_display_name: row.try_get("creator_display_name")?, - creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?, - private_bucket_hash: row.try_get("private_bucket_hash")?, - maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?, - maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?, - cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?, - cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?, - cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?, - maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?, - cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + maybe_creator_user_token: row.try_get::, _>("maybe_creator_user_token")? + .and_then(|token| Some(UserToken::new_from_str(&token))), + maybe_creator_username: row.try_get("maybe_creator_username")?, + maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, + maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, - user_deleted_at: row.try_get("user_deleted_at")?, - mod_deleted_at: row.try_get("mod_deleted_at")?, }) } } From a88d888c18cb4f51e4d5e2bff9ee3b25b44281b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 16:35:07 -0500 Subject: [PATCH 31/40] featured media weights endpoint --- .../media_files/list_media_files_by_tokens.rs | 12 +- .../media_files/list_featured_media_files.rs | 170 ++++++++++++++++++ .../http_server/endpoints/media_files/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 5 + 4 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs index a6e06e49f..d55cf8be8 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_by_tokens.rs @@ -15,12 +15,13 @@ use tokens::tokens::users::UserToken; pub struct MediaFilesByTokensRecord { pub token: MediaFileToken, + pub media_type: MediaFileType, + pub origin_category: MediaFileOriginCategory, pub origin_product_category: MediaFileOriginProductCategory, pub maybe_origin_model_type: Option, pub maybe_origin_model_token: Option, - pub media_type: MediaFileType, pub public_bucket_directory_hash: String, pub maybe_public_bucket_prefix: Option, pub maybe_public_bucket_extension: Option, @@ -59,6 +60,7 @@ async fn get_raw_media_files_by_tokens( QueryBuilder::new(r#" SELECT m.token, + m.media_type, m.origin_category, m.origin_product_category, @@ -70,7 +72,6 @@ async fn get_raw_media_files_by_tokens( users.display_name as maybe_creator_display_name, users.email_gravatar_hash as maybe_creator_email_gravatar_hash, - m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, @@ -91,6 +92,7 @@ async fn get_raw_media_files_by_tokens( QueryBuilder::new(r#" SELECT m.token, + m.media_type, m.origin_category, m.origin_product_category, @@ -102,7 +104,6 @@ async fn get_raw_media_files_by_tokens( users.display_name as maybe_creator_display_name, users.email_gravatar_hash as maybe_creator_email_gravatar_hash, - m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, @@ -177,6 +178,8 @@ fn map_to_media_files(dataset:Vec) -> Vec, @@ -187,7 +190,6 @@ fn map_to_media_files(dataset:Vec) -> Vec, pub maybe_creator_email_gravatar_hash: Option, - pub media_type: MediaFileType, pub public_bucket_directory_hash: String, pub maybe_public_bucket_prefix: Option, pub maybe_public_bucket_extension: Option, @@ -212,6 +214,7 @@ impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { fn from_row(row: &MySqlRow) -> Result { Ok(Self { token: MediaFileToken::new(row.try_get("token")?), + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, @@ -221,7 +224,6 @@ impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser { maybe_creator_username: row.try_get("maybe_creator_username")?, maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?, - media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs new file mode 100644 index 000000000..55cbfaef7 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_featured_media_files.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use chrono::{DateTime, Utc}; +use log::{debug, error, warn}; +use r2d2_redis::redis::Commands; +use utoipa::ToSchema; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; + +use enums::by_table::model_weights::{ + weights_category::WeightsCategory, + weights_types::WeightsType, +}; +use mysql_queries::queries::media_files::list_media_files_by_tokens::list_media_files_by_tokens; +use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::server_state::ServerState; + +#[derive(Serialize, ToSchema)] +pub struct ListFeaturedMediaFilesSuccessResponse { + pub success: bool, + pub media_files: Vec, +} + +#[derive(Serialize, ToSchema)] +pub struct MediaFile { + pub token: MediaFileToken, + + /// Type of media will dictate which fields are populated and what + /// the frontend should display (eg. video player vs audio player). + pub media_type: MediaFileType, + + /// URL to the media file + pub public_bucket_path: String, + + // TODO: Provenance data - hide some of the models we use, especially zero shot. + //pub origin_category: MediaFileOriginCategory, + //pub origin_product_category: MediaFileOriginProductCategory, + //pub maybe_origin_model_type: Option, + //pub maybe_origin_model_token: Option, + + pub maybe_creator: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// The key we store featured media file tokens under +const REDIS_KEY : &str = "featured_media_files_list"; + +#[derive(Debug, ToSchema)] +pub enum ListFeaturedMediaFilesError { + NotAuthorized, + ServerError, +} + +impl std::fmt::Display for ListFeaturedMediaFilesError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl ResponseError for ListFeaturedMediaFilesError { + fn status_code(&self) -> StatusCode { + match *self { + ListFeaturedMediaFilesError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListFeaturedMediaFilesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[utoipa::path( + get, + path = "/v1/media_files/list_featured", + responses( + (status = 200, description = "List Featured Media Files", body = ListFeaturedMediaFilesSuccessResponse), + (status = 401, description = "Not authorized", body = ListFeaturedMediaFilesError), + (status = 500, description = "Server error", body = ListFeaturedMediaFilesError), + ), +)] +pub async fn list_featured_media_files_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let mut redis = server_state.redis_pool.get() + .map_err(|err| { + error!("Could not obtain redis: {err}"); + ListFeaturedMediaFilesError::ServerError + })?; + + let token_list : Option = redis.get(REDIS_KEY) + .map_err(|err| { + error!("Could not get redis result: {err}"); + ListFeaturedMediaFilesError::ServerError + })?; + + let media_file_tokens = token_list + .unwrap_or_else(|| "".to_string()) + .split(",") + .into_iter() + .map(|item| item.trim()) + .filter(|item| !item.is_empty()) + .map(|item| MediaFileToken::new_from_str(item)) + .collect::>(); + + debug!("Weight tokens from Redis: {:?}", media_file_tokens); + + let mut media_files = Vec::new(); + + if !media_file_tokens.is_empty() { + let query_results = + list_media_files_by_tokens(&server_state.mysql_pool, &media_file_tokens, false).await; + + media_files = match query_results { + Ok(media_files) => media_files, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListFeaturedMediaFilesError::ServerError); + } + }; + } + + let response = ListFeaturedMediaFilesSuccessResponse { + success: true, + media_files: media_files.into_iter() + .map(|m| { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix.as_deref(), + m.maybe_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string(); + + MediaFile { + token: m.token, + media_type: m.media_type, + public_bucket_path, + //origin_category: m.origin_category, + //origin_product_category: m.origin_product_category, + //maybe_origin_model_type: m.maybe_origin_model_type, + //maybe_origin_model_token: m.maybe_origin_model_token, + maybe_creator: UserDetailsLight::from_optional_db_fields_owned( + m.maybe_creator_user_token, + m.maybe_creator_username, + m.maybe_creator_display_name, + m.maybe_creator_email_gravatar_hash + ), + created_at: m.created_at, + updated_at: m.updated_at, + } + }).collect::>(), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListFeaturedMediaFilesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index ab3552ac2..2893bf89c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,5 +1,6 @@ pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; +pub mod list_featured_media_files; pub mod list_media_files; pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 5585ee137..7e9569dad 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -45,6 +45,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_featured_media_files::list_featured_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; @@ -1024,6 +1025,10 @@ fn add_media_file_routes (app: App) -> App .route(web::get().to(list_media_files_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/list_featured") + .route(web::get().to(list_featured_media_files_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/list/user/{username}") .route(web::get().to(list_media_files_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 60d419921617164e499e8efa1402216ecdb4e339 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 16:58:37 -0500 Subject: [PATCH 32/40] fix logout The reason this is so labyrinthine is because of the users and billing components. They bind dynamic behaviors that are defined in storyteller-web (one of them being clearing the redis session cache). Actually, the users component _didn't_, because it was defined in the billing component. So I hoisted it out into a new component library. In the fullness of time I imagine all of this will be consolidated back under storyteller-web so we don't have to worry about binding actions at a distance. Anyway, this is tested and works locally. --- Cargo.lock | 13 +++++++++ Cargo.toml | 1 + crates/service/plugins/billing/Cargo.toml | 1 + ...pe_create_checkout_session_json_handler.rs | 2 +- ...reate_checkout_session_redirect_handler.rs | 2 +- ...te_customer_portal_session_json_handler.rs | 2 +- ...ustomer_portal_session_redirect_handler.rs | 2 +- .../plugins/billing/src/stripe/traits/mod.rs | 1 - .../plugins/user_traits_component/Cargo.toml | 27 +++++++++++++++++++ .../plugins/user_traits_component/src/lib.rs | 21 +++++++++++++++ .../traits/internal_session_cache_purge.rs | 0 .../user_traits_component/src/traits/mod.rs | 1 + crates/service/plugins/users/Cargo.toml | 1 + .../users/src/endpoints/logout_handler.rs | 5 ++++ crates/service/web/storyteller_web/Cargo.toml | 1 + .../internal_session_cache_purge_impl.rs | 2 +- .../service/web/storyteller_web/src/main.rs | 16 +++++------ 17 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 crates/service/plugins/user_traits_component/Cargo.toml create mode 100644 crates/service/plugins/user_traits_component/src/lib.rs rename crates/service/plugins/{billing/src/stripe => user_traits_component/src}/traits/internal_session_cache_purge.rs (100%) create mode 100644 crates/service/plugins/user_traits_component/src/traits/mod.rs diff --git a/Cargo.lock b/Cargo.lock index af9894f87..8d6d76535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -813,6 +813,7 @@ dependencies = [ "tokio", "toml 0.5.11", "url_config", + "user_traits_component", ] [[package]] @@ -5954,6 +5955,7 @@ dependencies = [ "twitch_oauth2", "url_config", "user_input_common", + "user_traits_component", "users_component", "utoipa", "utoipa-swagger-ui", @@ -7066,6 +7068,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "user_traits_component" +version = "0.0.1" +dependencies = [ + "actix-web", + "errors", + "mockall", + "my-workspace-hack", +] + [[package]] name = "users_component" version = "0.0.1" @@ -7109,6 +7121,7 @@ dependencies = [ "tokens", "toml 0.5.11", "user_input_common", + "user_traits_component", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1ea5d8fef..411f131e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ members = [ # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.) "crates/service/plugins/billing", "crates/service/plugins/inference_workloads", + "crates/service/plugins/user_traits_component", "crates/service/plugins/users", # Vendored code diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 869eee779..56a817678 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -20,6 +20,7 @@ http_server_common = { path = "../../../lib/deprecated/http_server_common" } mysql_queries = { path = "../../../lib/mysql_queries" } reusable_types = { path = "../../../_types/reusable_types" } url_config = { path = "../../../lib/url_config" } +user_traits_component = { path = "../user_traits_component" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs index bdc4de4af..c81a63b39 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_json_handler.rs @@ -3,12 +3,12 @@ use actix_web::web::Json; use reusable_types::server_environment::ServerEnvironment; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs index 3f6ccc221..657761495 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/checkout/create/stripe_create_checkout_session_redirect_handler.rs @@ -4,12 +4,12 @@ use actix_web::web::Query; use reusable_types::server_environment::ServerEnvironment; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError; use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs index 3e41b3b2b..bb21b8162 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_json_handler.rs @@ -2,12 +2,12 @@ use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::web::Json; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs index c4e69a328..4aafd6e37 100644 --- a/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs +++ b/crates/service/plugins/billing/src/stripe/http_endpoints/customer_portal/stripe_create_customer_portal_session_redirect_handler.rs @@ -3,12 +3,12 @@ use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::web::Query; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError; use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared; use crate::stripe::stripe_config::StripeConfig; use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::stripe::traits::internal_user_lookup::InternalUserLookup; // =============== Request =============== diff --git a/crates/service/plugins/billing/src/stripe/traits/mod.rs b/crates/service/plugins/billing/src/stripe/traits/mod.rs index 2cf3b026f..44d38b0ea 100644 --- a/crates/service/plugins/billing/src/stripe/traits/mod.rs +++ b/crates/service/plugins/billing/src/stripe/traits/mod.rs @@ -1,4 +1,3 @@ pub mod internal_product_to_stripe_lookup; -pub mod internal_session_cache_purge; pub mod internal_subscription_product_lookup; pub mod internal_user_lookup; \ No newline at end of file diff --git a/crates/service/plugins/user_traits_component/Cargo.toml b/crates/service/plugins/user_traits_component/Cargo.toml new file mode 100644 index 000000000..6521233d9 --- /dev/null +++ b/crates/service/plugins/user_traits_component/Cargo.toml @@ -0,0 +1,27 @@ +[package] +publish = false +name = "user_traits_component" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[lib] +name = "user_traits_component" +path = "src/lib.rs" + +[dependencies] + +# Internal +errors = { path = "../../../lib/errors" } + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } + +# Web +actix-web = "4.4.0" + +[dev-dependencies] +mockall = "0.11.3" diff --git a/crates/service/plugins/user_traits_component/src/lib.rs b/crates/service/plugins/user_traits_component/src/lib.rs new file mode 100644 index 000000000..a490ebd66 --- /dev/null +++ b/crates/service/plugins/user_traits_component/src/lib.rs @@ -0,0 +1,21 @@ +//! user_traits_component +//! +//! Expose some traits for other plugins / components. +//! This will probably go away once we start consolidating the plugins into the main service. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod traits; diff --git a/crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs b/crates/service/plugins/user_traits_component/src/traits/internal_session_cache_purge.rs similarity index 100% rename from crates/service/plugins/billing/src/stripe/traits/internal_session_cache_purge.rs rename to crates/service/plugins/user_traits_component/src/traits/internal_session_cache_purge.rs diff --git a/crates/service/plugins/user_traits_component/src/traits/mod.rs b/crates/service/plugins/user_traits_component/src/traits/mod.rs new file mode 100644 index 000000000..01df49e0e --- /dev/null +++ b/crates/service/plugins/user_traits_component/src/traits/mod.rs @@ -0,0 +1 @@ +pub mod internal_session_cache_purge; diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 86db22147..7d2a8f616 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -30,6 +30,7 @@ reusable_types = { path = "../../../_types/reusable_types" } server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } user_input_common = { path = "../../../lib/user_input_common" } +user_traits_component = { path = "../user_traits_component" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/src/endpoints/logout_handler.rs b/crates/service/plugins/users/src/endpoints/logout_handler.rs index 806b525d4..4c5137535 100644 --- a/crates/service/plugins/users/src/endpoints/logout_handler.rs +++ b/crates/service/plugins/users/src/endpoints/logout_handler.rs @@ -13,6 +13,7 @@ use sqlx::MySqlPool; use http_server_common::response::response_error_helpers::to_simple_json_error; use mysql_queries::queries::users::user_sessions::delete_user_session::delete_user_session; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use crate::cookies::session::session_cookie_manager::SessionCookieManager; @@ -53,8 +54,12 @@ pub async fn logout_handler( http_request: HttpRequest, session_cookie_manager: web::Data, mysql_pool: web::Data, + internal_session_cache_purge: web::Data, ) -> Result { + // Best effort to delete Redis session cache + internal_session_cache_purge.best_effort_purge_session_cache(&http_request); + let mut delete_cookie = match http_request.cookie("session") { Some(cookie) => { match session_cookie_manager.decode_session_token(&cookie) { diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index bc0e4b61b..6767b3324 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -52,6 +52,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # Internal Components billing_component = { path = "../../plugins/billing" } +user_traits_component = { path = "../../plugins/user_traits_component" } users_component = { path = "../../plugins/users" } # Workspace hack (faster compile times) diff --git a/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs b/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs index c296ed529..2dcf6eb52 100644 --- a/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs +++ b/crates/service/web/storyteller_web/src/billing/internal_session_cache_purge_impl.rs @@ -1,8 +1,8 @@ use actix_web::HttpRequest; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use redis_caching::redis_ttl_cache::RedisTtlCache; use redis_common::redis_cache_keys::RedisCacheKeys; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use users_component::utils::session_checker::SessionChecker; pub struct InternalSessionCachePurgeImpl { diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 57f09a82c..8b815cd6f 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -20,15 +20,13 @@ use std::sync::Arc; use std::time::Duration; -use actix_cors::Cors; -use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::{App, HttpServer, middleware, web}; use actix_web::middleware::{DefaultHeaders, Logger}; use anyhow::anyhow; use elasticsearch::Elasticsearch; use elasticsearch::http::transport::Transport; use futures::Future; -use limitation::Limiter; -use log::{error, info}; +use log::info; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; @@ -48,7 +46,6 @@ use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exa use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; @@ -56,10 +53,9 @@ use cloud_storage::bucket_client::BucketClient; use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; use email_sender::smtp_email_sender::SmtpEmailSender; use errors::AnyhowResult; -use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use http_server_common::cors::build_cors_config; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; @@ -67,6 +63,7 @@ use redis_caching::redis_ttl_cache::RedisTtlCache; use reusable_types::server_environment::ServerEnvironment; use twitch_common::twitch_secrets::TwitchSecrets; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge; use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; use users_component::cookies::session::session_cookie_manager::SessionCookieManager; use users_component::utils::session_checker::SessionChecker; @@ -76,12 +73,11 @@ use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurge use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limiters; -use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::configs::static_api_tokens::StaticApiTokenSet; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; -use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; use crate::routes::add_routes; -use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; From e9cd505e0bbe5a2c6a547e0fc75d84f3ec026f5c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 18:11:34 -0500 Subject: [PATCH 33/40] Fix clap in various CLI tools post 2.x -> 4.x upgrade --- Cargo.lock | 2 - crates/cli/dev_database_seed/Cargo.toml | 2 +- crates/cli/dev_database_seed/src/cli_args.rs | 25 +++----- crates/cli/dev_upload_media_file/Cargo.toml | 2 +- .../cli/dev_upload_media_file/src/cli_args.rs | 29 ++++----- crates/cli/elasticsearch-cli/Cargo.toml | 2 +- crates/cli/elasticsearch-cli/src/cli_args.rs | 63 +++++++------------ crates/cli/scrape_test/Cargo.toml | 2 +- crates/cli/scrape_test/src/main.rs | 34 +++++----- .../service/job/email_sender_job/Cargo.toml | 1 - .../service/job/tts_inference_job/Cargo.toml | 1 - .../service/job/tts_inference_job/src/main.rs | 16 +---- 12 files changed, 63 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc0a9d964..7af174d2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2068,7 +2068,6 @@ dependencies = [ "bootstrap", "buckets", "chrono", - "clap", "cloud_storage", "concurrency", "config", @@ -6707,7 +6706,6 @@ version = "0.0.1" dependencies = [ "anyhow", "chrono", - "clap", "cloud_storage", "collections", "config", diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index b065225a7..26c93882f 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -33,7 +33,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External log = "0.4.14" -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" rand = "0.8.5" # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. diff --git a/crates/cli/dev_database_seed/src/cli_args.rs b/crates/cli/dev_database_seed/src/cli_args.rs index 6fb2e244c..6642fe625 100644 --- a/crates/cli/dev_database_seed/src/cli_args.rs +++ b/crates/cli/dev_database_seed/src/cli_args.rs @@ -1,28 +1,17 @@ -use clap::{App, Arg}; +use clap::Parser; use errors::AnyhowResult; +#[derive(Parser, Debug)] +#[command(name="dev-database-seed")] pub struct CliArgs { + #[arg(name="seed_cloud_bucket", long="bucket", help="Seed the cloud bucket with files", required=false, default_value_t=false)] pub seed_cloud_bucket: bool, + + #[arg(name="seed_elasticsearch", long="elasticsearch", help="Seed the local elasticsearch", required=false, default_value_t=false)] pub seed_elasticsearch: bool, } pub fn parse_cli_args() -> AnyhowResult { - let matches = App::new("dev-database-seed") - .arg(Arg::with_name("seed_cloud_bucket") - .long("bucket") - .help("Seed the cloud bucket with files") - .takes_value(false) - .required(false)) - .arg(Arg::with_name("seed_elasticsearch") - .long("elasticsearch") - .help("Seed the local elasticsearch") - .takes_value(false) - .required(false)) - .get_matches(); - - Ok(CliArgs { - seed_cloud_bucket: matches.is_present("seed_cloud_bucket"), - seed_elasticsearch: matches.is_present("seed_elasticsearch"), - }) + Ok(CliArgs::parse()) } diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 9d928ea9e..ad35f9d3b 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -29,7 +29,7 @@ tokens = { path = "../../_types/tokens" } my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" log = "0.4.14" # Runtime diff --git a/crates/cli/dev_upload_media_file/src/cli_args.rs b/crates/cli/dev_upload_media_file/src/cli_args.rs index 1cf2ca590..32ad7a5f4 100644 --- a/crates/cli/dev_upload_media_file/src/cli_args.rs +++ b/crates/cli/dev_upload_media_file/src/cli_args.rs @@ -1,27 +1,24 @@ use std::path::PathBuf; -use clap::{App, Arg}; -use errors::{anyhow, AnyhowResult}; + +use clap::Parser; + +use errors::AnyhowResult; + +#[derive(Parser, Debug)] +#[command(name="dev-upload-media-file")] +pub struct RawArgs { + #[arg(name="file", short='f', long="file", help="File to upload", required=true)] + pub file_path: String, +} pub struct CliArgs { pub file_path: PathBuf, } pub fn parse_cli_args() -> AnyhowResult { - let matches = App::new("dev_upload_media") - .arg(Arg::with_name("file") - .short("f") - .long("file") - .value_name("FILE") - .help("File to upload") - .takes_value(true) - .required(true)) - .get_matches(); - - let file_path = matches.value_of("file") - .map(|val| val.trim().to_string()) - .ok_or(anyhow!("no file path supplied"))?; + let args = RawArgs::parse(); Ok(CliArgs { - file_path: PathBuf::from(file_path), + file_path: PathBuf::from(args.file_path), }) } diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index 9b0af1cde..6921e54cd 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -29,7 +29,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External log = "0.4.14" -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" strum = { version = "0.25", features = ["derive"] } # Runtime diff --git a/crates/cli/elasticsearch-cli/src/cli_args.rs b/crates/cli/elasticsearch-cli/src/cli_args.rs index 5c2c7e1a4..9b7a25e81 100644 --- a/crates/cli/elasticsearch-cli/src/cli_args.rs +++ b/crates/cli/elasticsearch-cli/src/cli_args.rs @@ -1,12 +1,12 @@ use std::str::FromStr; -use clap::{App, Arg, ArgMatches}; +use clap::Parser; use strum::{EnumCount, EnumString, IntoEnumIterator}; use strum::EnumIter; use errors::{anyhow, AnyhowResult, bail}; -pub struct CliArgs { +pub struct ParsedArgs { pub mysql_environment: Environment, pub elasticsearch_environment: Environment, pub action: Action, @@ -25,53 +25,38 @@ pub enum Action { SearchTts, } -pub fn parse_cli_args() -> AnyhowResult { - let matches = App::new("dev-database-seed") - .arg(Arg::with_name("action") - .long("action") - .help("action to take") - .takes_value(true) - .required(true)) - .arg(Arg::with_name("mysql") - .long("mysql") - .help("Production or development") - .takes_value(true) - .required(false)) - .arg(Arg::with_name("elasticsearch") - .long("elasticsearch") - .help("Production or development") - .takes_value(true) - .required(false)) - .get_matches(); +#[derive(Parser, Debug)] +#[command(name="elasticsearch-cli")] +pub struct Args { + #[arg(name="action", long="action", help="action to take", required=true)] + action: String, - Ok(CliArgs { - mysql_environment: to_environment(&matches, "mysql")?, - elasticsearch_environment: to_environment(&matches, "elasticsearch")?, - action: get_action(&matches)?, + #[arg(name="mysql", long="mysql", help="production or development")] + mysql: Option, + + #[arg(name="elasticsearch", long="elasticsearch", help="production or development")] + elasticsearch: Option, +} + +pub fn parse_cli_args() -> AnyhowResult { + let args = Args::parse(); + + Ok(ParsedArgs { + mysql_environment: to_environment(args.mysql.as_deref())?, + elasticsearch_environment: to_environment(args.elasticsearch.as_deref())?, + action: action_from_str(&args.action)?, }) } -fn to_environment(matches: &ArgMatches, key: &str) -> AnyhowResult { - let value = matches.value_of(key) - .map(|s| s.to_lowercase()); - - Ok(match value.as_deref() { +fn to_environment(environment: Option<&str>) -> AnyhowResult { + Ok(match environment { None => Environment::Development, Some("dev") | Some("development") => Environment::Development, Some("prod") | Some("production") => Environment::Production, - _ => bail!("invalid environment: {:?}", value), + _ => bail!("invalid environment: {:?}", environment), }) } -fn get_action(matches: &ArgMatches) -> AnyhowResult { - let action = matches.value_of("action") - .map(|s| action_from_str(s)) - .transpose()? - .ok_or(anyhow!("no action provided"))?; - - Ok(action) -} - fn action_from_str(value: &str) -> AnyhowResult { let action = Action::from_str(value) .map_err(|err| { diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml index f68f20dc3..64dfd5adb 100644 --- a/crates/cli/scrape_test/Cargo.toml +++ b/crates/cli/scrape_test/Cargo.toml @@ -22,5 +22,5 @@ web_scrapers = { path = "../../lib/web_scrapers" } my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } # External -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated +clap = "4.4.8" tokio = { version = "1.34.0", features = ["macros"] } diff --git a/crates/cli/scrape_test/src/main.rs b/crates/cli/scrape_test/src/main.rs index 71abbd5d1..42d3c8a51 100644 --- a/crates/cli/scrape_test/src/main.rs +++ b/crates/cli/scrape_test/src/main.rs @@ -1,30 +1,24 @@ -use clap::{App, Arg}; +use clap::Parser; use errors::{anyhow, AnyhowResult}; use web_scrapers::scrape_supported_webpage::scrape_supported_webpage; +#[derive(Parser, Debug)] +#[command(name = "scrape_test")] +pub struct Args { + #[arg(name="url", short='u', long = "url", help = "URL of the page to scrape", required = true)] + url: String, + + #[arg(name="print_html", long = "html", help = "Print the HTML scraped", required = true)] + print_html: bool, +} + #[tokio::main] pub async fn main() -> AnyhowResult<()> { - let matches = App::new("scrape_test") - .arg(Arg::with_name("url") - .short("u") - .long("url") - .value_name("URL") - .help("URL of the page to scrape") - .takes_value(true) - .required(true)) - .arg(Arg::with_name("print_html") - .long("html") - .help("Print the HTML scraped") - .takes_value(false) - .required(false)) - .get_matches(); + let args = Args::parse(); - let url = matches.value_of("url") - .map(|val| val.trim().to_string()) - .ok_or(anyhow!("no url supplied"))?; - - let print_html = matches.is_present("print_html"); + let url = args.url.trim().to_string(); + let print_html = args.print_html; let scrape_result = scrape_supported_webpage(&url).await?; diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 07db875a2..7fd93708f 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -47,7 +47,6 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 02b6a01ed..3695ed9a2 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -33,7 +33,6 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.75" chrono = { version = "0.4.22", features = ["serde"] } -clap = "4.4.8" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_inference_job/src/main.rs b/crates/service/job/tts_inference_job/src/main.rs index db471482a..18b2dc41c 100644 --- a/crates/service/job/tts_inference_job/src/main.rs +++ b/crates/service/job/tts_inference_job/src/main.rs @@ -18,7 +18,6 @@ use std::path::PathBuf; use std::time::Duration; -use clap::{App, Arg}; use log::{error, info, warn}; use r2d2_redis::r2d2; use r2d2_redis::RedisConnectionManager; @@ -93,15 +92,6 @@ async fn main() -> AnyhowResult<()> { // This file should only contain *development* secrets, never production. let _ = dotenv::from_filename(".env-secrets").ok(); - let matches = App::new("tts-inference-job") - .arg(Arg::with_name("sidecar_hostname") - .long("sidecar_hostname") - .value_name("HOSTNAME") - .help("Hostname for the TTS inference sidecar") - .takes_value(true) - .required(false)) - .get_matches(); - info!("Obtaining worker hostname..."); let server_hostname = hostname::get() @@ -150,13 +140,9 @@ async fn main() -> AnyhowResult<()> { Some(bucket_timeout), )?; - let mut sidecar_hostname = + let sidecar_hostname = easyenv::get_env_string_required(ENV_TTS_INFERENCE_SIDECAR_HOSTNAME)?; - if let Some(hostname) = matches.value_of("sidecar_hostname") { - sidecar_hostname = hostname.to_string(); - } - info!("Sidecar hostname: {:?}", sidecar_hostname); let tts_inference_sidecar_client = From 6c9b0a5c2abe241ce336b52928be5302b2eaef3e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 18:30:54 -0500 Subject: [PATCH 34/40] fix bad github merge --- .../http_server/endpoints/media_files/mod.rs | 3 +-- .../service/web/storyteller_web/src/routes.rs | 19 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 1883621d7..8365645e1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,8 +1,7 @@ pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; -pub mod delete_media_file; -pub mod update_media_file; pub mod list_featured_media_files; pub mod list_media_files; pub mod list_media_files_for_user; +pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 79c10e153..aafc1d144 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -29,11 +29,6 @@ use crate::http_server::endpoints::comments::list_comments_handler::list_comment use crate::http_server::endpoints::download_job::enqueue_generic_download::enqueue_generic_download_handler; use crate::http_server::endpoints::download_job::get_generic_upload_job_status::get_generic_download_job_status_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; -use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler; -use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler; -use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; -use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; -use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler; use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; @@ -45,10 +40,10 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; -use crate::http_server::endpoints::media_files::update_media_file::update_media_file_handler; use crate::http_server::endpoints::media_files::list_featured_media_files::list_featured_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; +use crate::http_server::endpoints::media_files::update_media_file::update_media_file_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -112,6 +107,11 @@ use crate::http_server::endpoints::twitch::oauth::check_oauth_status::check_oaut use crate::http_server::endpoints::twitch::oauth::oauth_begin_json::oauth_begin_enroll_json; use crate::http_server::endpoints::twitch::oauth::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::http_server::endpoints::twitch::oauth::oauth_end::oauth_end_enroll_from_redirect; +use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; +use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; use crate::http_server::endpoints::user_ratings::get_user_rating_handler::get_user_rating_handler; use crate::http_server::endpoints::user_ratings::set_user_rating_handler::set_user_rating_handler; use crate::http_server::endpoints::vocoders::get_vocoder::get_vocoder_handler; @@ -154,14 +154,12 @@ use crate::http_server::endpoints::w2l::list_user_w2l_inference_results::list_us use crate::http_server::endpoints::w2l::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; - -use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::delete_weight::delete_weight_handler; -use crate::http_server::endpoints::weights::update_weight::update_weight_handler; +use crate::http_server::endpoints::weights::get_weight::get_weight_handler; use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler; use crate::http_server::endpoints::weights::list_featured_weights::list_featured_weights_handler; use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler; - +use crate::http_server::endpoints::weights::update_weight::update_weight_handler; pub fn add_routes (app: App, server_environment: ServerEnvironment) -> App where @@ -1024,6 +1022,7 @@ fn add_media_file_routes (app: App) -> App ) .service(web::resource("/file/{token}/update") .route(web::post().to(update_media_file_handler)) + ) .service(web::resource("/list") .route(web::get().to(list_media_files_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 186979ecbdbe432b86f64362b27d8f8f075d7d02 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 18:40:44 -0500 Subject: [PATCH 35/40] fix lints and other issues --- .../src/seeding/seed_model_weights.rs | 12 ++++++------ crates/cli/scrape_test/src/main.rs | 2 +- .../queries/media_files/list_media_files_for_user.rs | 5 +---- .../job_types/videofilter/process_single_vf_job.rs | 7 ------- .../service/plugins/user_traits_component/src/lib.rs | 2 +- 5 files changed, 9 insertions(+), 19 deletions(-) diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs index ac6388b87..e3e025f85 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs @@ -123,8 +123,8 @@ pub async fn seed_weights_files( let create_model_weights_args = CreateModelWeightsArgs { token: &model_weight_token, - weights_type: weights_type, // Assuming weights_type is defined elsewhere - weights_category: weights_category, // Assuming weights_category is defined elsewhere + weights_type, // Assuming weights_type is defined elsewhere + weights_category, // Assuming weights_category is defined elsewhere title: title.to_string(), maybe_thumbnail_token: Some(thumbnail_token), description_markdown: description.to_string(), @@ -140,11 +140,11 @@ pub async fn seed_weights_files( private_bucket_hash: public_bucket_hash, maybe_private_bucket_prefix: Some(public_bucket_prefix.to_string()), maybe_private_bucket_extension: Some(public_bucket_extension.to_string()), - cached_user_ratings_total_count: cached_user_ratings_total_count, - cached_user_ratings_positive_count: cached_user_ratings_positive_count, - cached_user_ratings_negative_count: cached_user_ratings_negative_count, + cached_user_ratings_total_count, + cached_user_ratings_positive_count, + cached_user_ratings_negative_count, maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio), - version: version, + version, mysql_pool: &mysql_pool, // Assuming mysql_pool is defined elsewhere }; diff --git a/crates/cli/scrape_test/src/main.rs b/crates/cli/scrape_test/src/main.rs index 42d3c8a51..4053e22a8 100644 --- a/crates/cli/scrape_test/src/main.rs +++ b/crates/cli/scrape_test/src/main.rs @@ -1,6 +1,6 @@ use clap::Parser; -use errors::{anyhow, AnyhowResult}; +use errors::AnyhowResult; use web_scrapers::scrape_supported_webpage::scrape_supported_webpage; #[derive(Parser, Debug)] diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index d7115cd5a..c8f32280d 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -1,5 +1,4 @@ use chrono::{DateTime, Utc}; -use log::warn; use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; @@ -12,8 +11,6 @@ use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; - - pub struct MediaFileListPage { pub records: Vec, @@ -210,7 +207,7 @@ WHERE m.user_deleted_at IS NULL query_builder.push(" ORDER BY m.id DESC "); } - if (enforce_limits) { + if enforce_limits { if cursor_is_reversed { query_builder.push(format!(" LIMIT {limit_start}, {limit_end} ")); } else { diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index 9d7cc0163..d5ab7b371 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,11 +1,4 @@ -use anyhow::anyhow; -use log::info; - -use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; -use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_files::get_media_file::get_media_file; -use tokens::tokens::media_files::MediaFileToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; diff --git a/crates/service/plugins/user_traits_component/src/lib.rs b/crates/service/plugins/user_traits_component/src/lib.rs index a490ebd66..e3e74a585 100644 --- a/crates/service/plugins/user_traits_component/src/lib.rs +++ b/crates/service/plugins/user_traits_component/src/lib.rs @@ -12,7 +12,7 @@ #![forbid(unreachable_patterns)] #![forbid(unused_imports)] #![forbid(unused_mut)] -#![forbid(unused_variables)] +#![allow(unused_variables)] // NB: issue with automock // Always allow #![allow(dead_code)] From 6fc1dd263c1b9d0c96cfcba52fd725df50e26743 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 20:23:03 -0500 Subject: [PATCH 36/40] migration dir had a space in it --- .../down.sql | 0 .../up.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename _sql/migrations/{2023-12-11-115710_add user_bookmarks => 2023-12-11-115710_add_user_bookmarks}/down.sql (100%) rename _sql/migrations/{2023-12-11-115710_add user_bookmarks => 2023-12-11-115710_add_user_bookmarks}/up.sql (100%) diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql b/_sql/migrations/2023-12-11-115710_add_user_bookmarks/down.sql similarity index 100% rename from _sql/migrations/2023-12-11-115710_add user_bookmarks/down.sql rename to _sql/migrations/2023-12-11-115710_add_user_bookmarks/down.sql diff --git a/_sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql b/_sql/migrations/2023-12-11-115710_add_user_bookmarks/up.sql similarity index 100% rename from _sql/migrations/2023-12-11-115710_add user_bookmarks/up.sql rename to _sql/migrations/2023-12-11-115710_add_user_bookmarks/up.sql From 5108f94c0f33ca53dd6f4d3b34f7094345505069 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Dec 2023 21:45:44 -0500 Subject: [PATCH 37/40] Lots of schema changes afoot - model_weights migration, covers, samples Model weights migration (old tokens): - model_weights.maybe_migration_old_model_token Model weights migration (new tokens): - tts_models.maybe_migration_new_model_weights_token - vocoder_models.maybe_migration_new_model_weights_token - voice_conversion_models.maybe_migration_new_model_weights_token Cover and banner images for models: - model_weights.maybe_banner_media_file_token - model_weights.maybe_cover_media_file_token Avatar and banner images for users: - users.maybe_avatar_media_file_token - users.maybe_cover_media_file_token Sample multimedia for models: - model_weights_samples (new table) --- .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 6 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 6 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 10 + .../down.sql | 5 + .../up.sql | 47 ++++ _sql/migrations_squashed/model_weights.sql | 202 ++++++++++++++++++ .../tts_tables/tts_models.sql | 5 + .../tts_tables/vocoder_models.sql | 5 + .../migrations_squashed/user_tables/users.sql | 8 + .../voice_conversion_models.sql | 5 + 23 files changed, 399 insertions(+) create mode 100644 _sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/down.sql create mode 100644 _sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/up.sql create mode 100644 _sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/down.sql create mode 100644 _sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/up.sql create mode 100644 _sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/down.sql create mode 100644 _sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/up.sql create mode 100644 _sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/down.sql create mode 100644 _sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/up.sql create mode 100644 _sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/down.sql create mode 100644 _sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/up.sql create mode 100644 _sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/down.sql create mode 100644 _sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/up.sql create mode 100644 _sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/down.sql create mode 100644 _sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/up.sql create mode 100644 _sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/down.sql create mode 100644 _sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/up.sql create mode 100644 _sql/migrations/2023-12-15-044416_model_weights_samples/down.sql create mode 100644 _sql/migrations/2023-12-15-044416_model_weights_samples/up.sql create mode 100644 _sql/migrations_squashed/model_weights.sql diff --git a/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/down.sql b/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/down.sql new file mode 100644 index 000000000..6a822bee2 --- /dev/null +++ b/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights DROP COLUMN maybe_migration_old_model_token; diff --git a/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/up.sql b/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/up.sql new file mode 100644 index 000000000..dbcdd42f8 --- /dev/null +++ b/_sql/migrations/2023-12-15-015254_alter_model_weights_add_old_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights + ADD COLUMN maybe_migration_old_model_token VARCHAR(32) DEFAULT NULL + AFTER cached_user_ratings_last_updated_at; + +ALTER TABLE model_weights + ADD UNIQUE INDEX unique_index_maybe_migration_old_model_token (maybe_migration_old_model_token); diff --git a/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/down.sql b/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/down.sql new file mode 100644 index 000000000..9297e603a --- /dev/null +++ b/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE voice_conversion_models + DROP COLUMN maybe_migration_new_model_weights_token; diff --git a/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/up.sql b/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/up.sql new file mode 100644 index 000000000..b14d52115 --- /dev/null +++ b/_sql/migrations/2023-12-15-015311_alter_voice_conversion_models_add_new_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE voice_conversion_models + ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL + AFTER maybe_mod_comments; + +ALTER TABLE voice_conversion_models + ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token); diff --git a/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/down.sql b/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/down.sql new file mode 100644 index 000000000..8fae0f7f6 --- /dev/null +++ b/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models DROP COLUMN maybe_migration_new_model_weights_token; diff --git a/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/up.sql b/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/up.sql new file mode 100644 index 000000000..569ab1f59 --- /dev/null +++ b/_sql/migrations/2023-12-15-015319_alter_tts_models_add_new_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models + ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL + AFTER maybe_mod_user_token; + +ALTER TABLE tts_models + ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token); diff --git a/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/down.sql b/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/down.sql new file mode 100644 index 000000000..1238b82f6 --- /dev/null +++ b/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE vocoder_models DROP COLUMN maybe_migration_new_model_weights_token; diff --git a/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/up.sql b/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/up.sql new file mode 100644 index 000000000..f804fad6f --- /dev/null +++ b/_sql/migrations/2023-12-15-015329_alter_vocoder_models_add_new_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE vocoder_models + ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL + AFTER maybe_mod_user_token; + +ALTER TABLE vocoder_models + ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token); diff --git a/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/down.sql b/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/down.sql new file mode 100644 index 000000000..2ae3d04fc --- /dev/null +++ b/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights DROP COLUMN maybe_avatar_media_file_token; diff --git a/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/up.sql b/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/up.sql new file mode 100644 index 000000000..2596c1afb --- /dev/null +++ b/_sql/migrations/2023-12-15-024344_alter_model_weights_add_avatar_media_file_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights + ADD COLUMN maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL + AFTER maybe_thumbnail_token; + +ALTER TABLE model_weights + ADD INDEX index_maybe_avatar_media_file_token (maybe_avatar_media_file_token); diff --git a/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/down.sql b/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/down.sql new file mode 100644 index 000000000..6458b4459 --- /dev/null +++ b/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights DROP COLUMN maybe_cover_media_file_token; + diff --git a/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/up.sql b/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/up.sql new file mode 100644 index 000000000..a04e0a1b1 --- /dev/null +++ b/_sql/migrations/2023-12-15-024350_alter_model_weights_add_cover_media_file_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights + ADD COLUMN maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL + AFTER maybe_avatar_media_file_token; + +ALTER TABLE model_weights + ADD INDEX index_maybe_cover_media_file_token (maybe_cover_media_file_token); diff --git a/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/down.sql b/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/down.sql new file mode 100644 index 000000000..2e84ec6bc --- /dev/null +++ b/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE users DROP COLUMN maybe_avatar_media_file_token; diff --git a/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/up.sql b/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/up.sql new file mode 100644 index 000000000..64c9a8a3d --- /dev/null +++ b/_sql/migrations/2023-12-15-034614_alter_users_add_avatar_media_file_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE users + ADD COLUMN maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL + AFTER ip_address_last_update; + +ALTER TABLE users + ADD INDEX index_maybe_avatar_media_file_token (maybe_avatar_media_file_token); diff --git a/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/down.sql b/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/down.sql new file mode 100644 index 000000000..45237d79d --- /dev/null +++ b/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE users DROP COLUMN maybe_cover_media_file_token; diff --git a/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/up.sql b/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/up.sql new file mode 100644 index 000000000..a467e5066 --- /dev/null +++ b/_sql/migrations/2023-12-15-034644_alter_users_add_cover_media_file_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE users + ADD COLUMN maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL + AFTER maybe_avatar_media_file_token; + +ALTER TABLE users + ADD INDEX index_maybe_cover_media_file_token (maybe_cover_media_file_token); diff --git a/_sql/migrations/2023-12-15-044416_model_weights_samples/down.sql b/_sql/migrations/2023-12-15-044416_model_weights_samples/down.sql new file mode 100644 index 000000000..9834047d6 --- /dev/null +++ b/_sql/migrations/2023-12-15-044416_model_weights_samples/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE model_weights_samples; diff --git a/_sql/migrations/2023-12-15-044416_model_weights_samples/up.sql b/_sql/migrations/2023-12-15-044416_model_weights_samples/up.sql new file mode 100644 index 000000000..f8d6752a8 --- /dev/null +++ b/_sql/migrations/2023-12-15-044416_model_weights_samples/up.sql @@ -0,0 +1,47 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE model_weights_samples ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- ========== WEIGHTS CLASSIFICATION ========== + + -- Foreign key to model weights. + -- This is in a composite index with media_file_token to make sure a media file only applies + -- to a given model once. + model_weight_token VARCHAR(32) NOT NULL, + + -- Foreign key to media files + -- This is in a composite index with model_weight_token to make sure a media file only applies + -- to a given model once. + media_file_token VARCHAR(32) NOT NULL, + + -- Information about who set the sample + -- * automation + -- * author + -- * community + set_by VARCHAR(16) NOT NULL, + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weight_token, media_file_token), + + KEY index_model_weight_token (model_weight_token), + KEY index_media_file_token (media_file_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights.sql b/_sql/migrations_squashed/model_weights.sql new file mode 100644 index 000000000..dee5401c7 --- /dev/null +++ b/_sql/migrations_squashed/model_weights.sql @@ -0,0 +1,202 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Every type of fine tuned model that users can upload. +CREATE TABLE model_weights ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- ========== WEIGHTS CLASSIFICATION ========== + + -- The exact type of weights + -- Currently supported values: + -- * 'hifigan_tt2' - hifigan for Tacotron2; TTS vocoder + -- * 'rvc_v2' - RVC (v2); voice conversion + -- * 'sd_1.5' - Stable Diffusion 1.5; image generation + -- * 'sdxl' - Stable Diffusion XL; image generation + -- * 'so_vits_svc' - SVC; voice conversion + -- * 'tt2' - Tacotron2; TTS + weights_type VARCHAR(32) NOT NULL, + + -- The broader category of the weights + -- Currently supported values: + -- * 'image_generation' (SD1.5, SDXL) + -- * 'text_to_speech' (TT2) + -- * 'vocoder' (Hifigan) + -- * 'voice_conversion' (RVCv2, SVC) + weights_category VARCHAR(32) NOT NULL, + + + -- ========== MODEL DESCRIPTION ========== + + -- The "name" of the model, which might be complicated. + title VARCHAR(255) NOT NULL, + + -- TODO(bt, 2023-12-14): Delete the `maybe_thumbnail_token` column later. + -- We're using other columns instead. + maybe_thumbnail_token VARCHAR(32) DEFAULT NULL, + + -- The "avatar" image is a media file of type image that serves as a + -- small avatar or profile picture icon. + maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL, + + -- The "cover" image is a media file of type image that covers the + -- top of the page. + maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL, + + -- The description of the model in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, + + + -- ========== CREATOR DETAILS ========== + + -- The person that created the model. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6. + -- IP address of creator at time of creation. + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this model. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + -- "weights" synthetic ids are incremented over all media files + -- creator_weights_synthetic_id BIGINT(20) DEFAULT NULL, + + -- "weights_type" synthetic ids are incremented over a given weight type + -- creator_weights_type_synthetic_id BIGINT(20) DEFAULT NULL, + + -- The last user to edit the model. This could be the creator or a moderator. + -- Check the future audit logs table for more info. + maybe_last_update_user_token VARCHAR(32) DEFAULT NULL, + + + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, + + -- The filename that was used at upload time (if available) + original_filename VARCHAR(255) DEFAULT NULL, + + -- File characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, + + -- Checksum of the original weights file + -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)] + -- Note that SHA2 is a hash family (SHA-228, SHA-256, SHA-384, SHA-512, SHA-512/224, SHA-512/256, ...) + -- and produces different lengths output depending on the choice of algorithm. + file_checksum_sha2 CHAR(64) NOT NULL, + + -- ========== BUCKET STORAGE ========== + + -- The hash for the bucket directory that contains the original upload + -- as well as any associated other files. + -- The bucket filename for the primary file (not including the directory path) is given by: + -- `[{maybe_private_bucket_prefix}]{private_bucket_hash}[{maybe_private_bucket_extension}]` + private_bucket_hash VARCHAR(32) NOT NULL, + + -- An optional prefix on the bucket filename. + -- If present, this will be prepended to the beginning of the bucket filename to access the file. + -- The bucket filename for the primary file (not including the directory path) is given by: + -- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]` + maybe_private_bucket_prefix VARCHAR(16) DEFAULT NULL, + + -- An optional appended extension on the bucket filename. + -- If present, this will be appended to the end of the bucket filename to access the file. + -- To allow for flexibility, this extension typically will contain a leading period if + -- the file needs it (eg ".mp4" rather than "mp4")! + -- The bucket filename for the primary file (not including the directory path) is given by: + -- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]` + maybe_private_bucket_extension VARCHAR(16) DEFAULT NULL, + + + -- ========== MODERATION DETAILS ========== + + -- Mods may have to approve of vc models for them to show up in a public index. + -- Models can be used by the author (or other parties that know the URL) before + -- they're approved, but unapproved models won't show up in public indices. + -- This is independent of other visibility controls (is_mod_disabled, deleted_at, + -- and creator_set_visibility) + -- is_public_listing_approved BOOLEAN DEFAULT NULL, + + -- Extremely popular models may be locked from deletion or modification by users. + -- is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + + + -- ========== RATING AND RANKING ========== + + -- User ratings are cached on the model, but are *not* a source of truth. + -- Total count only includes "positive" and "negative" votes, not neutral ones. + cached_user_ratings_total_count INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- Total positive + cached_user_ratings_positive_count INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- Total negative + cached_user_ratings_negative_count INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- User ratings are cached on the model, but are *not* a source of truth. + -- Total count only includes "positive" and "negative" votes, not neutral ones. + maybe_cached_user_ratings_ratio FLOAT, + + -- When the cached ratings were last calculated and updated. + cached_user_ratings_last_updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + + -- ========== MIGRATION DETAILS ========== + + -- If migrated from another table, this is the token of the original model. + maybe_migration_old_model_token VARCHAR(32) DEFAULT NULL, + + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token) + + -- TODO: + -- KEY index_model_type (model_type), + -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- KEY fk_creator_user_token (creator_user_token), + -- KEY index_creator_set_visibility (creator_set_visibility), + -- KEY index_private_bucket_hash (private_bucket_hash), + -- KEY index_is_public_listing_approved (is_public_listing_approved), + -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/tts_tables/tts_models.sql b/_sql/migrations_squashed/tts_tables/tts_models.sql index 493d2c3cd..ca6c71e56 100644 --- a/_sql/migrations_squashed/tts_tables/tts_models.sql +++ b/_sql/migrations_squashed/tts_tables/tts_models.sql @@ -200,6 +200,11 @@ CREATE TABLE tts_models ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== MIGRATION DETAILS ========== + + -- If migrated to model_weights, this is the new token + maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL, + -- ========== VECTOR CLOCK ========== -- Incremented with every update. diff --git a/_sql/migrations_squashed/tts_tables/vocoder_models.sql b/_sql/migrations_squashed/tts_tables/vocoder_models.sql index d11793c78..5c4a9c38a 100644 --- a/_sql/migrations_squashed/tts_tables/vocoder_models.sql +++ b/_sql/migrations_squashed/tts_tables/vocoder_models.sql @@ -99,6 +99,11 @@ CREATE TABLE vocoder_models ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== MIGRATION DETAILS ========== + + -- If migrated to model_weights, this is the new token + maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL, + -- ========== VECTOR CLOCK ========== -- Incremented with every update. diff --git a/_sql/migrations_squashed/user_tables/users.sql b/_sql/migrations_squashed/user_tables/users.sql index 43b6805ad..1ea5fca0e 100644 --- a/_sql/migrations_squashed/user_tables/users.sql +++ b/_sql/migrations_squashed/user_tables/users.sql @@ -53,6 +53,14 @@ CREATE TABLE users ( -- ========== DISPLAY / PROFILE ========== + -- The "avatar" image is a media file of type image that serves as a + -- small avatar or profile picture icon. + maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL, + + -- The "cover" image is a media file of type image that covers the + -- top of the page. + maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL, + -- Gravatar image hashes are precomputed. email_gravatar_hash CHAR(32) NOT NULL, diff --git a/_sql/migrations_squashed/voice_conversion_models.sql b/_sql/migrations_squashed/voice_conversion_models.sql index cff4f387a..d28f338ae 100644 --- a/_sql/migrations_squashed/voice_conversion_models.sql +++ b/_sql/migrations_squashed/voice_conversion_models.sql @@ -121,6 +121,11 @@ CREATE TABLE voice_conversion_models ( -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, + -- ========== MIGRATION DETAILS ========== + + -- If migrated to model_weights, this is the new token + maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL, + -- ========== VECTOR CLOCK ========== -- Incremented with every update. From 11e3a4e429e1a016ef94c1a5384e57b986eac03e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 15 Dec 2023 12:47:33 -0500 Subject: [PATCH 38/40] rename `model_weights_samples` -> `model_weights_preview_samples` - doing this before anyone else runs these migrations - also added "squashed" documentation on the model_weights and associated tables - also added (commented) good indices for model_weights --- .../down.sql | 2 +- .../up.sql | 2 +- .../{ => model_weights}/model_weights.sql | 11 ++- ...model_weights_image_generation_details.sql | 50 +++++++++++++ .../model_weights_preview_samples.sql | 47 ++++++++++++ .../model_weights_tts_details.sql | 73 +++++++++++++++++++ .../model_weights_vocoder_details.sql | 50 +++++++++++++ ...model_weights_voice_conversion_details.sql | 62 ++++++++++++++++ 8 files changed, 289 insertions(+), 8 deletions(-) rename _sql/migrations/{2023-12-15-044416_model_weights_samples => 2023-12-15-044416_model_weights_preview_samples}/down.sql (75%) rename _sql/migrations/{2023-12-15-044416_model_weights_samples => 2023-12-15-044416_model_weights_preview_samples}/up.sql (96%) rename _sql/migrations_squashed/{ => model_weights}/model_weights.sql (95%) create mode 100644 _sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql create mode 100644 _sql/migrations_squashed/model_weights/model_weights_preview_samples.sql create mode 100644 _sql/migrations_squashed/model_weights/model_weights_tts_details.sql create mode 100644 _sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql create mode 100644 _sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql diff --git a/_sql/migrations/2023-12-15-044416_model_weights_samples/down.sql b/_sql/migrations/2023-12-15-044416_model_weights_preview_samples/down.sql similarity index 75% rename from _sql/migrations/2023-12-15-044416_model_weights_samples/down.sql rename to _sql/migrations/2023-12-15-044416_model_weights_preview_samples/down.sql index 9834047d6..a28ccc776 100644 --- a/_sql/migrations/2023-12-15-044416_model_weights_samples/down.sql +++ b/_sql/migrations/2023-12-15-044416_model_weights_preview_samples/down.sql @@ -2,4 +2,4 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile -DROP TABLE model_weights_samples; +DROP TABLE model_weights_preview_samples; diff --git a/_sql/migrations/2023-12-15-044416_model_weights_samples/up.sql b/_sql/migrations/2023-12-15-044416_model_weights_preview_samples/up.sql similarity index 96% rename from _sql/migrations/2023-12-15-044416_model_weights_samples/up.sql rename to _sql/migrations/2023-12-15-044416_model_weights_preview_samples/up.sql index f8d6752a8..a4f5276ca 100644 --- a/_sql/migrations/2023-12-15-044416_model_weights_samples/up.sql +++ b/_sql/migrations/2023-12-15-044416_model_weights_preview_samples/up.sql @@ -2,7 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile -CREATE TABLE model_weights_samples ( +CREATE TABLE model_weights_preview_samples ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/_sql/migrations_squashed/model_weights.sql b/_sql/migrations_squashed/model_weights/model_weights.sql similarity index 95% rename from _sql/migrations_squashed/model_weights.sql rename to _sql/migrations_squashed/model_weights/model_weights.sql index dee5401c7..f9e0bf483 100644 --- a/_sql/migrations_squashed/model_weights.sql +++ b/_sql/migrations_squashed/model_weights/model_weights.sql @@ -190,13 +190,12 @@ CREATE TABLE model_weights ( PRIMARY KEY (id), UNIQUE KEY (token) - -- TODO: - -- KEY index_model_type (model_type), - -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- TODO (this set should be good to go): + -- KEY index_weights_type (weights_type), + -- KEY index_weights_category (weights_category), -- KEY fk_creator_user_token (creator_user_token), -- KEY index_creator_set_visibility (creator_set_visibility), - -- KEY index_private_bucket_hash (private_bucket_hash), - -- KEY index_is_public_listing_approved (is_public_listing_approved), - -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + -- KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token), + -- KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql b/_sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql new file mode 100644 index 000000000..26b3c9b7b --- /dev/null +++ b/_sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql @@ -0,0 +1,50 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Every type of fine tuned model that users can upload. +CREATE TABLE model_weights_image_generation_details ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Simultaneous primary and foreign key. + -- This should match a single record in the `model_weights` table. + -- This turns this table into a polymorphic extension table. + model_weights_token VARCHAR(32) NOT NULL, + + + -- ========== IMAGE GENERATION MODEL METADATA ========== + + -- RVC (v2) specific - whether the model has an index file associated with it + -- These files improve the quality of the results. + vc_has_index_file BOOLEAN NOT NULL DEFAULT FALSE, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weights_token) + -- TODO: + -- KEY index_model_type (model_type), + -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- KEY fk_creator_user_token (creator_user_token), + -- KEY index_creator_set_visibility (creator_set_visibility), + -- KEY index_private_bucket_hash (private_bucket_hash), + -- KEY index_is_public_listing_approved (is_public_listing_approved), + -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights/model_weights_preview_samples.sql b/_sql/migrations_squashed/model_weights/model_weights_preview_samples.sql new file mode 100644 index 000000000..a4f5276ca --- /dev/null +++ b/_sql/migrations_squashed/model_weights/model_weights_preview_samples.sql @@ -0,0 +1,47 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE model_weights_preview_samples ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- ========== WEIGHTS CLASSIFICATION ========== + + -- Foreign key to model weights. + -- This is in a composite index with media_file_token to make sure a media file only applies + -- to a given model once. + model_weight_token VARCHAR(32) NOT NULL, + + -- Foreign key to media files + -- This is in a composite index with model_weight_token to make sure a media file only applies + -- to a given model once. + media_file_token VARCHAR(32) NOT NULL, + + -- Information about who set the sample + -- * automation + -- * author + -- * community + set_by VARCHAR(16) NOT NULL, + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weight_token, media_file_token), + + KEY index_model_weight_token (model_weight_token), + KEY index_media_file_token (media_file_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights/model_weights_tts_details.sql b/_sql/migrations_squashed/model_weights/model_weights_tts_details.sql new file mode 100644 index 000000000..540e53453 --- /dev/null +++ b/_sql/migrations_squashed/model_weights/model_weights_tts_details.sql @@ -0,0 +1,73 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Extra metadata on tts models +CREATE TABLE model_weights_tts_details ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Simultaneous primary and foreign key. + -- This should match a single record in the `model_weights` table. + -- This turns this table into a polymorphic extension table. + model_weights_token VARCHAR(32) NOT NULL, + + -- ========== TTS MODEL METADATA ========== + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- (Not that it matters apart from categorization, since voice conversion is universal.) + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") + -- (Not that it matters apart from categorization, since voice conversion is universal.) + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- This is for Tacotron2, specifically. + -- The name of the type of pipeline to use, eg "legacy_fakeyou", "legacy_vocodes", "english_v1", etc. + -- New pipelines may be added at any point in the future and the source of truth for them will mostly + -- live in the `storyteller-ml` repository. + -- Older models and brand new models will have this field set to null, and we'll infer which value to + -- lazily backfill based on the creation date and language setting. + -- tts_text_pipeline_type VARCHAR(64) DEFAULT NULL, + + -- If true, multiply the mel outputs before being vocoded by a globally default constant. + use_default_mel_multiply_factor BOOLEAN NOT NULL DEFAULT FALSE, + + -- If not null, multiply the mel outputs before being vocoded by this value. + -- This is used instead of the default if `tts_use_default_mel_multiply_factor` is + -- set (ie. `tts_use_default_mel_multiply_factor` is ignored and this custom value is used instead) + maybe_custom_mel_multiply_factor DOUBLE DEFAULT NULL, + + -- If set, use a different pretrained vocoder. + -- If not set, use the website default. (Currently HifiGan-SS) + maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL, + + -- If set, we'll use a custom vocoder (from table vocoder_models) instead of a default vocoder. + -- If not set, we'll use `maybe_default_pretrained_vocoder` (or the website default). + maybe_custom_vocoder_token VARCHAR(32) DEFAULT NULL, + + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weights_token) + -- TODO: + -- KEY index_model_type (model_type), + -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- KEY fk_creator_user_token (creator_user_token), + -- KEY index_creator_set_visibility (creator_set_visibility), + -- KEY index_private_bucket_hash (private_bucket_hash), + -- KEY index_is_public_listing_approved (is_public_listing_approved), + -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql b/_sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql new file mode 100644 index 000000000..f04ac7f34 --- /dev/null +++ b/_sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql @@ -0,0 +1,50 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Every type of fine tuned model that users can upload. +CREATE TABLE model_weights_vocoder_details ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Simultaneous primary and foreign key. + -- This should match a single record in the `model_weights` table. + -- This turns this table into a polymorphic extension table. + model_weights_token VARCHAR(32) NOT NULL, + + + -- ========== VOCODER MODEL METADATA ========== + + -- Whether the vocoder is recommended for use + -- vocoder_is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE, + + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weights_token) + -- TODO: + -- KEY index_model_type (model_type), + -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- KEY fk_creator_user_token (creator_user_token), + -- KEY index_creator_set_visibility (creator_set_visibility), + -- KEY index_private_bucket_hash (private_bucket_hash), + -- KEY index_is_public_listing_approved (is_public_listing_approved), + -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql b/_sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql new file mode 100644 index 000000000..8dd5e0c9a --- /dev/null +++ b/_sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql @@ -0,0 +1,62 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Every type of fine tuned model that users can upload. +CREATE TABLE model_weights_voice_conversion_details ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Simultaneous primary and foreign key. + -- This should match a single record in the `model_weights` table. + -- This turns this table into a polymorphic extension table. + model_weights_token VARCHAR(32) NOT NULL, + + + -- ========== VOICE CONVERSION MODEL METADATA ========== + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- (Not that it matters apart from categorization, since voice conversion is universal.) + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") + -- (Not that it matters apart from categorization, since voice conversion is universal.) + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- RVC (v2) specific - whether the model has an index file associated with it + -- These files improve the quality of the results. + has_index_file BOOLEAN NOT NULL DEFAULT FALSE, + + -- Pitch algorithm to use by default + default_pitch_algorithm VARCHAR(64) DEFAULT NULL, + + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_weights_token) + -- TODO: + -- KEY index_model_type (model_type), + -- KEY fk_maybe_vocoder_token (maybe_vocoder_token), + -- KEY fk_creator_user_token (creator_user_token), + -- KEY index_creator_set_visibility (creator_set_visibility), + -- KEY index_private_bucket_hash (private_bucket_hash), + -- KEY index_is_public_listing_approved (is_public_listing_approved), + -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 98fa0a67ea11b27bc1a841e3306afd6ab0b1c1c8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 15 Dec 2023 13:33:14 -0500 Subject: [PATCH 39/40] rename extension tables --- .../down.sql | 9 +++++++++ .../up.sql | 9 +++++++++ ...model_weights_extension_image_generation_details.sql} | 2 +- ...tails.sql => model_weights_extension_tts_details.sql} | 2 +- ...s.sql => model_weights_extension_vocoder_details.sql} | 2 +- ...model_weights_extension_voice_conversion_details.sql} | 2 +- 6 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 _sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/down.sql create mode 100644 _sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/up.sql rename _sql/migrations_squashed/model_weights/{model_weights_image_generation_details.sql => model_weights_extension_image_generation_details.sql} (96%) rename _sql/migrations_squashed/model_weights/{model_weights_tts_details.sql => model_weights_extension_tts_details.sql} (98%) rename _sql/migrations_squashed/model_weights/{model_weights_vocoder_details.sql => model_weights_extension_vocoder_details.sql} (96%) rename _sql/migrations_squashed/model_weights/{model_weights_voice_conversion_details.sql => model_weights_extension_voice_conversion_details.sql} (97%) diff --git a/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/down.sql b/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/down.sql new file mode 100644 index 000000000..842ea3a25 --- /dev/null +++ b/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/down.sql @@ -0,0 +1,9 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- I'm doing this to visually align these tables and sort them together, because there will likely be other model_weights_* tables. +RENAME TABLE model_weights_extension_image_generation_details TO model_weights_image_generation_details; +RENAME TABLE model_weights_extension_tts_details TO model_weights_tts_details; +RENAME TABLE model_weights_extension_vocoder_details TO model_weights_vocoder_details; +RENAME TABLE model_weights_extension_voice_conversion_details TO model_weights_voice_conversion_details; diff --git a/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/up.sql b/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/up.sql new file mode 100644 index 000000000..8a9490ffc --- /dev/null +++ b/_sql/migrations/2023-12-15-182157_rename_model_weights_extension_tables/up.sql @@ -0,0 +1,9 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- I'm doing this to visually align these tables and sort them together, because there will likely be other model_weights_* tables. +RENAME TABLE model_weights_image_generation_details TO model_weights_extension_image_generation_details; +RENAME TABLE model_weights_tts_details TO model_weights_extension_tts_details; +RENAME TABLE model_weights_vocoder_details TO model_weights_extension_vocoder_details; +RENAME TABLE model_weights_voice_conversion_details TO model_weights_extension_voice_conversion_details; diff --git a/_sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql b/_sql/migrations_squashed/model_weights/model_weights_extension_image_generation_details.sql similarity index 96% rename from _sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql rename to _sql/migrations_squashed/model_weights/model_weights_extension_image_generation_details.sql index 26b3c9b7b..44d1ee408 100644 --- a/_sql/migrations_squashed/model_weights/model_weights_image_generation_details.sql +++ b/_sql/migrations_squashed/model_weights/model_weights_extension_image_generation_details.sql @@ -3,7 +3,7 @@ -- noinspection SqlResolveForFile -- Every type of fine tuned model that users can upload. -CREATE TABLE model_weights_image_generation_details ( +CREATE TABLE model_weights_extension_image_generation_details ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/_sql/migrations_squashed/model_weights/model_weights_tts_details.sql b/_sql/migrations_squashed/model_weights/model_weights_extension_tts_details.sql similarity index 98% rename from _sql/migrations_squashed/model_weights/model_weights_tts_details.sql rename to _sql/migrations_squashed/model_weights/model_weights_extension_tts_details.sql index 540e53453..01478a453 100644 --- a/_sql/migrations_squashed/model_weights/model_weights_tts_details.sql +++ b/_sql/migrations_squashed/model_weights/model_weights_extension_tts_details.sql @@ -3,7 +3,7 @@ -- noinspection SqlResolveForFile -- Extra metadata on tts models -CREATE TABLE model_weights_tts_details ( +CREATE TABLE model_weights_extension_tts_details ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/_sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql b/_sql/migrations_squashed/model_weights/model_weights_extension_vocoder_details.sql similarity index 96% rename from _sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql rename to _sql/migrations_squashed/model_weights/model_weights_extension_vocoder_details.sql index f04ac7f34..c3e4625f4 100644 --- a/_sql/migrations_squashed/model_weights/model_weights_vocoder_details.sql +++ b/_sql/migrations_squashed/model_weights/model_weights_extension_vocoder_details.sql @@ -3,7 +3,7 @@ -- noinspection SqlResolveForFile -- Every type of fine tuned model that users can upload. -CREATE TABLE model_weights_vocoder_details ( +CREATE TABLE model_weights_extension_vocoder_details ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/_sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql b/_sql/migrations_squashed/model_weights/model_weights_extension_voice_conversion_details.sql similarity index 97% rename from _sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql rename to _sql/migrations_squashed/model_weights/model_weights_extension_voice_conversion_details.sql index 8dd5e0c9a..e32c19ca5 100644 --- a/_sql/migrations_squashed/model_weights/model_weights_voice_conversion_details.sql +++ b/_sql/migrations_squashed/model_weights/model_weights_extension_voice_conversion_details.sql @@ -3,7 +3,7 @@ -- noinspection SqlResolveForFile -- Every type of fine tuned model that users can upload. -CREATE TABLE model_weights_voice_conversion_details ( +CREATE TABLE model_weights_extension_voice_conversion_details ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, From a6806d36166d737717f7f6bca41697d51991789e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 15 Dec 2023 13:57:22 -0500 Subject: [PATCH 40/40] indices on model_weights and users (there weren't any on model_weights!) --- .../down.sql | 10 ++++++++++ .../up.sql | 10 ++++++++++ .../down.sql | 6 ++++++ .../up.sql | 6 ++++++ .../model_weights/model_weights.sql | 15 +++++++-------- _sql/migrations_squashed/user_tables/users.sql | 4 +++- 6 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 _sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/down.sql create mode 100644 _sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/up.sql create mode 100644 _sql/migrations/2023-12-15-185147_alter_users_add_indices/down.sql create mode 100644 _sql/migrations/2023-12-15-185147_alter_users_add_indices/up.sql diff --git a/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/down.sql b/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/down.sql new file mode 100644 index 000000000..6f900e0a9 --- /dev/null +++ b/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/down.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP INDEX index_weights_type ON model_weights; +DROP INDEX index_weights_category ON model_weights; +DROP INDEX fk_creator_user_token ON model_weights; +DROP INDEX index_creator_set_visibility ON model_weights; +DROP INDEX fk_maybe_avatar_media_file_token ON model_weights; +DROP INDEX fk_maybe_cover_media_file_token ON model_weights; diff --git a/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/up.sql b/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/up.sql new file mode 100644 index 000000000..125b17588 --- /dev/null +++ b/_sql/migrations/2023-12-15-184731_alter_model_weights_add_indices/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE INDEX index_weights_type ON model_weights (weights_type); +CREATE INDEX index_weights_category ON model_weights (weights_category); +CREATE INDEX fk_creator_user_token ON model_weights (creator_user_token); +CREATE INDEX index_creator_set_visibility ON model_weights (creator_set_visibility); +CREATE INDEX fk_maybe_avatar_media_file_token ON model_weights (maybe_avatar_media_file_token); +CREATE INDEX fk_maybe_cover_media_file_token ON model_weights (maybe_cover_media_file_token); diff --git a/_sql/migrations/2023-12-15-185147_alter_users_add_indices/down.sql b/_sql/migrations/2023-12-15-185147_alter_users_add_indices/down.sql new file mode 100644 index 000000000..ce09b6680 --- /dev/null +++ b/_sql/migrations/2023-12-15-185147_alter_users_add_indices/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP INDEX fk_maybe_avatar_media_file_token ON users; +DROP INDEX fk_maybe_cover_media_file_token ON users; diff --git a/_sql/migrations/2023-12-15-185147_alter_users_add_indices/up.sql b/_sql/migrations/2023-12-15-185147_alter_users_add_indices/up.sql new file mode 100644 index 000000000..12565c806 --- /dev/null +++ b/_sql/migrations/2023-12-15-185147_alter_users_add_indices/up.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE INDEX fk_maybe_avatar_media_file_token ON users (maybe_avatar_media_file_token); +CREATE INDEX fk_maybe_cover_media_file_token ON users (maybe_cover_media_file_token); diff --git a/_sql/migrations_squashed/model_weights/model_weights.sql b/_sql/migrations_squashed/model_weights/model_weights.sql index f9e0bf483..f17a5e106 100644 --- a/_sql/migrations_squashed/model_weights/model_weights.sql +++ b/_sql/migrations_squashed/model_weights/model_weights.sql @@ -188,14 +188,13 @@ CREATE TABLE model_weights ( -- INDICES -- PRIMARY KEY (id), - UNIQUE KEY (token) + UNIQUE KEY (token), - -- TODO (this set should be good to go): - -- KEY index_weights_type (weights_type), - -- KEY index_weights_category (weights_category), - -- KEY fk_creator_user_token (creator_user_token), - -- KEY index_creator_set_visibility (creator_set_visibility), - -- KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token), - -- KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token) + KEY index_weights_type (weights_type), + KEY index_weights_category (weights_category), + KEY fk_creator_user_token (creator_user_token), + KEY index_creator_set_visibility (creator_set_visibility), + KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token), + KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/user_tables/users.sql b/_sql/migrations_squashed/user_tables/users.sql index 1ea5fca0e..874b0028b 100644 --- a/_sql/migrations_squashed/user_tables/users.sql +++ b/_sql/migrations_squashed/user_tables/users.sql @@ -170,7 +170,9 @@ CREATE TABLE users ( UNIQUE KEY (token), UNIQUE KEY (username), UNIQUE KEY (email_address), - KEY fk_user_role_slug (user_role_slug) + KEY fk_user_role_slug (user_role_slug), + KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token), + KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;