From b55435d99287ca011f94a731954449069749b037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:30:06 +0000 Subject: [PATCH 01/11] Bump rustix from 0.38.25 to 0.38.26 Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.38.25 to 0.38.26. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Commits](https://github.com/bytecodealliance/rustix/compare/v0.38.25...v0.38.26) --- updated-dependencies: - dependency-name: rustix dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 124 +++++++++++++++++++++++++++-------- my-workspace-hack/Cargo.toml | 8 +-- 2 files changed, 99 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..62ac6ff06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,7 +521,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -531,7 +531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2196,12 +2196,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2220,7 +2220,7 @@ checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ "cfg-if 1.0.0", "home", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2714,7 +2714,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3079,7 +3079,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3600,7 +3600,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3736,7 +3736,7 @@ dependencies = [ "url", "void", "winapi 0.3.9", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4131,7 +4131,7 @@ dependencies = [ "libc", "redox_syscall 0.4.1", "smallvec 1.11.2", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -4841,7 +4841,7 @@ dependencies = [ "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4951,15 +4951,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -5061,7 +5061,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -5514,7 +5514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6231,7 +6231,7 @@ dependencies = [ "fastrand 2.0.1", "redox_syscall 0.4.1", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6405,7 +6405,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.5", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -7419,7 +7419,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -7428,7 +7428,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -7437,13 +7446,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -7452,42 +7476,84 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -7495,7 +7561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if 1.0.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] diff --git a/my-workspace-hack/Cargo.toml b/my-workspace-hack/Cargo.toml index 2f062e5a69..22f41a903d 100644 --- a/my-workspace-hack/Cargo.toml +++ b/my-workspace-hack/Cargo.toml @@ -142,22 +142,22 @@ url = { version = "2.4.1", features = ["serde"] } [target.x86_64-unknown-linux-gnu.dependencies] openssl = { version = "0.10.59", features = ["vendored"] } openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-unknown-linux-gnu.build-dependencies] openssl = { version = "0.10.59", features = ["vendored"] } openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] } -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } [target.x86_64-apple-darwin.dependencies] -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } spin = { version = "0.9.8" } [target.x86_64-apple-darwin.build-dependencies] -rustix = { version = "0.38.25", features = ["fs", "termios"] } +rustix = { version = "0.38.26", features = ["fs", "termios"] } socket2 = { version = "0.5.5", default-features = false, features = ["all"] } spin = { version = "0.9.8" } From ac13be29009ae7ecc390948ee1cec4da2d634c9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:30:26 +0000 Subject: [PATCH 02/11] Bump lettre from 0.11.1 to 0.11.2 Bumps [lettre](https://github.com/lettre/lettre) from 0.11.1 to 0.11.2. - [Release notes](https://github.com/lettre/lettre/releases) - [Changelog](https://github.com/lettre/lettre/blob/master/CHANGELOG.md) - [Commits](https://github.com/lettre/lettre/compare/v0.11.1...v0.11.2) --- updated-dependencies: - dependency-name: lettre dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 18 ++++-------------- crates/lib/email_sender/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..ffe82d3765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2907,16 +2907,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -3215,9 +3205,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "lettre" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a466bc111374ccf4d90877dba636924a2185e67e5be4b35d32043199365097b2" +checksum = "a48c2e9831b370bc2d7233c2620298c45f3a158ed6b4b8d7416b2ada5a268fd8" dependencies = [ "base64 0.21.5", "chumsky", @@ -3227,7 +3217,7 @@ dependencies = [ "futures-util", "hostname", "httpdate", - "idna 0.4.0", + "idna", "mime", "native-tls", "nom", @@ -6935,7 +6925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", "serde", ] diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/email_sender/Cargo.toml index e640833ea8..323b40abd4 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/email_sender/Cargo.toml @@ -11,5 +11,5 @@ path = "src/lib.rs" [dependencies] errors = { path = "../errors" } -lettre = "0.11.0" +lettre = "0.11.2" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } From 2e3ff8a73024e441b90e595fec5992bb0d2edaa3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 4 Dec 2023 20:34:39 -0500 Subject: [PATCH 03/11] list media files for user --- .../media_files/list_media_files_for_user.rs | 203 +++++++++++++++++ .../src/queries/media_files/mod.rs | 11 +- .../media_files/list_media_files_for_user.rs | 215 ++++++++++++++++++ .../http_server/endpoints/media_files/mod.rs | 3 +- .../service/web/storyteller_web/src/routes.rs | 5 + 5 files changed, 431 insertions(+), 6 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs new file mode 100644 index 0000000000..1c7bcdee62 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -0,0 +1,203 @@ +use chrono::{DateTime, Utc}; +use sqlx::{MySql, MySqlPool, QueryBuilder}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; + +pub struct MediaFileListPage { + pub records: Vec, + + pub sort_ascending: bool, + + /// ID of the first record. + pub first_id: Option, + + /// ID of the last record. + pub last_id: Option, +} + +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Clone, Copy)] +pub enum ViewAs { + Author, + Moderator, + AnotherUser, +} + +pub struct ListMediaFileForUserArgs<'a> { + pub username: &'a str, + pub limit: usize, + pub maybe_filter_media_type: Option, + pub maybe_offset: Option, + pub cursor_is_reversed: bool, + pub view_as: ViewAs, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult { + + let mut query = query_builder( + args.maybe_filter_media_type, + args.username, + args.limit, + args.maybe_offset, + args.cursor_is_reversed, + args.view_as, + ); + + let query = query.build_query_as::(); + + let results = query.fetch_all(args.mysql_pool).await?; + + let first_id = results.first() + .map(|raw_result| raw_result.id); + + let last_id = results.last() + .map(|raw_result| raw_result.id); + + let results = results.into_iter() + .map(|record| { + MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + Ok(MediaFileListPage { + records: results, + sort_ascending: !args.cursor_is_reversed, + first_id, + last_id, + }) +} + +fn query_builder<'a>( + maybe_filter_media_type: Option, + username: &'a str, + limit: usize, + maybe_offset: Option, + cursor_is_reversed: bool, + view_as: ViewAs, +) -> QueryBuilder<'a, MySql> { + + // NB: Query cannot be statically checked by sqlx + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +SELECT + m.id, + m.token as `token: tokens::tokens::media_files::MediaFileToken`, + + m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, + m.origin_product_category as `origin_product_category: enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory`, + + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users AS u + ON m.maybe_creator_user_token = u.token + +WHERE u.username = ? +AND m.user_deleted_at IS NULL +AND m.mod_deleted_at IS NULL + "# + ); + + query_builder.push_bind(username); + + if let Some(media_type) = maybe_filter_media_type { + query_builder.push(" AND m.media_file_type = ? "); + query_builder.push_bind(media_type); + } + + match view_as { + ViewAs::Author => {} + ViewAs::Moderator => {} + ViewAs::AnotherUser => { + query_builder.push(" AND m.creator_set_visibility = 'public' "); + } + } + + if let Some(offset) = maybe_offset { + if cursor_is_reversed { + query_builder.push(format!(" AND m.id < {offset} ")); + } else { + query_builder.push(format!(" AND m.id > {offset} ")); + } + } + + if cursor_is_reversed { + query_builder.push(" ORDER BY m.id ASC "); + } else { + query_builder.push(" ORDER BY m.id DESC "); + } + + query_builder.push(format!(" LIMIT {limit} ")); + + query_builder +} + +#[derive(sqlx::FromRow)] +struct MediaFileListItemInternal { + id: i64, + token: MediaFileToken, + + origin_category: MediaFileOriginCategory, + origin_product_category: MediaFileOriginProductCategory, + maybe_origin_model_type: Option, + maybe_origin_model_token: Option, + + media_type: MediaFileType, + public_bucket_directory_hash: String, + maybe_public_bucket_prefix: Option, + maybe_public_bucket_extension: Option, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 3225ced58e..619f018ce5 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -1,7 +1,8 @@ -pub mod get_media_file; -pub mod insert_media_file_from_face_animation; -pub mod insert_media_file_from_zero_shot_tts; -pub mod insert_media_file_from_cli_tool; -pub mod insert_media_file_from_voice_conversion; pub mod delete_media_file; +pub mod get_media_file; +pub mod insert_media_file_from_cli_tool; +pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_voice_conversion; +pub mod insert_media_file_from_zero_shot_tts; +pub mod list_media_files_for_user; pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs new file mode 100644 index 0000000000..35ae3f074c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -0,0 +1,215 @@ +use std::sync::Arc; + +use actix_web::{HttpMessage, HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Query}; +use chrono::{DateTime, Utc}; +use log::{info, warn}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct PathInfo { + username: String, +} + +#[derive(Deserialize)] +pub struct QueryParams { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, + pub filter_media_type: Option, +} + +#[derive(Serialize)] +pub struct SuccessResponse { + pub success: bool, + pub results: Vec, + pub cursor_next: Option, + pub cursor_previous: Option, +} + +#[derive(Serialize)] +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ErrorResponse { + ServerError, +} + +impl ResponseError for ErrorResponse { + fn status_code(&self) -> StatusCode { + match *self { + ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ErrorResponse::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +impl std::fmt::Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_media_files_for_user_handler( + http_request: HttpRequest, + path: Path, + query: Query, + server_state: web::Data> +) -> Result +{ + info!("Fetching inference results for user: {}", &path.username); + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ErrorResponse::ServerError + })?; + + let mut is_author = false; + let mut is_mod = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_author = session.username == path.username; + is_mod = session.can_ban_users; + }, + }; + + // TODO(bt,2023-12-04): Enforce real maximums and defaults + let limit = query.limit.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor as usize) + } else { + None + }; + + let view_as = if is_author { + ViewAs::Author + } else if is_mod { + ViewAs::Moderator + } else { + ViewAs::AnotherUser + }; + + let query_results = list_media_files_for_user(ListMediaFileForUserArgs { + username: &path.username, + limit, + maybe_filter_media_type: query.filter_media_type, + maybe_offset: cursor, + cursor_is_reversed, + view_as, + mysql_pool: &server_state.mysql_pool, + }).await; + + + let results_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ErrorResponse::ServerError); + } + }; + + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let results = results_page.records.into_iter() + .map(|record| MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + }) + .collect::>(); + + let response = SuccessResponse { + success: true, + results, + cursor_next, + cursor_previous, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ErrorResponse::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 014fa68854..6af4273b7a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; -pub mod delete_media_file; +pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..9aafd5340c 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -44,6 +44,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -1006,6 +1007,10 @@ fn add_media_file_routes (app: App) -> App .route(web::delete().to(delete_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/user/{username}") + .route(web::get().to(list_media_files_for_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } From b9f604dd1875dc4ffac0e7920fffdff26f0c5892 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 4 Dec 2023 23:11:41 -0500 Subject: [PATCH 04/11] reusable pagination types --- .../src/http_server/common_responses/mod.rs | 2 + .../common_responses/pagination_cursors.rs | 53 +++++++++++++++++++ .../common_responses/pagination_page.rs | 33 ++++++++++++ .../media_files/list_media_files_for_user.rs | 11 ++-- 4 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs index 1aa45ef781..d84a319a04 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs @@ -1 +1,3 @@ +pub mod pagination_cursors; +pub mod pagination_page; pub mod user_details_lite; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs new file mode 100644 index 0000000000..c801faea59 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_cursors.rs @@ -0,0 +1,53 @@ +/// Pagination by cursors +/// These types of pagination are for "infinite scrolling", which do not reveal a number of pages. +/// This is good so that investors and competitors cannot reveal how many database records we have. +/// This is typically used for "discovery" type pages, not user profiles. +/// +/// See `PaginationPage` for the other type of pagination. +#[derive(Serialize)] +pub struct PaginationCursors { + + /// The "next" cursor. + /// This is an opaque (typically even encrypted) handle. + pub maybe_next: Option, + + /// The "previous" cursor. + /// This is an opaque (typically even encrypted) handle. + pub maybe_previous: Option, + + /// Details whether we're walking forward or backward. + pub cursor_is_reversed: bool, +} + +#[cfg(test)] +mod tests { + use crate::http_server::common_responses::pagination_cursors::PaginationCursors; + + #[test] + fn test_serialize_empty_json() { + // Make sure this interface doesn't change. + let pagination = PaginationCursors { + maybe_next: None, + maybe_previous: None, + cursor_is_reversed: false, + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"maybe_next\":null,\"maybe_previous\":null,\"cursor_is_reversed\":false}") + } + + #[test] + fn test_serialize_full_json() { + // Make sure this interface doesn't change. + let pagination = PaginationCursors { + maybe_next: Some("foo".to_string()), + maybe_previous: Some("bar".to_string()), + cursor_is_reversed: true, + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"maybe_next\":\"foo\",\"maybe_previous\":\"bar\",\"cursor_is_reversed\":true}") + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs new file mode 100644 index 0000000000..3e64af9626 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/pagination_page.rs @@ -0,0 +1,33 @@ +/// Pagination by page id +/// This type of pagination is by "page id" and "page count". +/// This should never be used to walk the entire public database as competitors and investors +/// can use it to glean information about the scale of our service. +/// This is best used for user profiles and scoped down results. +/// +/// See `PaginationCursors` for the other type of pagination. +#[derive(Serialize)] +pub struct PaginationPage { + /// The current page number + pub current: u64, + + /// How many pages there are in total + pub total_page_count: u64, +} + +#[cfg(test)] +mod tests { + use crate::http_server::common_responses::pagination_page::PaginationPage; + + #[test] + fn test_serialize() { + let pagination = PaginationPage { + current: 5, + total_page_count: 123, + + }; + + let json = serde_json::to_string(&pagination).expect("serialization"); + + assert_eq!(json, "{\"current\":5,\"total_page_count\":123}") + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs index 35ae3f074c..9cfe696a27 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -15,6 +15,7 @@ use enums::common::visibility::Visibility; use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs}; use tokens::tokens::media_files::MediaFileToken; +use crate::http_server::common_responses::pagination_cursors::PaginationCursors; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -36,8 +37,7 @@ pub struct QueryParams { pub struct SuccessResponse { pub success: bool, pub results: Vec, - pub cursor_next: Option, - pub cursor_previous: Option, + pub pagination: PaginationCursors, } #[derive(Serialize)] @@ -202,8 +202,11 @@ pub async fn list_media_files_for_user_handler( let response = SuccessResponse { success: true, results, - cursor_next, - cursor_previous, + pagination: PaginationCursors { + maybe_next: cursor_next, + maybe_previous: cursor_previous, + cursor_is_reversed, + } }; let body = serde_json::to_string(&response) From 36f3003576ca86b88c0ab30fd99af3672ed3f8a1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 17:04:11 -0500 Subject: [PATCH 05/11] Fix QueryBuilder with respect to column names and type juggling --- .../media_files/media_file_origin_category.rs | 2 + .../media_file_origin_model_type.rs | 2 + .../media_file_origin_product_category.rs | 1 + .../by_table/media_files/media_file_type.rs | 1 + crates/_types/enums/src/common/visibility.rs | 1 + crates/_types/enums/src/lib.rs | 1 + .../enums/src/macros/impl_mysql_from_row.rs | 63 ++++++++++++++++++ crates/_types/enums/src/macros/mod.rs | 1 + crates/_types/enums/src/traits/mod.rs | 1 + .../_types/enums/src/traits/mysql_from_row.rs | 11 ++++ .../media_files/list_media_files_for_user.rs | 65 +++++++++++++++---- 11 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 crates/_types/enums/src/macros/impl_mysql_from_row.rs create mode 100644 crates/_types/enums/src/traits/mod.rs create mode 100644 crates/_types/enums/src/traits/mysql_from_row.rs diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs index 5db6daa07e..39c87a33e0 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; + use serde::Deserialize; use serde::Serialize; #[cfg(test)] @@ -29,6 +30,7 @@ pub enum MediaFileOriginCategory { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginCategory); impl_mysql_enum_coders!(MediaFileOriginCategory); +impl_mysql_from_row!(MediaFileOriginCategory); /// NB: Legacy API for older code. impl MediaFileOriginCategory { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index 72683e99ae..b38f90a6dc 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -24,6 +24,7 @@ pub enum MediaFileOriginModelType { /// so-vits-svc voice conversion models #[serde(rename = "so_vits_svc")] SoVitsSvc, + #[serde(rename = "vall_e_x")] VallEX, } @@ -31,6 +32,7 @@ pub enum MediaFileOriginModelType { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginModelType); impl_mysql_enum_coders!(MediaFileOriginModelType); +impl_mysql_from_row!(MediaFileOriginModelType); /// NB: Legacy API for older code. impl MediaFileOriginModelType { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 8bc2a4b81a..2d27b3824d 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,6 +39,7 @@ pub enum MediaFileOriginProductCategory { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileOriginProductCategory); impl_mysql_enum_coders!(MediaFileOriginProductCategory); +impl_mysql_from_row!(MediaFileOriginProductCategory); /// NB: Legacy API for older code. impl MediaFileOriginProductCategory { diff --git a/crates/_types/enums/src/by_table/media_files/media_file_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_type.rs index 15bb135098..08fe71f572 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_type.rs @@ -25,6 +25,7 @@ pub enum MediaFileType { // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaFileType); impl_mysql_enum_coders!(MediaFileType); +impl_mysql_from_row!(MediaFileType); /// NB: Legacy API for older code. impl MediaFileType { diff --git a/crates/_types/enums/src/common/visibility.rs b/crates/_types/enums/src/common/visibility.rs index e5a8494005..f7240ca332 100644 --- a/crates/_types/enums/src/common/visibility.rs +++ b/crates/_types/enums/src/common/visibility.rs @@ -38,6 +38,7 @@ pub enum Visibility { impl_enum_display_and_debug_using_to_str!(Visibility); +impl_mysql_from_row!(Visibility); // For reference, here's what the serde implementation might be if manually written. // This may be useful for designing composite types in the future: diff --git a/crates/_types/enums/src/lib.rs b/crates/_types/enums/src/lib.rs index 8a725a12ed..e558ab9d23 100644 --- a/crates/_types/enums/src/lib.rs +++ b/crates/_types/enums/src/lib.rs @@ -32,3 +32,4 @@ mod macros; pub mod by_table; pub mod common; +pub mod traits; diff --git a/crates/_types/enums/src/macros/impl_mysql_from_row.rs b/crates/_types/enums/src/macros/impl_mysql_from_row.rs new file mode 100644 index 0000000000..91380fbf65 --- /dev/null +++ b/crates/_types/enums/src/macros/impl_mysql_from_row.rs @@ -0,0 +1,63 @@ + +/// Implement `MySqlFromRow` on a type. +macro_rules! impl_mysql_from_row { + ($t:ident) => { + + // Try to convert a MySQL row and named field into the value type (for non-nullable fields). + impl crate::traits::mysql_from_row::MySqlFromRow<$t> for $t { + fn try_from_mysql_row(row: &sqlx_mysql::MySqlRow, field_name: &str) -> Result<$t, sqlx::Error> { + use sqlx::{Error, Row}; + + // NB(bt,2023-12-05): For now only string encodings are considered. + // We may want to revisit in the future if we deal with binary data. + let value : String = row.try_get(field_name)?; + + let output_type = Self::from_str(&value) + .map_err(|err| { + Error::ColumnDecode { + index: format!("mysql_from_row failure on field {}: {:?}", field_name, err), + source: format!("mysql_from_row failure on field {}: {:?}", field_name, err).into(), + } + })?; + + Ok(output_type) + } + + // Try to convert a MySQL row and named field into the value type (for nullable fields). + fn try_from_mysql_row_nullable(row: &sqlx_mysql::MySqlRow, field_name: &str) -> Result, sqlx::Error> { + use sqlx::{Error, Row}; + + // NB(bt,2023-12-05): For now only string encodings are considered. + // We may want to revisit in the future if we deal with binary data. + // NB(2): Nullable fields decode as Option. + let maybe_value : Option = row.try_get(field_name)?; + + let value = match maybe_value { + Some(value) => value, + None => return Ok(None), + }; + + let output_type = Self::from_str(&value) + .map_err(|err| { + Error::ColumnDecode { + index: format!("mysql_from_row failure on field {}: {:?}", field_name, err), + source: format!("mysql_from_row failure on field {}: {:?}", field_name, err).into(), + } + })?; + + Ok(Some(output_type)) + } + } + + //// TODO: Tests. + //#[cfg(test)] + //#[test] + //fn test_name() { + // use strum::IntoEnumIterator; + // for variant in $t::iter() { + // assert_eq!(format!("{}", variant), variant.to_str()); + // } + //} + + } +} diff --git a/crates/_types/enums/src/macros/mod.rs b/crates/_types/enums/src/macros/mod.rs index 2f678901d2..93cba5b4dc 100644 --- a/crates/_types/enums/src/macros/mod.rs +++ b/crates/_types/enums/src/macros/mod.rs @@ -1,3 +1,4 @@ #[macro_use] pub mod impl_enum_display_and_debug_using_to_str; #[macro_use] pub mod impl_mysql_enum_coders; +#[macro_use] pub mod impl_mysql_from_row; #[macro_use] pub mod impl_sqlite_enum_coders; diff --git a/crates/_types/enums/src/traits/mod.rs b/crates/_types/enums/src/traits/mod.rs new file mode 100644 index 0000000000..ceb15f8868 --- /dev/null +++ b/crates/_types/enums/src/traits/mod.rs @@ -0,0 +1 @@ +pub mod mysql_from_row; \ No newline at end of file diff --git a/crates/_types/enums/src/traits/mysql_from_row.rs b/crates/_types/enums/src/traits/mysql_from_row.rs new file mode 100644 index 0000000000..7d3c3cbb67 --- /dev/null +++ b/crates/_types/enums/src/traits/mysql_from_row.rs @@ -0,0 +1,11 @@ +use sqlx_mysql::MySqlRow; + +/// This is a trait (in conjunction with the macro implementation) that makes up for sqlx's +/// QueryBuilder not playing well with FromRow implementation or derivation of named types. +pub trait MySqlFromRow { + + /// Attempt to parse a named column from a MySqlRow into the type. + fn try_from_mysql_row(row: &MySqlRow, field_name: &str) -> Result; + + fn try_from_mysql_row_nullable(row: &MySqlRow, field_name: &str) -> Result, sqlx::Error>; +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index 1c7bcdee62..1f9480ee93 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -1,11 +1,13 @@ use chrono::{DateTime, Utc}; -use sqlx::{MySql, MySqlPool, QueryBuilder}; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; +use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; use tokens::tokens::media_files::MediaFileToken; @@ -119,21 +121,21 @@ fn query_builder<'a>( r#" SELECT m.id, - m.token as `token: tokens::tokens::media_files::MediaFileToken`, + m.token, - m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, - m.origin_product_category as `origin_product_category: enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory`, + m.origin_category, + m.origin_product_category, m.maybe_origin_model_type, m.maybe_origin_model_token, - m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + m.media_type, m.public_bucket_directory_hash, m.maybe_public_bucket_prefix, m.maybe_public_bucket_extension, - m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + m.creator_set_visibility, m.created_at, m.updated_at @@ -141,24 +143,30 @@ FROM media_files AS m LEFT OUTER JOIN users AS u ON m.maybe_creator_user_token = u.token -WHERE u.username = ? -AND m.user_deleted_at IS NULL -AND m.mod_deleted_at IS NULL +WHERE m.user_deleted_at IS NULL + AND m.mod_deleted_at IS NULL "# ); + query_builder.push(" AND u.username = "); query_builder.push_bind(username); if let Some(media_type) = maybe_filter_media_type { - query_builder.push(" AND m.media_file_type = ? "); - query_builder.push_bind(media_type); + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.media_file_type = "); } match view_as { ViewAs::Author => {} ViewAs::Moderator => {} ViewAs::AnotherUser => { - query_builder.push(" AND m.creator_set_visibility = 'public' "); + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.creator_set_visibility = "); + query_builder.push_bind(Visibility::Public.to_str()); } } @@ -181,7 +189,6 @@ AND m.mod_deleted_at IS NULL query_builder } -#[derive(sqlx::FromRow)] struct MediaFileListItemInternal { id: i64, token: MediaFileToken, @@ -201,3 +208,35 @@ struct MediaFileListItemInternal { created_at: DateTime, updated_at: DateTime, } + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for MediaFileListItemInternal { + fn from_row(row: &MySqlRow) -> Result { + Ok(Self { + id: row.try_get("id")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, + creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) + } +} From e986a44e9daf942be923e2ef1276cd552114bd0d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:00:44 -0500 Subject: [PATCH 06/11] documentation system open api spec. --- Cargo.lock | 114 +++++++++++++ crates/_types/tokens/Cargo.toml | 3 + .../src/tokens/generic_inference_jobs.rs | 3 +- crates/service/web/storyteller_web/Cargo.toml | 6 + .../web/storyteller_web/src/api_doc.rs | 11 ++ .../web/storyteller_web/src/api_doc_server.rs | 151 ++++++++++++++++++ .../inference/enqueue_tts_request.rs | 23 ++- .../service/web/storyteller_web/src/routes.rs | 4 + 8 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/api_doc.rs create mode 100644 crates/service/web/storyteller_web/src/api_doc_server.rs diff --git a/Cargo.lock b/Cargo.lock index ae154198df..561b3a18b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,6 +2964,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", + "serde", ] [[package]] @@ -4393,6 +4394,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -4876,6 +4901,41 @@ dependencies = [ "quick-xml 0.30.0", ] +[[package]] +name = "rust-embed" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn 2.0.39", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada" +dependencies = [ + "sha2 0.10.8", + "walkdir", +] + [[package]] name = "rust-ini" version = "0.18.0" @@ -5411,6 +5471,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5894,6 +5963,8 @@ dependencies = [ "url_config", "user_input_common", "users_component", + "utoipa", + "utoipa-swagger-ui", "uuid 0.8.2", ] @@ -6387,6 +6458,8 @@ dependencies = [ "sqlx-core", "strum", "toml", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -7025,6 +7098,47 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utoipa" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "regex", + "syn 2.0.39", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" +dependencies = [ + "actix-web", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "utoipa", + "zip", +] + [[package]] name = "uuid" version = "0.8.2" diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..f6276b98ab 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -28,6 +28,9 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } + [dev-dependencies] # Enum utilities diff --git a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs index 1c455e457e..48c3a78260 100644 --- a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs +++ b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs @@ -4,9 +4,10 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for "generic" inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct InferenceJobToken(String); diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..8cedfffde2 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -13,6 +13,10 @@ publish = false name = "storyteller-web" path = "src/main.rs" +[[bin]] +name = "docs" +path = "src/api_doc_server.rs" + [dependencies] # Internal @@ -160,6 +164,8 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs new file mode 100644 index 0000000000..f76feb1e76 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -0,0 +1,11 @@ + +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; + +use utoipa::OpenApi; +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), + components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] +pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs new file mode 100644 index 0000000000..b9b4213409 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -0,0 +1,151 @@ +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +//#![forbid(warnings)] +#![allow(unreachable_patterns)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use utoipa_swagger_ui::SwaggerUi; + +use std::sync::Arc; +use std::time::Duration; + +use actix_cors::Cors; +use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::middleware::{DefaultHeaders, Logger}; +use anyhow::anyhow; +use elasticsearch::Elasticsearch; +use elasticsearch::http::transport::Transport; +use futures::Future; +use limitation::Limiter; +use log::{error, info}; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use r2d2_redis::RedisConnectionManager; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; +use tokio::runtime::Runtime; + + + +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; +use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; +use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; +use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; +use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; +use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; +use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; +use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; +use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; +use cloud_storage::bucket_client::BucketClient; +use config::common_env::CommonEnv; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; +use email_sender::smtp_email_sender::SmtpEmailSender; +use errors::AnyhowResult; +use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use mysql_queries::mediators::badge_granter::BadgeGranter; +use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use redis_caching::redis_ttl_cache::RedisTtlCache; +use reusable_types::server_environment::ServerEnvironment; +use twitch_common::twitch_secrets::TwitchSecrets; +use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; +use users_component::cookies::session::session_cookie_manager::SessionCookieManager; +use users_component::utils::session_checker::SessionChecker; + +use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; +use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; +use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; +use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::routes::add_routes; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; +use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; +use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; + +pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); + +pub mod billing; +pub mod configs; +pub mod cookies; +pub mod http_server; +pub mod memory_cache; +pub mod model; +pub mod routes; +pub mod server_state; +pub mod subscriptions; +pub mod threads; +pub mod user_avatars; +pub mod util; +pub mod validations; + +const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &str = "SECRET_KEY"; +const ENV_REGION_NAME : &str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; + +use std::{ + error::Error, + future::{self, Ready}, + net::Ipv4Addr, +}; + +pub mod api_doc; +use api_doc::ApiDoc; + +use futures::future::LocalBoxFuture; + +use utoipa::OpenApi; +#[actix_web::main] +async fn main() -> Result<(), impl Error> { + HttpServer::new(move || { + App::new() + .service( + SwaggerUi::new("/{_:.*}") + .url("/api-docs/openapi.json", ApiDoc::openapi()), + ) + }) + .bind(("localhost", 8989)).unwrap() + .run().await +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index d394edabcd..32ae10d58d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,8 +1,8 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] +use utoipa::ToSchema; -// TODO MOVE into own file. use std::fmt::Debug; use std::sync::Arc; @@ -42,7 +42,9 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; /// This is useful for catching the live logs or intercepting the job. const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; + #[derive(Deserialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, @@ -50,12 +52,14 @@ pub struct EnqueueTTSRequest { } #[derive(Serialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] +#[derive(ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, @@ -95,13 +99,26 @@ impl std::fmt::Display for EnqueueTTSRequestError { // Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions // insert generic inference job.rs // Need to convert it to generic inference job. +#[utoipa::path( + post, + path = "/inference/enqueue_tts/", + responses( + (status = 200, description = "Enqueue TTS generically", body = EnqueueTTSRequestSuccessResponse), + (status = 400, description = "Bad input", body = EnqueueTTSRequestError), + (status = 401, description = "Not authorized", body = EnqueueTTSRequestError), + (status = 429, description = "Rate limited", body = EnqueueTTSRequestError), + (status = 500, description = "Server error", body = EnqueueTTSRequestError) + ), + params( + ("request" = EnqueueTTSRequest, description = "Payload for TTS Request") + ) +)] pub async fn enqueue_tts_request( http_request: HttpRequest, request: web::Json, server_state: web::Data> ) -> Result { - println!("Received payload"); - + let mut maybe_user_token: Option = None; let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..d84410d643 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,6 +161,10 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { + use utoipa::OpenApi; + + + let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ From e32f8094e5edbe2d1c23f35bfa5e30ed2b36d7d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 01:29:52 -0500 Subject: [PATCH 07/11] global list of media files --- .../queries/media_files/list_media_files.rs | 262 ++++++++++++++++++ .../media_files/list_media_files_for_user.rs | 1 + .../src/queries/media_files/mod.rs | 1 + .../endpoints/media_files/list_media_files.rs | 214 ++++++++++++++ .../media_files/list_media_files_for_user.rs | 2 - .../http_server/endpoints/media_files/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 7 +- 7 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs new file mode 100644 index 0000000000..15571f0563 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files.rs @@ -0,0 +1,262 @@ +use chrono::{DateTime, Utc}; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; +use sqlx::mysql::MySqlRow; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use enums::traits::mysql_from_row::MySqlFromRow; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +pub struct MediaFileListPage { + pub records: Vec, + + pub sort_ascending: bool, + + /// ID of the first record. + pub first_id: Option, + + /// ID of the last record. + pub last_id: Option, +} + +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Clone, Copy)] +pub enum ViewAs { + Moderator, + NonModerator, +} + +pub struct ListMediaFilesArgs<'a> { + pub limit: usize, + pub maybe_filter_media_type: Option, + pub maybe_offset: Option, + pub cursor_is_reversed: bool, + pub view_as: ViewAs, + pub mysql_pool: &'a MySqlPool, +} + +pub async fn list_media_files(args: ListMediaFilesArgs<'_>) -> AnyhowResult { + + let mut query = query_builder( + args.maybe_filter_media_type, + args.limit, + args.maybe_offset, + args.cursor_is_reversed, + args.view_as, + ); + + let query = query.build_query_as::(); + + let results = query.fetch_all(args.mysql_pool).await?; + + let first_id = results.first() + .map(|raw_result| raw_result.id); + + let last_id = results.last() + .map(|raw_result| raw_result.id); + + let results = results.into_iter() + .map(|record| { + MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_username: record.maybe_creator_username, + maybe_creator_display_name: record.maybe_creator_display_name, + maybe_creator_gravatar_hash: record.maybe_creator_gravatar_hash, + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + Ok(MediaFileListPage { + records: results, + sort_ascending: !args.cursor_is_reversed, + first_id, + last_id, + }) +} + +fn query_builder<'a>( + maybe_filter_media_type: Option, + limit: usize, + maybe_offset: Option, + cursor_is_reversed: bool, + view_as: ViewAs, +) -> QueryBuilder<'a, MySql> { + + // NB: Query cannot be statically checked by sqlx + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +SELECT + m.id, + m.token, + + m.origin_category, + m.origin_product_category, + + m.maybe_origin_model_type, + m.maybe_origin_model_token, + + m.media_type, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.maybe_creator_user_token, + u.username as maybe_creator_username, + u.display_name as maybe_creator_display_name, + u.email_gravatar_hash as maybe_creator_gravatar_hash, + + m.creator_set_visibility, + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users AS u + ON m.maybe_creator_user_token = u.token + +WHERE m.user_deleted_at IS NULL +AND m.mod_deleted_at IS NULL + "# + ); + + if let Some(media_type) = maybe_filter_media_type { + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.media_file_type = "); + query_builder.push_bind(media_type.to_str()); + } + + match view_as { + ViewAs::Moderator => {} + ViewAs::NonModerator=> { + // FIXME: Binding shouldn't require to_str(). + // Otherwise, it's calling the Display trait on the raw type which is resulting in an + // incorrect binding and runtime error. + query_builder.push(" AND m.creator_set_visibility = "); + query_builder.push_bind(Visibility::Public.to_str()); + } + } + + if let Some(offset) = maybe_offset { + if cursor_is_reversed { + query_builder.push(format!(" AND m.id < {offset} ")); + } else { + query_builder.push(format!(" AND m.id > {offset} ")); + } + } + + if cursor_is_reversed { + query_builder.push(" ORDER BY m.id ASC "); + } else { + query_builder.push(" ORDER BY m.id DESC "); + } + + query_builder.push(format!(" LIMIT {limit} ")); + + query_builder +} + +struct MediaFileListItemInternal { + id: i64, + token: MediaFileToken, + + origin_category: MediaFileOriginCategory, + origin_product_category: MediaFileOriginProductCategory, + maybe_origin_model_type: Option, + maybe_origin_model_token: Option, + + media_type: MediaFileType, + public_bucket_directory_hash: String, + maybe_public_bucket_prefix: Option, + maybe_public_bucket_extension: Option, + + maybe_creator_user_token: Option, + maybe_creator_username: Option, + maybe_creator_display_name: Option, + maybe_creator_gravatar_hash: Option, + + creator_set_visibility: Visibility, + + created_at: DateTime, + updated_at: DateTime, +} + +// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or +// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually +// implemented. Perhaps this will improve in the future, but for now manually constructed queries +// cannot have type hints, eg. the following: +// +// m.token as `token: tokens::tokens::media_files::MediaFileToken`, +// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`, +// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, +// +// This results in the automatic mapping not being able to be found by name (for macro derive), and +// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the +// full "as" clause). +impl FromRow<'_, MySqlRow> for MediaFileListItemInternal { + fn from_row(row: &MySqlRow) -> Result { + let maybe_creator_user_token : Option = row.try_get("maybe_creator_user_token")?; + let maybe_creator_user_token = maybe_creator_user_token.map(|user_token| UserToken::new(user_token)); + + Ok(Self { + id: row.try_get("id")?, + token: MediaFileToken::new(row.try_get("token")?), + origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?, + origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?, + maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?, + maybe_origin_model_token: row.try_get("maybe_origin_model_token")?, + media_type: MediaFileType::try_from_mysql_row(row, "media_type")?, + public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?, + maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, + maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, + maybe_creator_user_token, + maybe_creator_username: row.try_get("maybe_creator_username")?, + maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, + maybe_creator_gravatar_hash: row.try_get("maybe_creator_gravatar_hash")?, + creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs index 1f9480ee93..87bcc96ee8 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list_media_files_for_user.rs @@ -156,6 +156,7 @@ WHERE m.user_deleted_at IS NULL // Otherwise, it's calling the Display trait on the raw type which is resulting in an // incorrect binding and runtime error. query_builder.push(" AND m.media_file_type = "); + query_builder.push_bind(media_type.to_str()); } match view_as { diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 619f018ce5..0590f82159 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -4,5 +4,6 @@ pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; +pub mod list_media_files; pub mod list_media_files_for_user; pub mod update_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs new file mode 100644 index 0000000000..df1e91d32d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files.rs @@ -0,0 +1,214 @@ +use std::sync::Arc; + +use actix_web::{HttpMessage, HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Query}; +use chrono::{DateTime, Utc}; +use log::{info, warn}; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use mysql_queries::queries::media_files::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::common_responses::pagination_cursors::PaginationCursors; +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +#[derive(Deserialize)] +pub struct QueryParams { + pub sort_ascending: Option, + pub per_page: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, + pub filter_media_type: Option, +} + +#[derive(Serialize)] +pub struct SuccessResponse { + pub success: bool, + pub results: Vec, + pub pagination: PaginationCursors, +} + +#[derive(Serialize)] +pub struct MediaFileListItem { + pub token: MediaFileToken, + + pub origin_category: MediaFileOriginCategory, + pub origin_product_category: MediaFileOriginProductCategory, + pub maybe_origin_model_type: Option, + pub maybe_origin_model_token: Option, + + pub media_type: MediaFileType, + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub maybe_creator: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug)] +pub enum ErrorResponse { + ServerError, +} + +impl ResponseError for ErrorResponse { + fn status_code(&self) -> StatusCode { + match *self { + ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ErrorResponse::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +impl std::fmt::Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn list_media_files_handler( + http_request: HttpRequest, + query: Query, + server_state: web::Data> +) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ErrorResponse::ServerError + })?; + + let mut is_mod = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_mod = session.can_ban_users; + }, + }; + + // TODO(bt,2023-12-04): Enforce real maximums and defaults + let limit = query.per_page.unwrap_or(25); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor as usize) + } else { + None + }; + + let view_as = if is_mod { + ViewAs::Moderator + } else { + ViewAs::NonModerator + }; + + let query_results = list_media_files(ListMediaFilesArgs { + limit, + maybe_filter_media_type: query.filter_media_type, + maybe_offset: cursor, + cursor_is_reversed, + view_as, + mysql_pool: &server_state.mysql_pool, + }).await; + + + let results_page = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ErrorResponse::ServerError); + } + }; + + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + ErrorResponse::ServerError + })?; + Some(cursor) + } else { + None + }; + + let results = results_page.records.into_iter() + .map(|record| MediaFileListItem { + token: record.token, + origin_category: record.origin_category, + origin_product_category: record.origin_product_category, + maybe_origin_model_type: record.maybe_origin_model_type, + maybe_origin_model_token: record.maybe_origin_model_token, + media_type: record.media_type, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + maybe_creator: UserDetailsLight::from_optional_db_fields_owned( + record.maybe_creator_user_token, + record.maybe_creator_username, + record.maybe_creator_display_name, + record.maybe_creator_gravatar_hash, + ), + creator_set_visibility: record.creator_set_visibility, + created_at: record.created_at, + updated_at: record.updated_at, + }) + .collect::>(); + + let response = SuccessResponse { + success: true, + results, + pagination: PaginationCursors { + maybe_next: cursor_next, + maybe_previous: cursor_previous, + cursor_is_reversed, + } + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ErrorResponse::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs index 9cfe696a27..bd3eda2b78 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/list_media_files_for_user.rs @@ -94,8 +94,6 @@ pub async fn list_media_files_for_user_handler( server_state: web::Data> ) -> Result { - info!("Fetching inference results for user: {}", &path.username); - let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 6af4273b7a..ab3552ac20 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1,4 +1,5 @@ pub mod delete_media_file; pub mod drain_multipart_request; pub mod get_media_file; +pub mod list_media_files; pub mod list_media_files_for_user; diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 9aafd5340c..bae83ecc53 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -44,6 +44,7 @@ use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enab use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::media_files::delete_media_file::delete_media_file_handler; use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; +use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; @@ -1007,7 +1008,11 @@ fn add_media_file_routes (app: App) -> App .route(web::delete().to(delete_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(web::resource("/user/{username}") + .service(web::resource("/list") + .route(web::get().to(list_media_files_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/list/user/{username}") .route(web::get().to(list_media_files_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) From b5fcc3cdfac8493932780047495e809941d1393f Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:14:02 -0500 Subject: [PATCH 08/11] Added documentation about the documentation --- _develop/docs/README.md | 4 +++- _develop/docs/local_development/general_setup.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/_develop/docs/README.md b/_develop/docs/README.md index 9ba35171fa..8b55ea0f13 100644 --- a/_develop/docs/README.md +++ b/_develop/docs/README.md @@ -29,7 +29,9 @@ Web API Overview * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only * `w2l-download-job` - (*deprecated*) Async download of videos and images * `w2l-inference-job` - (*deprecated*) Async W2L inference - + * Documentation Server: + * `docs` - this will run a local swagger server with the endpoints that you have access to, http://127.0.0.1:8989/api-docs/openapi.json use this to port everything over to the post man. + ### PowerStream * [PowerStream](https://power.stream) (deprecated) is powered by the following applications: diff --git a/_develop/docs/local_development/general_setup.md b/_develop/docs/local_development/general_setup.md index 24d849051e..25c499e0c4 100644 --- a/_develop/docs/local_development/general_setup.md +++ b/_develop/docs/local_development/general_setup.md @@ -380,3 +380,16 @@ of this error message. You can essentially ignore it. ### [Fix] Can't connect to local MySQL after install on Ubuntu - If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux). + +### API Server +``` +cargo run --bin docs +``` +This setups an api server with swagger docs to show you responses and requests with each associated endpoint +It is run at http://127.0.0.1:8989/ +To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json +We use this: `https://github.com/juhaku/utoipa` +The documentation example is in: +`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs` +Write into the api doc to: +`crates/service/web/storyteller_web/src/api_doc.rs` \ No newline at end of file From 404853e129a6e21a3ff833df6df69a794a133d0b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 5 Dec 2023 19:16:55 -0500 Subject: [PATCH 09/11] cleanup misleading old comment --- crates/_types/enums/Cargo.toml | 4 ---- crates/_types/tokens/Cargo.toml | 4 ---- crates/cli/dev_database_seed/Cargo.toml | 4 ---- crates/cli/dev_upload_media_file/Cargo.toml | 4 ---- crates/cli/elasticsearch-cli/Cargo.toml | 4 ---- crates/lib/mysql_queries/Cargo.toml | 4 ---- crates/lib/sqlite_queries/Cargo.toml | 5 ----- crates/lib/sqlx_mysql_helpers/Cargo.toml | 4 ---- crates/programs/aichatbot-sidecar/Cargo.toml | 5 ----- crates/service/job/analytics_job/Cargo.toml | 4 ---- crates/service/job/download_job/Cargo.toml | 4 ---- crates/service/job/email_sender_job/Cargo.toml | 4 ---- crates/service/job/inference_job/Cargo.toml | 4 ---- crates/service/job/tts_download_job/Cargo.toml | 4 ---- crates/service/job/tts_inference_job/Cargo.toml | 4 ---- crates/service/job/w2l_download_job/Cargo.toml | 4 ---- crates/service/job/w2l_inference_job/Cargo.toml | 4 ---- crates/service/other/twitch_pubsub_subscriber/Cargo.toml | 5 ----- crates/service/plugins/billing/Cargo.toml | 4 ---- crates/service/plugins/users/Cargo.toml | 4 ---- crates/service/web/storyteller_web/Cargo.toml | 4 ---- crates/service/web/websocket_gateway/Cargo.toml | 4 ---- 22 files changed, 91 deletions(-) diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 8493b4c80b..1cb6c94409 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -17,10 +17,6 @@ path = "src/lib.rs" serde = "1.0.125" serde_derive = "1.0.125" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] } sqlx-core = { version = "0.7.2" } sqlx-mysql= { version = "0.7.2" } diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..cc8825e725 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -20,10 +20,6 @@ serde_derive = "1.0.125" # Random rand = "0.8.5" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev_database_seed/Cargo.toml index b2a4dcf25e..d2ef105cce 100644 --- a/crates/cli/dev_database_seed/Cargo.toml +++ b/crates/cli/dev_database_seed/Cargo.toml @@ -36,10 +36,6 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml index 2b3c6b2626..c1f7ed8790 100644 --- a/crates/cli/dev_upload_media_file/Cargo.toml +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -33,10 +33,6 @@ log = "0.4.14" # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/cli/elasticsearch-cli/Cargo.toml b/crates/cli/elasticsearch-cli/Cargo.toml index c3a234024c..9876936696 100644 --- a/crates/cli/elasticsearch-cli/Cargo.toml +++ b/crates/cli/elasticsearch-cli/Cargo.toml @@ -33,10 +33,6 @@ strum = { version = "0.25", features = ["derive"] } # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } elasticsearch = "8.5.0-alpha.1" diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index 02274f5412..bcdff745f8 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -39,10 +39,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" serial_test = "2.0.0" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 tokio = { version = "1.6.0", features = ["full"] } rand = "0.8" diff --git a/crates/lib/sqlite_queries/Cargo.toml b/crates/lib/sqlite_queries/Cargo.toml index 9427adfeb5..17f25f8d0b 100644 --- a/crates/lib/sqlite_queries/Cargo.toml +++ b/crates/lib/sqlite_queries/Cargo.toml @@ -28,11 +28,6 @@ log = "0.4.14" once_cell = "1.9.0" regex = "1.5.4" -# SQlite Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ sqlx = { version = "0.7.2", features = [ "sqlite", "runtime-tokio-native-tls", "chrono" ] } [dev-dependencies] diff --git a/crates/lib/sqlx_mysql_helpers/Cargo.toml b/crates/lib/sqlx_mysql_helpers/Cargo.toml index e7837fde69..647e980426 100644 --- a/crates/lib/sqlx_mysql_helpers/Cargo.toml +++ b/crates/lib/sqlx_mysql_helpers/Cargo.toml @@ -16,10 +16,6 @@ path = "src/lib.rs" # Internal errors = { path = "../errors" } -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index b6a9068b0a..cebc56b8a1 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -42,11 +42,6 @@ rand = "0.8.3" regex = "1.5.4" strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc. -# SQlite Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ sqlx = { version = "0.7.2", features = [ "sqlite", "runtime-tokio-native-tls", "chrono" ] } # Async Runtime diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 30640d7446..6a803df2a4 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -32,10 +32,6 @@ hostname = "0.3.1" log = "0.4.14" rand = "0.8.5" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Runtime diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 7b69a8a6cc..6a4c49eb42 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -43,10 +43,6 @@ subprocess = "0.2.9" tempdir = "0.3.7" zip = "0.6.6" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index 8a9324340d..b1385b31e8 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -56,10 +56,6 @@ once_cell = "1.18.0" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 36194c73a9..f82dbe6589 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -52,10 +52,6 @@ once_cell = "1.18.0" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index ccdd04bd2b..b57ea434e8 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -35,10 +35,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 846405c909..2cbff86bf7 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -41,10 +41,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index c71e96c24b..aa55cde374 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -35,10 +35,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 04d38e114d..6aefb7d7f6 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -33,10 +33,6 @@ log = "0.4.14" subprocess = "0.2.9" tempdir = "0.3.7" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 8de44b6db1..94e3520a39 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -47,11 +47,6 @@ futures = { version = "0.3.17", features = ["thread-pool"] } # feature for Threa futures-util = "0.3.29" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate - -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index fac7fcba86..5e879949eb 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -50,10 +50,6 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt # include support for async fn in traits. async-trait = "0.1.58" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index bee1b60deb..5d852fa380 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -58,10 +58,6 @@ serde_derive = "1.0.125" serde_json = "1.0.108" toml = "0.5.8" -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # TODO: Shouldn't really be a dependency, but need to add caching for the DDoS attack. diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..eef8bc4162 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -97,10 +97,6 @@ futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older ve futures-core = "0.3.29" # NB: For middleware/futures upgrade pin-project-lite = "0.2.9" # NB: For middleware/futures upgrade -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index f1202bcdd1..85df5ba18d 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -47,10 +47,6 @@ futures = { version = "0.3.17", features = ["thread-pool"] } # feature for Threa futures-util = "0.3.29" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } # Redis Database From 4856ac28aa15a034a39f8e9f7d7deae89f619c11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:19:36 +0000 Subject: [PATCH 10/11] Bump chrono from 0.4.22 to 0.4.31 Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.22 to 0.4.31. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.22...v0.4.31) --- updated-dependencies: - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 15 ++++++++++----- crates/_types/reusable_types/Cargo.toml | 2 +- crates/lib/datetimes/Cargo.toml | 2 +- .../lib/deprecated/http_server_common/Cargo.toml | 2 +- crates/lib/elasticsearch_schema/Cargo.toml | 2 +- crates/lib/jobs_common/Cargo.toml | 2 +- crates/lib/mysql_queries/Cargo.toml | 2 +- crates/lib/tts_common/Cargo.toml | 2 +- crates/lib/web_scrapers/Cargo.toml | 2 +- crates/service/job/analytics_job/Cargo.toml | 2 +- crates/service/job/download_job/Cargo.toml | 2 +- crates/service/job/email_sender_job/Cargo.toml | 2 +- crates/service/job/inference_job/Cargo.toml | 2 +- crates/service/job/tts_download_job/Cargo.toml | 2 +- crates/service/job/tts_inference_job/Cargo.toml | 2 +- crates/service/job/w2l_download_job/Cargo.toml | 2 +- crates/service/job/w2l_inference_job/Cargo.toml | 2 +- .../other/twitch_pubsub_subscriber/Cargo.toml | 2 +- crates/service/plugins/billing/Cargo.toml | 2 +- crates/service/plugins/users/Cargo.toml | 2 +- crates/service/web/dummy_service/Cargo.toml | 2 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- crates/service/web/websocket_gateway/Cargo.toml | 2 +- my-workspace-hack/Cargo.toml | 4 ++-- 24 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae154198df..b5ac403216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,6 +468,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -1037,18 +1043,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", - "winapi 0.3.9", + "windows-targets", ] [[package]] diff --git a/crates/_types/reusable_types/Cargo.toml b/crates/_types/reusable_types/Cargo.toml index 7b87cc85e7..0548cf530e 100644 --- a/crates/_types/reusable_types/Cargo.toml +++ b/crates/_types/reusable_types/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" # (none) # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } # Serialization serde = "1.0.125" diff --git a/crates/lib/datetimes/Cargo.toml b/crates/lib/datetimes/Cargo.toml index 8cd99557fa..2961491dce 100644 --- a/crates/lib/datetimes/Cargo.toml +++ b/crates/lib/datetimes/Cargo.toml @@ -13,7 +13,7 @@ name = "datetimes" path = "src/lib.rs" [dependencies] -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } once_cell = "1.18.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/deprecated/http_server_common/Cargo.toml b/crates/lib/deprecated/http_server_common/Cargo.toml index 82f7101d72..91bda125b9 100644 --- a/crates/lib/deprecated/http_server_common/Cargo.toml +++ b/crates/lib/deprecated/http_server_common/Cargo.toml @@ -21,7 +21,7 @@ reusable_types = { path = "../../../_types/reusable_types" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/elasticsearch_schema/Cargo.toml b/crates/lib/elasticsearch_schema/Cargo.toml index 7926f55ac1..d2dd451dae 100644 --- a/crates/lib/elasticsearch_schema/Cargo.toml +++ b/crates/lib/elasticsearch_schema/Cargo.toml @@ -23,7 +23,7 @@ elasticsearch = "8.5.0-alpha.1" log = "0.4.14" # Serialization -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.108" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/jobs_common/Cargo.toml b/crates/lib/jobs_common/Cargo.toml index ad3a0c9a3e..6f68d2eb5f 100644 --- a/crates/lib/jobs_common/Cargo.toml +++ b/crates/lib/jobs_common/Cargo.toml @@ -24,7 +24,7 @@ redis_common = { path = "../redis_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" r2d2_redis = "0.14.0" my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/mysql_queries/Cargo.toml b/crates/lib/mysql_queries/Cargo.toml index bcdff745f8..2af35c4ffc 100644 --- a/crates/lib/mysql_queries/Cargo.toml +++ b/crates/lib/mysql_queries/Cargo.toml @@ -25,7 +25,7 @@ tokens = { path = "../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 6e42e04e6a..e424163e24 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -20,7 +20,7 @@ user_input_common = { path = "../user_input_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index a5e01837ce..f17589be4d 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -19,7 +19,7 @@ errors = { path = "../../lib/errors" } # External bytes = "1.3" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } log = "0.4.14" once_cell = "1.18.0" strum = { version = "0.25", features = ["derive"] } # Fancy enum iteration, etc. diff --git a/crates/service/job/analytics_job/Cargo.toml b/crates/service/job/analytics_job/Cargo.toml index 6a803df2a4..436d75347c 100644 --- a/crates/service/job/analytics_job/Cargo.toml +++ b/crates/service/job/analytics_job/Cargo.toml @@ -26,7 +26,7 @@ tokens = { path = "../../../_types/tokens" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 6a4c49eb42..76d139dcb8 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -35,7 +35,7 @@ subprocess_common = { path = "../../../lib/subprocess_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/email_sender_job/Cargo.toml b/crates/service/job/email_sender_job/Cargo.toml index b1385b31e8..9c4d7f9eec 100644 --- a/crates/service/job/email_sender_job/Cargo.toml +++ b/crates/service/job/email_sender_job/Cargo.toml @@ -46,7 +46,7 @@ tts_common = { path = "../../../lib/tts_common" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index f82dbe6589..53334a6465 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -43,7 +43,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lfu = "0.2.5" diff --git a/crates/service/job/tts_download_job/Cargo.toml b/crates/service/job/tts_download_job/Cargo.toml index b57ea434e8..79741dc0d3 100644 --- a/crates/service/job/tts_download_job/Cargo.toml +++ b/crates/service/job/tts_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/tts_inference_job/Cargo.toml b/crates/service/job/tts_inference_job/Cargo.toml index 2cbff86bf7..02757a7af7 100644 --- a/crates/service/job/tts_inference_job/Cargo.toml +++ b/crates/service/job/tts_inference_job/Cargo.toml @@ -32,7 +32,7 @@ newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated dotenv = "0.15.0" hostname = "0.3.1" diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index aa55cde374..afb81aec18 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -28,7 +28,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/job/w2l_inference_job/Cargo.toml b/crates/service/job/w2l_inference_job/Cargo.toml index 6aefb7d7f6..a3aab860c6 100644 --- a/crates/service/job/w2l_inference_job/Cargo.toml +++ b/crates/service/job/w2l_inference_job/Cargo.toml @@ -26,7 +26,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" } # External anyhow = "1.0.75" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" log = "0.4.14" diff --git a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml index 94e3520a39..00af98d691 100644 --- a/crates/service/other/twitch_pubsub_subscriber/Cargo.toml +++ b/crates/service/other/twitch_pubsub_subscriber/Cargo.toml @@ -28,7 +28,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" lazy_static = "1.4.0" diff --git a/crates/service/plugins/billing/Cargo.toml b/crates/service/plugins/billing/Cargo.toml index 5e879949eb..fc99fded80 100644 --- a/crates/service/plugins/billing/Cargo.toml +++ b/crates/service/plugins/billing/Cargo.toml @@ -24,7 +24,7 @@ url_config = { path = "../../../lib/url_config" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" once_cell = "1.18.0" diff --git a/crates/service/plugins/users/Cargo.toml b/crates/service/plugins/users/Cargo.toml index 5d852fa380..09915d397c 100644 --- a/crates/service/plugins/users/Cargo.toml +++ b/crates/service/plugins/users/Cargo.toml @@ -34,7 +34,7 @@ user_input_common = { path = "../../../lib/user_input_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" lazy_static = "1.4.0" # Regex caching (TODO: Use OnceCell instead) log = "0.4.14" diff --git a/crates/service/web/dummy_service/Cargo.toml b/crates/service/web/dummy_service/Cargo.toml index abbd3f4e96..9c8de7f237 100644 --- a/crates/service/web/dummy_service/Cargo.toml +++ b/crates/service/web/dummy_service/Cargo.toml @@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" } reusable_types = { path = "../../../_types/reusable_types" } # External -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } dotenv = "0.15.0" hostname = "0.3.1" #once_cell = "1.9.0" diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index eef8bc4162..f298a33609 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -54,7 +54,7 @@ users_component = { path = "../../plugins/users" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hex = "0.4.3" diff --git a/crates/service/web/websocket_gateway/Cargo.toml b/crates/service/web/websocket_gateway/Cargo.toml index 85df5ba18d..5f051a7172 100644 --- a/crates/service/web/websocket_gateway/Cargo.toml +++ b/crates/service/web/websocket_gateway/Cargo.toml @@ -27,7 +27,7 @@ twitch_common = { path = "../../../lib/api_clients/twitch_common" } # External anyhow = "1.0.75" base64 = "0.21.0" -chrono = { version = "=0.4.22", features = ["serde"] } +chrono = { version = "=0.4.31", features = ["serde"] } derive_more = "0.99.17" dotenv = "0.15.0" hostname = "0.3.1" diff --git a/my-workspace-hack/Cargo.toml b/my-workspace-hack/Cargo.toml index 2f062e5a69..e93823db80 100644 --- a/my-workspace-hack/Cargo.toml +++ b/my-workspace-hack/Cargo.toml @@ -22,7 +22,7 @@ bitflags = { version = "2.4.1", default-features = false, features = ["serde", " block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] } bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1.5.0" } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] } crossbeam-utils = { version = "0.8.16" } deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -88,7 +88,7 @@ base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] } bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] } byteorder = { version = "1.5.0" } cc = { version = "1.0.83", default-features = false, features = ["parallel"] } -chrono = { version = "0.4.22", features = ["alloc", "serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } crossbeam-utils = { version = "0.8.16" } derive_more = { version = "0.99.17" } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } From 0033af73cef3a64c95ee95838b3bbf452257346e Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 20:26:07 -0500 Subject: [PATCH 11/11] Feedback addressed! --- Cargo.lock | 1 - crates/_types/tokens/Cargo.toml | 1 - .../web/storyteller_web/src/api_doc_server.rs | 82 +------------------ .../service/web/storyteller_web/src/routes.rs | 5 -- 4 files changed, 2 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 561b3a18b1..0a0f2d5d63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6459,7 +6459,6 @@ dependencies = [ "strum", "toml", "utoipa", - "utoipa-swagger-ui", ] [[package]] diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index f6276b98ab..7df9d68c0b 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -29,7 +29,6 @@ sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } -utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [dev-dependencies] diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index b9b4213409..e3fc865736 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -19,18 +19,9 @@ use utoipa_swagger_ui::SwaggerUi; -use std::sync::Arc; -use std::time::Duration; - -use actix_cors::Cors; -use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::{App, HttpServer, middleware, web}; use actix_web::middleware::{DefaultHeaders, Logger}; -use anyhow::anyhow; -use elasticsearch::Elasticsearch; -use elasticsearch::http::transport::Transport; -use futures::Future; -use limitation::Limiter; -use log::{error, info}; + use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; @@ -38,63 +29,10 @@ use sqlx::mysql::MySqlPoolOptions; use sqlx::MySqlPool; use tokio::runtime::Runtime; - - -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; -use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; -use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; -use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; -use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; -use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; -use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; -use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; -use config::common_env::CommonEnv; -use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; -use email_sender::smtp_email_sender::SmtpEmailSender; use errors::AnyhowResult; -use http_server_common::cors::{build_cors_config, build_production_cors_config}; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use mysql_queries::mediators::badge_granter::BadgeGranter; -use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use redis_caching::redis_ttl_cache::RedisTtlCache; -use reusable_types::server_environment::ServerEnvironment; -use twitch_common::twitch_secrets::TwitchSecrets; -use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; -use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; -use users_component::cookies::session::session_cookie_manager::SessionCookieManager; -use users_component::utils::session_checker::SessionChecker; -use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; -use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; -use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; -use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; -use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; -use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; -use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; -use crate::routes::add_routes; use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; -use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; -use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; -use crate::threads::poll_ip_banlist_thread::poll_ip_bans; -use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; -use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; -use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; - -pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); -pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); pub mod billing; pub mod configs; @@ -110,21 +48,6 @@ pub mod user_avatars; pub mod util; pub mod validations; -const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; - -// Buckets (shared config) -const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; -const ENV_SECRET_KEY : &str = "SECRET_KEY"; -const ENV_REGION_NAME : &str = "REGION_NAME"; - -// Buckets (private data) -const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; -// Buckets (public data) -const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; - -// Various bucket roots -const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; - use std::{ error::Error, future::{self, Ready}, @@ -135,7 +58,6 @@ pub mod api_doc; use api_doc::ApiDoc; use futures::future::LocalBoxFuture; - use utoipa::OpenApi; #[actix_web::main] async fn main() -> Result<(), impl Error> { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index d84410d643..48d3048c78 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,11 +161,6 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { - use utoipa::OpenApi; - - - - let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ app = add_w2l_routes(app); /* /w2l */