From e986a44e9daf942be923e2ef1276cd552114bd0d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:00:44 -0500 Subject: [PATCH 1/3] documentation system open api spec. --- Cargo.lock | 114 +++++++++++++ crates/_types/tokens/Cargo.toml | 3 + .../src/tokens/generic_inference_jobs.rs | 3 +- crates/service/web/storyteller_web/Cargo.toml | 6 + .../web/storyteller_web/src/api_doc.rs | 11 ++ .../web/storyteller_web/src/api_doc_server.rs | 151 ++++++++++++++++++ .../inference/enqueue_tts_request.rs | 23 ++- .../service/web/storyteller_web/src/routes.rs | 4 + 8 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/api_doc.rs create mode 100644 crates/service/web/storyteller_web/src/api_doc_server.rs diff --git a/Cargo.lock b/Cargo.lock index ae154198df..561b3a18b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,6 +2964,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", + "serde", ] [[package]] @@ -4393,6 +4394,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -4876,6 +4901,41 @@ dependencies = [ "quick-xml 0.30.0", ] +[[package]] +name = "rust-embed" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn 2.0.39", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada" +dependencies = [ + "sha2 0.10.8", + "walkdir", +] + [[package]] name = "rust-ini" version = "0.18.0" @@ -5411,6 +5471,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5894,6 +5963,8 @@ dependencies = [ "url_config", "user_input_common", "users_component", + "utoipa", + "utoipa-swagger-ui", "uuid 0.8.2", ] @@ -6387,6 +6458,8 @@ dependencies = [ "sqlx-core", "strum", "toml", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -7025,6 +7098,47 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utoipa" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "regex", + "syn 2.0.39", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" +dependencies = [ + "actix-web", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "utoipa", + "zip", +] + [[package]] name = "uuid" version = "0.8.2" diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..f6276b98ab 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -28,6 +28,9 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } + [dev-dependencies] # Enum utilities diff --git a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs index 1c455e457e..48c3a78260 100644 --- a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs +++ b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs @@ -4,9 +4,10 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for "generic" inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct InferenceJobToken(String); diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..8cedfffde2 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -13,6 +13,10 @@ publish = false name = "storyteller-web" path = "src/main.rs" +[[bin]] +name = "docs" +path = "src/api_doc_server.rs" + [dependencies] # Internal @@ -160,6 +164,8 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs new file mode 100644 index 0000000000..f76feb1e76 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -0,0 +1,11 @@ + +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; + +use utoipa::OpenApi; +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), + components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] +pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs new file mode 100644 index 0000000000..b9b4213409 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -0,0 +1,151 @@ +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +//#![forbid(warnings)] +#![allow(unreachable_patterns)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use utoipa_swagger_ui::SwaggerUi; + +use std::sync::Arc; +use std::time::Duration; + +use actix_cors::Cors; +use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::middleware::{DefaultHeaders, Logger}; +use anyhow::anyhow; +use elasticsearch::Elasticsearch; +use elasticsearch::http::transport::Transport; +use futures::Future; +use limitation::Limiter; +use log::{error, info}; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use r2d2_redis::RedisConnectionManager; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; +use tokio::runtime::Runtime; + + + +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; +use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; +use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; +use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; +use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; +use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; +use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; +use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; +use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; +use cloud_storage::bucket_client::BucketClient; +use config::common_env::CommonEnv; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; +use email_sender::smtp_email_sender::SmtpEmailSender; +use errors::AnyhowResult; +use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use mysql_queries::mediators::badge_granter::BadgeGranter; +use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use redis_caching::redis_ttl_cache::RedisTtlCache; +use reusable_types::server_environment::ServerEnvironment; +use twitch_common::twitch_secrets::TwitchSecrets; +use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; +use users_component::cookies::session::session_cookie_manager::SessionCookieManager; +use users_component::utils::session_checker::SessionChecker; + +use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; +use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; +use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; +use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::routes::add_routes; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; +use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; +use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; + +pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); + +pub mod billing; +pub mod configs; +pub mod cookies; +pub mod http_server; +pub mod memory_cache; +pub mod model; +pub mod routes; +pub mod server_state; +pub mod subscriptions; +pub mod threads; +pub mod user_avatars; +pub mod util; +pub mod validations; + +const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &str = "SECRET_KEY"; +const ENV_REGION_NAME : &str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; + +use std::{ + error::Error, + future::{self, Ready}, + net::Ipv4Addr, +}; + +pub mod api_doc; +use api_doc::ApiDoc; + +use futures::future::LocalBoxFuture; + +use utoipa::OpenApi; +#[actix_web::main] +async fn main() -> Result<(), impl Error> { + HttpServer::new(move || { + App::new() + .service( + SwaggerUi::new("/{_:.*}") + .url("/api-docs/openapi.json", ApiDoc::openapi()), + ) + }) + .bind(("localhost", 8989)).unwrap() + .run().await +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index d394edabcd..32ae10d58d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,8 +1,8 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] +use utoipa::ToSchema; -// TODO MOVE into own file. use std::fmt::Debug; use std::sync::Arc; @@ -42,7 +42,9 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; /// This is useful for catching the live logs or intercepting the job. const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; + #[derive(Deserialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, @@ -50,12 +52,14 @@ pub struct EnqueueTTSRequest { } #[derive(Serialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] +#[derive(ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, @@ -95,13 +99,26 @@ impl std::fmt::Display for EnqueueTTSRequestError { // Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions // insert generic inference job.rs // Need to convert it to generic inference job. +#[utoipa::path( + post, + path = "/inference/enqueue_tts/", + responses( + (status = 200, description = "Enqueue TTS generically", body = EnqueueTTSRequestSuccessResponse), + (status = 400, description = "Bad input", body = EnqueueTTSRequestError), + (status = 401, description = "Not authorized", body = EnqueueTTSRequestError), + (status = 429, description = "Rate limited", body = EnqueueTTSRequestError), + (status = 500, description = "Server error", body = EnqueueTTSRequestError) + ), + params( + ("request" = EnqueueTTSRequest, description = "Payload for TTS Request") + ) +)] pub async fn enqueue_tts_request( http_request: HttpRequest, request: web::Json, server_state: web::Data> ) -> Result { - println!("Received payload"); - + let mut maybe_user_token: Option = None; let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..d84410d643 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,6 +161,10 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { + use utoipa::OpenApi; + + + let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ From b5fcc3cdfac8493932780047495e809941d1393f Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 19:14:02 -0500 Subject: [PATCH 2/3] Added documentation about the documentation --- _develop/docs/README.md | 4 +++- _develop/docs/local_development/general_setup.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/_develop/docs/README.md b/_develop/docs/README.md index 9ba35171fa..8b55ea0f13 100644 --- a/_develop/docs/README.md +++ b/_develop/docs/README.md @@ -29,7 +29,9 @@ Web API Overview * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only * `w2l-download-job` - (*deprecated*) Async download of videos and images * `w2l-inference-job` - (*deprecated*) Async W2L inference - + * Documentation Server: + * `docs` - this will run a local swagger server with the endpoints that you have access to, http://127.0.0.1:8989/api-docs/openapi.json use this to port everything over to the post man. + ### PowerStream * [PowerStream](https://power.stream) (deprecated) is powered by the following applications: diff --git a/_develop/docs/local_development/general_setup.md b/_develop/docs/local_development/general_setup.md index 24d849051e..25c499e0c4 100644 --- a/_develop/docs/local_development/general_setup.md +++ b/_develop/docs/local_development/general_setup.md @@ -380,3 +380,16 @@ of this error message. You can essentially ignore it. ### [Fix] Can't connect to local MySQL after install on Ubuntu - If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux). + +### API Server +``` +cargo run --bin docs +``` +This setups an api server with swagger docs to show you responses and requests with each associated endpoint +It is run at http://127.0.0.1:8989/ +To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json +We use this: `https://github.com/juhaku/utoipa` +The documentation example is in: +`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs` +Write into the api doc to: +`crates/service/web/storyteller_web/src/api_doc.rs` \ No newline at end of file From 0033af73cef3a64c95ee95838b3bbf452257346e Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 5 Dec 2023 20:26:07 -0500 Subject: [PATCH 3/3] Feedback addressed! --- Cargo.lock | 1 - crates/_types/tokens/Cargo.toml | 1 - .../web/storyteller_web/src/api_doc_server.rs | 82 +------------------ .../service/web/storyteller_web/src/routes.rs | 5 -- 4 files changed, 2 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 561b3a18b1..0a0f2d5d63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6459,7 +6459,6 @@ dependencies = [ "strum", "toml", "utoipa", - "utoipa-swagger-ui", ] [[package]] diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index f6276b98ab..7df9d68c0b 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -29,7 +29,6 @@ sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } utoipa = { version = "4", features = ["actix_extras"] } -utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [dev-dependencies] diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index b9b4213409..e3fc865736 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -19,18 +19,9 @@ use utoipa_swagger_ui::SwaggerUi; -use std::sync::Arc; -use std::time::Duration; - -use actix_cors::Cors; -use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::{App, HttpServer, middleware, web}; use actix_web::middleware::{DefaultHeaders, Logger}; -use anyhow::anyhow; -use elasticsearch::Elasticsearch; -use elasticsearch::http::transport::Transport; -use futures::Future; -use limitation::Limiter; -use log::{error, info}; + use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; @@ -38,63 +29,10 @@ use sqlx::mysql::MySqlPoolOptions; use sqlx::MySqlPool; use tokio::runtime::Runtime; - - -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; -use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; -use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; -use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; -use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; -use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; -use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; -use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; -use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; -use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; -use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; -use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; -use config::common_env::CommonEnv; -use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use config::shared_constants::DEFAULT_RUST_LOG; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; -use email_sender::smtp_email_sender::SmtpEmailSender; use errors::AnyhowResult; -use http_server_common::cors::{build_cors_config, build_production_cors_config}; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use mysql_queries::mediators::badge_granter::BadgeGranter; -use mysql_queries::mediators::firehose_publisher::FirehosePublisher; -use redis_caching::redis_ttl_cache::RedisTtlCache; -use reusable_types::server_environment::ServerEnvironment; -use twitch_common::twitch_secrets::TwitchSecrets; -use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; -use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; -use users_component::cookies::session::session_cookie_manager::SessionCookieManager; -use users_component::utils::session_checker::SessionChecker; -use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; -use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; -use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; -use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; -use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; -use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; -use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; -use crate::routes::add_routes; use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; -use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; -use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; -use crate::threads::poll_ip_banlist_thread::poll_ip_bans; -use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; -use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; -use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; - -pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); -pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); pub mod billing; pub mod configs; @@ -110,21 +48,6 @@ pub mod user_avatars; pub mod util; pub mod validations; -const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; - -// Buckets (shared config) -const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; -const ENV_SECRET_KEY : &str = "SECRET_KEY"; -const ENV_REGION_NAME : &str = "REGION_NAME"; - -// Buckets (private data) -const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; -// Buckets (public data) -const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; - -// Various bucket roots -const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; - use std::{ error::Error, future::{self, Ready}, @@ -135,7 +58,6 @@ pub mod api_doc; use api_doc::ApiDoc; use futures::future::LocalBoxFuture; - use utoipa::OpenApi; #[actix_web::main] async fn main() -> Result<(), impl Error> { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index d84410d643..48d3048c78 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,11 +161,6 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { - use utoipa::OpenApi; - - - - let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ app = add_w2l_routes(app); /* /w2l */