diff --git a/Cargo.lock b/Cargo.lock index ae154198df..561b3a18b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,6 +2964,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", + "serde", ] [[package]] @@ -4393,6 +4394,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -4876,6 +4901,41 @@ dependencies = [ "quick-xml 0.30.0", ] +[[package]] +name = "rust-embed" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn 2.0.39", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada" +dependencies = [ + "sha2 0.10.8", + "walkdir", +] + [[package]] name = "rust-ini" version = "0.18.0" @@ -5411,6 +5471,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5894,6 +5963,8 @@ dependencies = [ "url_config", "user_input_common", "users_component", + "utoipa", + "utoipa-swagger-ui", "uuid 0.8.2", ] @@ -6387,6 +6458,8 @@ dependencies = [ "sqlx-core", "strum", "toml", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -7025,6 +7098,47 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utoipa" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "regex", + "syn 2.0.39", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" +dependencies = [ + "actix-web", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "utoipa", + "zip", +] + [[package]] name = "uuid" version = "0.8.2" diff --git a/crates/_types/tokens/Cargo.toml b/crates/_types/tokens/Cargo.toml index c688d59a69..f6276b98ab 100644 --- a/crates/_types/tokens/Cargo.toml +++ b/crates/_types/tokens/Cargo.toml @@ -28,6 +28,9 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", sqlx-core = { version = "0.7.2" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } + [dev-dependencies] # Enum utilities diff --git a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs index 1c455e457e..48c3a78260 100644 --- a/crates/_types/tokens/src/tokens/generic_inference_jobs.rs +++ b/crates/_types/tokens/src/tokens/generic_inference_jobs.rs @@ -4,9 +4,10 @@ use serde::Deserialize; use serde::Serialize; use crate::prefixes::TokenPrefix; +use utoipa::ToSchema; /// The primary key for "generic" inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)] #[sqlx(transparent)] pub struct InferenceJobToken(String); diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 50328fb0c6..8cedfffde2 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -13,6 +13,10 @@ publish = false name = "storyteller-web" path = "src/main.rs" +[[bin]] +name = "docs" +path = "src/api_doc_server.rs" + [dependencies] # Internal @@ -160,6 +164,8 @@ pulldown-cmark = "0.9.3" # include support for async fn in traits. async-trait = "0.1.58" my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" } +utoipa = { version = "4", features = ["actix_extras"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [build-dependencies] # NB: Only using diesel for migrations support diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs new file mode 100644 index 0000000000..f76feb1e76 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -0,0 +1,11 @@ + +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequest; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestSuccessResponse; +use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::EnqueueTTSRequestError; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; + +use utoipa::OpenApi; +#[derive(OpenApi)] +#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request), + components(schemas(EnqueueTTSRequest,EnqueueTTSRequestSuccessResponse,EnqueueTTSRequestError,InferenceJobToken)))] +pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs new file mode 100644 index 0000000000..b9b4213409 --- /dev/null +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -0,0 +1,151 @@ +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +//#![forbid(warnings)] +#![allow(unreachable_patterns)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use utoipa_swagger_ui::SwaggerUi; + +use std::sync::Arc; +use std::time::Duration; + +use actix_cors::Cors; +use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use actix_web::middleware::{DefaultHeaders, Logger}; +use anyhow::anyhow; +use elasticsearch::Elasticsearch; +use elasticsearch::http::transport::Transport; +use futures::Future; +use limitation::Limiter; +use log::{error, info}; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use r2d2_redis::RedisConnectionManager; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; +use tokio::runtime::Runtime; + + + +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_filter::BannedCidrFilter; +use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet; +use actix_helpers::middleware::banned_cidr_filter::load_cidr_ban_set_from_file::load_cidr_ban_set_from_file; +use actix_helpers::middleware::banned_ip_filter::banned_ip_filter::BannedIpFilter; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList; +use actix_helpers::middleware::banned_ip_filter::ip_ban_list::load_ip_ban_list_from_directory::load_ip_ban_list_from_directory; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filter::DisabledEndpointFilter; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints; +use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints; +use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets}; +use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup; +use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge; +use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup; +use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup; +use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; +use cloud_storage::bucket_client::BucketClient; +use config::common_env::CommonEnv; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; +use email_sender::smtp_email_sender::SmtpEmailSender; +use errors::AnyhowResult; +use http_server_common::cors::{build_cors_config, build_production_cors_config}; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use mysql_queries::mediators::badge_granter::BadgeGranter; +use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use redis_caching::redis_ttl_cache::RedisTtlCache; +use reusable_types::server_environment::ServerEnvironment; +use twitch_common::twitch_secrets::TwitchSecrets; +use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; +use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager; +use users_component::cookies::session::session_cookie_manager::SessionCookieManager; +use users_component::utils::session_checker::SessionChecker; + +use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl; +use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl; +use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl; +use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl; +use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; +use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::routes::add_routes; +use crate::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, RedisRateLimiters, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, TwitchOauth, TwitchOauthSecrets}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; +use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; +use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::troll_user_bans::load_troll_user_ban_list_from_directory::load_user_token_ban_list_from_directory; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; + +pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt"); + +pub mod billing; +pub mod configs; +pub mod cookies; +pub mod http_server; +pub mod memory_cache; +pub mod model; +pub mod routes; +pub mod server_state; +pub mod subscriptions; +pub mod threads; +pub mod user_avatars; +pub mod util; +pub mod validations; + +const DEFAULT_BIND_ADDRESS : &str = "0.0.0.0:12345"; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &str = "SECRET_KEY"; +const ENV_REGION_NAME : &str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &str = "AUDIO_UPLOADS_BUCKET_ROOT"; + +use std::{ + error::Error, + future::{self, Ready}, + net::Ipv4Addr, +}; + +pub mod api_doc; +use api_doc::ApiDoc; + +use futures::future::LocalBoxFuture; + +use utoipa::OpenApi; +#[actix_web::main] +async fn main() -> Result<(), impl Error> { + HttpServer::new(move || { + App::new() + .service( + SwaggerUi::new("/{_:.*}") + .url("/api-docs/openapi.json", ApiDoc::openapi()), + ) + }) + .bind(("localhost", 8989)).unwrap() + .run().await +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs index d394edabcd..32ae10d58d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs @@ -1,8 +1,8 @@ #![forbid(unused_imports)] #![forbid(unused_mut)] #![forbid(unused_variables)] +use utoipa::ToSchema; -// TODO MOVE into own file. use std::fmt::Debug; use std::sync::Arc; @@ -42,7 +42,9 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode"; /// This is useful for catching the live logs or intercepting the job. const ROUTING_TAG_HEADER_NAME: &str = "routing-tag"; + #[derive(Deserialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, @@ -50,12 +52,14 @@ pub struct EnqueueTTSRequest { } #[derive(Serialize)] +#[derive(ToSchema)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } #[derive(Debug)] +#[derive(ToSchema)] pub enum EnqueueTTSRequestError { BadInput(String), NotAuthorized, @@ -95,13 +99,26 @@ impl std::fmt::Display for EnqueueTTSRequestError { // Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions // insert generic inference job.rs // Need to convert it to generic inference job. +#[utoipa::path( + post, + path = "/inference/enqueue_tts/", + responses( + (status = 200, description = "Enqueue TTS generically", body = EnqueueTTSRequestSuccessResponse), + (status = 400, description = "Bad input", body = EnqueueTTSRequestError), + (status = 401, description = "Not authorized", body = EnqueueTTSRequestError), + (status = 429, description = "Rate limited", body = EnqueueTTSRequestError), + (status = 500, description = "Server error", body = EnqueueTTSRequestError) + ), + params( + ("request" = EnqueueTTSRequest, description = "Payload for TTS Request") + ) +)] pub async fn enqueue_tts_request( http_request: HttpRequest, request: web::Json, server_state: web::Data> ) -> Result { - println!("Received payload"); - + let mut maybe_user_token: Option = None; let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 071b76c46d..d84410d643 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -161,6 +161,10 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> InitError = (), >, { + use utoipa::OpenApi; + + + let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */