diff --git a/Cargo.lock b/Cargo.lock index 210075ab48..4e12b9dbc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1719,6 +1719,61 @@ dependencies = [ "http", ] +[[package]] +name = "http_server_common" +version = "0.0.1" +dependencies = [ + "actix", + "actix-cors", + "actix-files", + "actix-http", + "actix-multipart", + "actix-rt 2.2.0", + "actix-service", + "actix-utils 3.0.0", + "actix-web", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "data-encoding", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.15", + "hex", + "hmac 0.10.1", + "hostname", + "hyper", + "infer", + "jwt", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.3", + "regex", + "ring", + "rust-s3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "subprocess", + "tempdir", + "time 0.2.26", + "tokio 0.2.25", + "toml", + "uuid", +] + [[package]] name = "httparse" version = "1.5.1" @@ -2381,6 +2436,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper", "infer", "jwt", @@ -3500,6 +3556,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper", "infer", "jwt", diff --git a/Cargo.toml b/Cargo.toml index fb9eccd2cf..a5f41170c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "app/config", + "app/http_server_common", "app/obs_gateway", "app/storyteller_web", ] diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml new file mode 100644 index 0000000000..88c14d9ca3 --- /dev/null +++ b/app/http_server_common/Cargo.toml @@ -0,0 +1,108 @@ +[package] +name = "http_server_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "http_server_common" +path = "src/lib.rs" + +[dependencies] + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +hex = "0.4.3" +hostname = "0.3.1" +log = "0.4.14" +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. +#async-std = { version = "1.6", features = [ "attributes" ] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.26" } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +# Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +actix-rt = "2.2.0" +actix-service = "1.0.6" +actix-utils = "3.0.0" +actix-web = "3.3.2" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/storyteller_web/src/http_server/endpoints/default_route_404.rs b/app/http_server_common/src/endpoints/default_route_404.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/default_route_404.rs rename to app/http_server_common/src/endpoints/default_route_404.rs diff --git a/app/http_server_common/src/endpoints/mod.rs b/app/http_server_common/src/endpoints/mod.rs new file mode 100644 index 0000000000..9ae4ad6cf5 --- /dev/null +++ b/app/http_server_common/src/endpoints/mod.rs @@ -0,0 +1,2 @@ +pub mod default_route_404; +pub mod root_index; diff --git a/app/storyteller_web/src/http_server/endpoints/root_index.rs b/app/http_server_common/src/endpoints/root_index.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/root_index.rs rename to app/http_server_common/src/endpoints/root_index.rs diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs new file mode 100644 index 0000000000..34440e0e43 --- /dev/null +++ b/app/http_server_common/src/lib.rs @@ -0,0 +1,5 @@ +//! +//! Common HTTP Server components +//! + +pub mod endpoints; diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index bbb0580112..d4c5d95952 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } +http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 9c8a7c14db..2eadfc277a 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -21,6 +21,8 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::endpoints::default_route_404::default_route_404; +use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -186,8 +188,8 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> // .route(web::get().to(ws_index)) // .route(web::head().to(|| HttpResponse::Ok())) //) - //.service(get_root_index) - //.default_service( web::route().to(default_route_404)) + .service(get_root_index) + .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index ed5181cedd..3744dbf292 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,6 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../config" } +http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" diff --git a/app/storyteller_web/src/http_server/endpoints/mod.rs b/app/storyteller_web/src/http_server/endpoints/mod.rs index 38b3143d3f..3842da000d 100644 --- a/app/storyteller_web/src/http_server/endpoints/mod.rs +++ b/app/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,9 +1,7 @@ -pub mod default_route_404; pub mod events; pub mod leaderboard; pub mod misc; pub mod moderation; -pub mod root_index; pub mod tts; pub mod twitch; pub mod users; diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index b1ee31d84b..3d1aacdbb7 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -36,7 +36,6 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; -use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; @@ -55,7 +54,6 @@ use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; -use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; @@ -68,6 +66,7 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; use crate::http_server::endpoints::users::create_account::create_account_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; @@ -104,6 +103,8 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::endpoints::default_route_404::default_route_404; +use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -113,7 +114,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; // TODO TODO TODO TODO // TODO TODO TODO TODO