Make a shared "http server common" package

Cargo.lock

@@ -1719,6 +1719,61 @@ dependencies = [
  "http",
 ]
 
+[[package]]
+name = "http_server_common"
+version = "0.0.1"
+dependencies = [
+ "actix",
+ "actix-cors",
+ "actix-files",
+ "actix-http",
+ "actix-multipart",
+ "actix-rt 2.2.0",
+ "actix-service",
+ "actix-utils 3.0.0",
+ "actix-web",
+ "actix-web-actors",
+ "anyhow",
+ "base64 0.13.0",
+ "bcrypt",
+ "chrono",
+ "data-encoding",
+ "derive_more",
+ "diesel",
+ "dotenv",
+ "easyenv",
+ "futures 0.1.31",
+ "futures 0.3.15",
+ "hex",
+ "hmac 0.10.1",
+ "hostname",
+ "hyper",
+ "infer",
+ "jwt",
+ "limitation",
+ "log",
+ "lru_time_cache",
+ "magic-crypt",
+ "md-5",
+ "pulldown-cmark",
+ "r2d2_redis",
+ "rand 0.8.3",
+ "regex",
+ "ring",
+ "rust-s3",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "sha2",
+ "sqlx",
+ "subprocess",
+ "tempdir",
+ "time 0.2.26",
+ "tokio 0.2.25",
+ "toml",
+ "uuid",
+]
+
 [[package]]
 name = "httparse"
 version = "1.5.1"
@@ -2381,6 +2436,7 @@ dependencies = [
  "hex",
  "hmac 0.10.1",
  "hostname",
+ "http_server_common",
  "hyper",
  "infer",
  "jwt",
@@ -3500,6 +3556,7 @@ dependencies = [
  "hex",
  "hmac 0.10.1",
  "hostname",
+ "http_server_common",
  "hyper",
  "infer",
  "jwt",

Cargo.toml

@@ -2,6 +2,7 @@
 
 members = [
   "app/config",
+  "app/http_server_common",
   "app/obs_gateway",
   "app/storyteller_web",
 ]

app/http_server_common/Cargo.toml

@@ -0,0 +1,108 @@
+[package]
+name = "http_server_common"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "http_server_common"
+path = "src/lib.rs"
+
+[dependencies]
+
+# External
+anyhow = "1.0.40"
+base64 = "0.13.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+derive_more = "0.99.14"
+dotenv = "0.15.0"
+easyenv = "0.0.6"
+futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool
+futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate
+hex = "0.4.3"
+hostname = "0.3.1"
+log = "0.4.14"
+rand = "0.8.3"
+regex = "1.5.4"
+time = "0.2.26"
+uuid = { version = "0.8.2", features = ["v4"] }
+
+# Runtime
+# NB(1): We switched to Tokio from `async-std` for broader ecosystem support.
+#async-std = { version = "1.6", features = [ "attributes" ] }
+# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed:
+# https://github.com/durch/rust-s3/issues/198
+#tokio = { version = "1.6.0", features = ["macros"] }
+tokio = { version = "0.2.25", features = ["macros"] }
+
+# MySql Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] }
+# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade
+#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
+sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] }
+
+# Redis Database
+limitation = "0.1.1" # Redis rate limiter
+r2d2_redis = "0.14.0"
+
+# In-memory caching
+lru_time_cache = "0.11.11"
+
+# Files
+data-encoding = "2.3.2"
+ring = "0.17.0-alpha.10"
+# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories
+# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2
+#rust-s3 = "0.27.0-rc4"
+#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false }
+rust-s3 = { version = "0.26" }
+infer = "0.5.0"
+
+subprocess = "0.2.6"
+tempdir = "0.3.7"
+
+# Web
+actix = "0.10"
+#actix-web = "3"
+actix-web-actors = "3"
+
+actix-cors = "0.5.4"
+actix-files= "0.5"
+actix-http = "2.2.0"
+actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade
+actix-rt = "2.2.0"
+actix-service = "1.0.6"
+actix-utils = "3.0.0"
+actix-web = "3.3.2"
+jwt = "0.13.0"
+
+# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime
+# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper.
+hyper = { version = "0.13.10" }
+
+# Crypto
+bcrypt = "0.8" # password hashes
+hmac = "0.10.0" # cookies
+magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info
+md-5 = "0.9.1" # gravatar
+sha2 = "0.9.5" # file hashes, inference text hashes, etc.
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+serde_json = "1.0.64"
+toml = "0.5.8"
+
+# Text processing
+pulldown-cmark = "0.8.0"
+
+[build-dependencies]
+# NB: Only using diesel for migrations support
+diesel = { version = "1.4.6", features = ["mysql"] }

app/http_server_common/src/endpoints/mod.rs

@@ -0,0 +1,2 @@
+pub mod default_route_404;
+pub mod root_index;

app/http_server_common/src/lib.rs

@@ -0,0 +1,5 @@
+//!
+//! Common HTTP Server components
+//!
+
+pub mod endpoints;

app/obs_gateway/Cargo.toml

@@ -16,6 +16,7 @@ path = "src/main.rs"
 
 # Internal
 config = { path = "../config" }
+http_server_common = { path = "../http_server_common" }
 
 # External
 anyhow = "1.0.40"

app/obs_gateway/src/main.rs

@@ -21,6 +21,8 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
 use futures::Future;
 use futures::executor::ThreadPool;
+use http_server_common::endpoints::default_route_404::default_route_404;
+use http_server_common::endpoints::root_index::get_root_index;
 use limitation::Limiter;
 use log::{info};
 use r2d2_redis::RedisConnectionManager;
@@ -186,8 +188,8 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()>
         //      .route(web::get().to(ws_index))
         //      .route(web::head().to(|| HttpResponse::Ok()))
         //)
-        //.service(get_root_index)
-        //.default_service( web::route().to(default_route_404))
+        .service(get_root_index)
+        .default_service( web::route().to(default_route_404))
   })
       .bind(bind_address)?
       .workers(num_workers)

app/storyteller_web/Cargo.toml

@@ -39,6 +39,7 @@ path = "src/w2l_inference_job.rs"
 
 # Internal
 config = { path = "../config" }
+http_server_common = { path = "../http_server_common" }
 
 # External
 anyhow = "1.0.40"

app/storyteller_web/src/http_server/endpoints/mod.rs

@@ -1,9 +1,7 @@
-pub mod default_route_404;
 pub mod events;
 pub mod leaderboard;
 pub mod misc;
 pub mod moderation;
-pub mod root_index;
 pub mod tts;
 pub mod twitch;
 pub mod users;

app/storyteller_web/src/main.rs

@@ -36,7 +36,6 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
 use crate::database::mediators::badge_granter::BadgeGranter;
 use crate::database::mediators::firehose_publisher::FirehosePublisher;
-use crate::http_server::endpoints::default_route_404::default_route_404;
 use crate::http_server::endpoints::events::list_events::list_events_handler;
 use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler;
 use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler;
@@ -55,7 +54,6 @@ use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user
 use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler;
 use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler;
 use crate::http_server::endpoints::moderation::users::list_users::list_users_handler;
-use crate::http_server::endpoints::root_index::get_root_index;
 use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler;
 use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler;
 use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler;
@@ -68,6 +66,7 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u
 use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler;
 use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler;
 use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler;
+use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index;
 use crate::http_server::endpoints::users::create_account::create_account_handler;
 use crate::http_server::endpoints::users::edit_profile::edit_profile_handler;
 use crate::http_server::endpoints::users::get_profile::get_profile_handler;
@@ -104,6 +103,8 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache;
 use crate::util::encrypted_sort_id::SortKeyCrypto;
 use futures::Future;
 use futures::executor::ThreadPool;
+use http_server_common::endpoints::default_route_404::default_route_404;
+use http_server_common::endpoints::root_index::get_root_index;
 use limitation::Limiter;
 use log::{info};
 use r2d2_redis::RedisConnectionManager;
@@ -113,7 +114,6 @@ use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::sync::Arc;
 use std::time::Duration;
-use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index;
 
 // TODO TODO TODO TODO
 // TODO TODO TODO TODO