From 0c409c6af1bef2c8e568f28473401409fcf26df8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 15:03:42 -0500 Subject: [PATCH] Shared config package --- Cargo.lock | 60 ++++++ Cargo.toml | 1 + app/config/Cargo.toml | 13 ++ app/config/src/lib.rs | 2 + .../src/shared_constants.rs | 0 app/obs_gateway/Cargo.toml | 102 +++++++++ app/obs_gateway/src/main.rs | 200 +++++++++++++++++- app/storyteller_web/Cargo.toml | 5 + app/storyteller_web/src/common_env.rs | 2 +- ...ist_tts_inference_results_query_builder.rs | 2 +- ...ist_w2l_inference_results_query_builder.rs | 2 +- app/storyteller_web/src/main.rs | 7 +- app/storyteller_web/src/obs_gateway_server.rs | 200 ------------------ app/storyteller_web/src/tts_download_job.rs | 5 +- app/storyteller_web/src/tts_inference_job.rs | 5 +- app/storyteller_web/src/w2l_download_job.rs | 5 +- app/storyteller_web/src/w2l_inference_job.rs | 5 +- 17 files changed, 394 insertions(+), 222 deletions(-) create mode 100644 app/config/Cargo.toml create mode 100644 app/config/src/lib.rs rename app/{storyteller_web => config}/src/shared_constants.rs (100%) delete mode 100644 app/storyteller_web/src/obs_gateway_server.rs diff --git a/Cargo.lock b/Cargo.lock index 2a3988d4cf..210075ab48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -995,6 +995,10 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "config" +version = "0.0.1" + [[package]] name = "const_fn" version = "0.4.7" @@ -2351,6 +2355,61 @@ dependencies = [ [[package]] name = "obs-gateway-server" version = "0.0.1" +dependencies = [ + "actix", + "actix-cors", + "actix-files", + "actix-http", + "actix-multipart", + "actix-rt 2.2.0", + "actix-service", + "actix-utils 3.0.0", + "actix-web", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "config", + "data-encoding", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.15", + "hex", + "hmac 0.10.1", + "hostname", + "hyper", + "infer", + "jwt", + "lazy_static", + "lfu", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "newrelic-telemetry", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.3", + "regex", + "ring", + "rust-s3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "subprocess", + "tempdir", + "time 0.2.26", + "tokio 0.2.25", + "toml", + "uuid", +] [[package]] name = "once_cell" @@ -3430,6 +3489,7 @@ dependencies = [ "base64 0.13.0", "bcrypt", "chrono", + "config", "data-encoding", "derive_more", "diesel", diff --git a/Cargo.toml b/Cargo.toml index 6c5e053d99..fb9eccd2cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ + "app/config", "app/obs_gateway", "app/storyteller_web", ] diff --git a/app/config/Cargo.toml b/app/config/Cargo.toml new file mode 100644 index 0000000000..fc10d38795 --- /dev/null +++ b/app/config/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "config" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "config" +path = "src/lib.rs" diff --git a/app/config/src/lib.rs b/app/config/src/lib.rs new file mode 100644 index 0000000000..6faad52aea --- /dev/null +++ b/app/config/src/lib.rs @@ -0,0 +1,2 @@ + +pub mod shared_constants; \ No newline at end of file diff --git a/app/storyteller_web/src/shared_constants.rs b/app/config/src/shared_constants.rs similarity index 100% rename from app/storyteller_web/src/shared_constants.rs rename to app/config/src/shared_constants.rs diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 377d0d234c..bbb0580112 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -11,3 +11,105 @@ publish = false [[bin]] name = "obs-gateway-server" path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../config" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +hex = "0.4.3" +hostname = "0.3.1" +lazy_static = "1.4.0" +lfu = "0.2.5" +log = "0.4.14" +#newrelic = "0.2.2" +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. +#async-std = { version = "1.6", features = [ "attributes" ] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.26" } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +# Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +actix-rt = "2.2.0" +actix-service = "1.0.6" +actix-utils = "3.0.0" +actix-web = "3.3.2" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index c4357a72be..9c8a7c14db 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -1,4 +1,198 @@ +#![forbid(private_in_public)] +#![forbid(unused_must_use)] +//#![forbid(warnings)] -pub fn main() { - println!("TBD"); -} \ No newline at end of file +#![allow(dead_code)] +#![allow(non_snake_case)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use actix_cors::Cors; +use actix_http::http; +use actix_web::middleware::{Logger, DefaultHeaders}; +use actix_web::{HttpServer, web, HttpResponse, App}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use futures::Future; +use futures::executor::ThreadPool; +use limitation::Limiter; +use log::{info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::sync::Arc; +use std::time::Duration; + +const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; + +pub type AnyhowResult = anyhow::Result; + +/// State that is injected into every endpoint. +#[derive(Clone)] +pub struct ObsGatewayServerState { + /// Configuration from ENV vars. + /// Some of this might not be used. + pub env_config: EnvConfig, + pub hostname: String, +} + +#[derive(Clone)] +pub struct EnvConfig { + // Number of thread workers. + pub num_workers: usize, + pub bind_address: String, + pub cookie_domain: String, + pub cookie_secure: bool, + pub cookie_http_only: bool, + pub website_homepage_redirect: String, +} + +#[actix_web::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("obs-gateway-server-unknown".to_string()); + + info!("Hostname: {}", &server_hostname); + + info!("Connecting to database..."); + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_CONNECTION_STRING); + + let redis_connection_string = + easyenv::get_env_string_or_default( + "REDIS_URL", + DEFAULT_REDIS_CONNECTION_STRING); + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + + info!("Reading env vars and setting up utils..."); + + let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); + let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); + let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); + let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); + let website_homepage_redirect = + easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); + + let server_state = ObsGatewayServerState { + env_config: EnvConfig { + num_workers, + bind_address, + cookie_domain, + cookie_secure, + cookie_http_only, + website_homepage_redirect, + }, + hostname: server_hostname, + }; + + serve(server_state) + .await?; + Ok(()) +} + +pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> +{ + let bind_address = server_state.env_config.bind_address.clone(); + let num_workers = server_state.env_config.num_workers.clone(); + let hostname = server_state.hostname.clone(); + + let server_state_arc = web::Data::new(Arc::new(server_state)); + + info!("Starting HTTP service."); + + let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; + + HttpServer::new(move || { + App::new() + .app_data(server_state_arc.clone()) + .wrap(Cors::default() + .allowed_origin("http://api.fakeyou.com") + .allowed_origin("http://api.jungle.horse") + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://fakeyou.com") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") + .allowed_origin("http://localhost:12345") + .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:5555") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:8000") + .allowed_origin("http://localhost:8080") + .allowed_origin("http://mumble.stream") + .allowed_origin("http://trumped.com") + .allowed_origin("http://vo.codes") + .allowed_origin("http://vocodes.com") + .allowed_origin("https://api.fakeyou.com") + .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://api.vo.codes") + .allowed_origin("https://fakeyou.com") + .allowed_origin("https://jungle.horse") + .allowed_origin("https://mumble.stream") + .allowed_origin("https://trumped.com") + .allowed_origin("https://vo.codes") + .allowed_origin("https://vocodes.com") + .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .supports_credentials() + .allowed_headers(vec![ + http::header::ACCEPT, + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax + http::header::CONTENT_TYPE, + http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 + http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends + ]) + .max_age(3600)) + .wrap(Logger::new(&log_format) + .exclude("/liveness") + .exclude("/readiness")) + .wrap(DefaultHeaders::new() + .header("X-Backend-Hostname", &hostname) + .header("X-Build-Sha", "")) + // Twitch + //.service( + // web::resource("/twitch") + // .route(web::get().to(ws_index)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service(get_root_index) + //.default_service( web::route().to(default_route_404)) + }) + .bind(bind_address)? + .workers(num_workers) + .run() + .await?; + + Ok(()) +} diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index 4407bc2745..ed5181cedd 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -36,6 +36,11 @@ name = "w2l-inference-job" path = "src/w2l_inference_job.rs" [dependencies] + +# Internal +config = { path = "../config" } + +# External anyhow = "1.0.40" base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } diff --git a/app/storyteller_web/src/common_env.rs b/app/storyteller_web/src/common_env.rs index eb2b6cc1eb..9013327abc 100644 --- a/app/storyteller_web/src/common_env.rs +++ b/app/storyteller_web/src/common_env.rs @@ -1,4 +1,4 @@ -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::util::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. diff --git a/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs index 6840023eef..011629933a 100644 --- a/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs +++ b/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::database::enums::record_visibility::RecordVisibility; -use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs index a0e5c6b8ea..0553bf0552 100644 --- a/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs +++ b/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::database::enums::record_visibility::RecordVisibility; -use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 3a933b5675..b1ee31d84b 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -20,7 +20,6 @@ pub mod database; pub mod http_clients; pub mod http_server; pub mod server_state; -pub mod shared_constants; pub mod threads; pub mod util; pub mod validations; @@ -32,6 +31,9 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; @@ -95,9 +97,6 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; diff --git a/app/storyteller_web/src/obs_gateway_server.rs b/app/storyteller_web/src/obs_gateway_server.rs deleted file mode 100644 index 114f8c4dcf..0000000000 --- a/app/storyteller_web/src/obs_gateway_server.rs +++ /dev/null @@ -1,200 +0,0 @@ -#![forbid(private_in_public)] -#![forbid(unused_must_use)] -//#![forbid(warnings)] - -#![allow(dead_code)] -#![allow(non_snake_case)] -#![allow(unused_imports)] -#![allow(unused_mut)] -#![allow(unused_variables)] - -#[macro_use] extern crate lazy_static; -#[macro_use] extern crate magic_crypt; -#[macro_use] extern crate serde_derive; - -pub mod shared_constants; - -use actix_cors::Cors; -use actix_http::http; -use actix_web::middleware::{Logger, DefaultHeaders}; -use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; -use futures::Future; -use futures::executor::ThreadPool; -use limitation::Limiter; -use log::{info}; -use r2d2_redis::RedisConnectionManager; -use r2d2_redis::r2d2; -use r2d2_redis::redis::Commands; -use sqlx::MySqlPool; -use sqlx::mysql::MySqlPoolOptions; -use std::sync::Arc; -use std::time::Duration; - -const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; - -pub type AnyhowResult = anyhow::Result; - -/// State that is injected into every endpoint. -#[derive(Clone)] -pub struct ObsGatewayServerState { - /// Configuration from ENV vars. - /// Some of this might not be used. - pub env_config: EnvConfig, - pub hostname: String, -} - -#[derive(Clone)] -pub struct EnvConfig { - // Number of thread workers. - pub num_workers: usize, - pub bind_address: String, - pub cookie_domain: String, - pub cookie_secure: bool, - pub cookie_http_only: bool, - pub website_homepage_redirect: String, -} - -#[actix_web::main] -async fn main() -> AnyhowResult<()> { - easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); - - // NB: Do not check this secrets-containing dotenv file into VCS. - // This file should only contain *development* secrets, never production. - let _ = dotenv::from_filename(".env-secrets").ok(); - - info!("Obtaining hostname..."); - - let server_hostname = hostname::get() - .ok() - .and_then(|h| h.into_string().ok()) - .unwrap_or("obs-gateway-server-unknown".to_string()); - - info!("Hostname: {}", &server_hostname); - - info!("Connecting to database..."); - - let db_connection_string = - easyenv::get_env_string_or_default( - "MYSQL_URL", - DEFAULT_MYSQL_CONNECTION_STRING); - - let redis_connection_string = - easyenv::get_env_string_or_default( - "REDIS_URL", - DEFAULT_REDIS_CONNECTION_STRING); - - let pool = MySqlPoolOptions::new() - .max_connections(5) - .connect(&db_connection_string) - .await?; - - let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; - - let redis_pool = r2d2::Pool::builder() - .build(redis_manager)?; - - info!("Reading env vars and setting up utils..."); - - let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); - let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; - let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); - let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); - let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); - let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); - let website_homepage_redirect = - easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); - - let server_state = ObsGatewayServerState { - env_config: EnvConfig { - num_workers, - bind_address, - cookie_domain, - cookie_secure, - cookie_http_only, - website_homepage_redirect, - }, - hostname: server_hostname, - }; - - serve(server_state) - .await?; - Ok(()) -} - -pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> -{ - let bind_address = server_state.env_config.bind_address.clone(); - let num_workers = server_state.env_config.num_workers.clone(); - let hostname = server_state.hostname.clone(); - - let server_state_arc = web::Data::new(Arc::new(server_state)); - - info!("Starting HTTP service."); - - let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; - - HttpServer::new(move || { - App::new() - .app_data(server_state_arc.clone()) - .wrap(Cors::default() - .allowed_origin("http://api.fakeyou.com") - .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://api.vo.codes") - .allowed_origin("http://fakeyou.com") - .allowed_origin("http://jungle.horse") - .allowed_origin("http://jungle.horse:12345") - .allowed_origin("http://jungle.horse:7000") - .allowed_origin("http://localhost:12345") - .allowed_origin("http://localhost:3000") - .allowed_origin("http://localhost:5555") - .allowed_origin("http://localhost:7000") - .allowed_origin("http://localhost:8000") - .allowed_origin("http://localhost:8080") - .allowed_origin("http://mumble.stream") - .allowed_origin("http://trumped.com") - .allowed_origin("http://vo.codes") - .allowed_origin("http://vocodes.com") - .allowed_origin("https://api.fakeyou.com") - .allowed_origin("https://api.jungle.horse") - .allowed_origin("https://api.vo.codes") - .allowed_origin("https://fakeyou.com") - .allowed_origin("https://jungle.horse") - .allowed_origin("https://mumble.stream") - .allowed_origin("https://trumped.com") - .allowed_origin("https://vo.codes") - .allowed_origin("https://vocodes.com") - .allowed_methods(vec!["GET", "POST", "OPTIONS"]) - .supports_credentials() - .allowed_headers(vec![ - http::header::ACCEPT, - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax - http::header::CONTENT_TYPE, - http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 - http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends - ]) - .max_age(3600)) - .wrap(Logger::new(&log_format) - .exclude("/liveness") - .exclude("/readiness")) - .wrap(DefaultHeaders::new() - .header("X-Backend-Hostname", &hostname) - .header("X-Build-Sha", "")) - // Twitch - //.service( - // web::resource("/twitch") - // .route(web::get().to(ws_index)) - // .route(web::head().to(|| HttpResponse::Ok())) - //) - //.service(get_root_index) - //.default_service( web::route().to(default_route_404)) - }) - .bind(bind_address)? - .workers(num_workers) - .run() - .await?; - - Ok(()) -} diff --git a/app/storyteller_web/src/tts_download_job.rs b/app/storyteller_web/src/tts_download_job.rs index e08dc1cc2f..7718e356b5 100644 --- a/app/storyteller_web/src/tts_download_job.rs +++ b/app/storyteller_web/src/tts_download_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::enums::tts_model_type::TtsModelType; use crate::database::mediators::badge_granter::BadgeGranter; @@ -32,8 +33,6 @@ use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; diff --git a/app/storyteller_web/src/tts_inference_job.rs b/app/storyteller_web/src/tts_inference_job.rs index 5f67dfd22b..89aa39a78a 100644 --- a/app/storyteller_web/src/tts_inference_job.rs +++ b/app/storyteller_web/src/tts_inference_job.rs @@ -15,11 +15,12 @@ pub mod database; pub mod http_clients; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; @@ -35,8 +36,6 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failur use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_permanently_dead; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; diff --git a/app/storyteller_web/src/w2l_download_job.rs b/app/storyteller_web/src/w2l_download_job.rs index e0b49db743..1d0c3344f1 100644 --- a/app/storyteller_web/src/w2l_download_job.rs +++ b/app/storyteller_web/src/w2l_download_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::{anyhow, Error}; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; @@ -34,8 +35,6 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; diff --git a/app/storyteller_web/src/w2l_inference_job.rs b/app/storyteller_web/src/w2l_inference_job.rs index a7d29b6aa6..75f42e4f63 100644 --- a/app/storyteller_web/src/w2l_inference_job.rs +++ b/app/storyteller_web/src/w2l_inference_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; @@ -33,8 +34,6 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier;