diff --git a/crates/lib/config/src/shared_constants.rs b/crates/lib/config/src/shared_constants.rs index a562d9e317..51701db66d 100644 --- a/crates/lib/config/src/shared_constants.rs +++ b/crates/lib/config/src/shared_constants.rs @@ -7,7 +7,11 @@ pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=war //const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // but sometimes we want to debug /// The default Redis connection string for use in development -pub const DEFAULT_REDIS_CONNECTION_STRING: &'static str = "redis://localhost"; +pub const DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING: &'static str = "redis://localhost/0"; + +/// The default Redis connection string for use in development +/// Database 1 +pub const DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING: &'static str = "redis://localhost/1"; /// The default MySql connection string for use in development pub const DEFAULT_MYSQL_CONNECTION_STRING: &'static str = "mysql://storyteller:password@localhost/storyteller"; diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 9cd82890b8..c7ce6f5071 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -23,7 +23,7 @@ use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::endpoints::oauth_begin::oauth_begin_enroll; use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; @@ -187,11 +187,6 @@ async fn main() -> AnyhowResult<()> { // "MYSQL_URL", // DEFAULT_MYSQL_CONNECTION_STRING); // -// let redis_connection_string = -// easyenv::get_env_string_or_default( -// "REDIS_URL", -// DEFAULT_REDIS_CONNECTION_STRING); -// // let pool = MySqlPoolOptions::new() // .max_connections(5) // .connect(&db_connection_string) @@ -227,16 +222,29 @@ async fn main() -> AnyhowResult<()> { "MYSQL_URL", DEFAULT_MYSQL_CONNECTION_STRING); - //let redis_connection_string = - // easyenv::get_env_string_or_default( - // "REDIS_URL", - // DEFAULT_REDIS_CONNECTION_STRING); + let redis_connection_string = + easyenv::get_env_string_or_default( + "REDIS_1_URL", + DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING); + + info!("Connecting to mysql..."); let pool = MySqlPoolOptions::new() .max_connections(5) .connect(&db_connection_string) .await?; + info!("Connecting to redis..."); + + let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + + // NB: Compiler can't figure out throwaway return type + let key = "hi-database-1".to_string(); + let _r : String = redis_pool.get().unwrap().set(&key, "bar").unwrap(); + let server_state = ObsGatewayServerState { env_config: EnvConfig { num_workers, @@ -259,6 +267,7 @@ async fn main() -> AnyhowResult<()> { hostname: server_hostname, backends: BackendsConfig { mysql_pool: pool, + redis_pool, } }; diff --git a/crates/service/obs_gateway/src/server_state.rs b/crates/service/obs_gateway/src/server_state.rs index def6e3e771..79bdb94610 100644 --- a/crates/service/obs_gateway/src/server_state.rs +++ b/crates/service/obs_gateway/src/server_state.rs @@ -1,3 +1,4 @@ +use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; use twitch_oauth2::{ClientId, ClientSecret}; @@ -18,6 +19,7 @@ pub struct ObsGatewayServerState { #[derive(Clone)] pub struct BackendsConfig { pub mysql_pool: MySqlPool, + pub redis_pool: r2d2::Pool, } // TODO: Many of these do not need to be passed around past server init. diff --git a/crates/service/storyteller_web/src/common_env.rs b/crates/service/storyteller_web/src/common_env.rs index 77db528953..a2e272762a 100644 --- a/crates/service/storyteller_web/src/common_env.rs +++ b/crates/service/storyteller_web/src/common_env.rs @@ -1,4 +1,5 @@ -use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; use container_common::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. @@ -9,8 +10,12 @@ use container_common::anyhow_result::AnyhowResult; pub struct CommonEnv { /// The Redis connection string. - /// Currently we only have a single Redis that serves all purposes. - pub redis_connection_string: String, + /// This has FakeYou voice stats, rate limiters, etc. + pub redis_0_connection_string: String, + + /// The Redis connection string. + /// This has OBS/Twitch related functionality + pub redis_1_connection_string: String, /// The amount of time to wait between job batches (not individual jobs). /// This prevents the outer loop of querying batches from flooding the DB. @@ -37,8 +42,10 @@ impl CommonEnv { pub fn read_from_env() -> AnyhowResult { Ok(Self { - redis_connection_string: easyenv::get_env_string_or_default("REDIS_URL", - DEFAULT_REDIS_CONNECTION_STRING), + redis_0_connection_string: easyenv::get_env_string_or_default("REDIS_0_URL", + DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING), + redis_1_connection_string: easyenv::get_env_string_or_default("REDIS_1_URL", + DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING), job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?, job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?, job_batch_size: easyenv::get_env_num("JOB_BATCH_SIZE", 10)?, diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index a203619c2c..1bd7ed1f4f 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -17,6 +17,7 @@ pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/rese pub const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt"); pub mod database; +pub mod common_env; pub mod http_clients; pub mod http_server; pub mod routes; @@ -40,9 +41,9 @@ use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; +use crate::common_env::CommonEnv; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -162,6 +163,8 @@ async fn main() -> AnyhowResult<()> { // This file should only contain *development* secrets, never production. let _ = dotenv::from_filename(".env-secrets").ok(); + let common_env = CommonEnv::read_from_env()?; + info!("Obtaining hostname..."); let server_hostname = hostname::get() @@ -178,11 +181,6 @@ async fn main() -> AnyhowResult<()> { "MYSQL_URL", DEFAULT_MYSQL_CONNECTION_STRING); - let redis_connection_string = - easyenv::get_env_string_or_default( - "REDIS_URL", - DEFAULT_REDIS_CONNECTION_STRING); - let pool = MySqlPoolOptions::new() .max_connections(5) .connect(&db_connection_string) @@ -197,7 +195,8 @@ async fn main() -> AnyhowResult<()> { firehose_publisher: firehose_publisher.clone(), // NB: Also safe }; - let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; + let redis_manager = RedisConnectionManager::new( + common_env.redis_0_connection_string.clone())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; @@ -206,7 +205,7 @@ async fn main() -> AnyhowResult<()> { let limiter_max_requests = easyenv::get_env_num("LIMITER_MAX_REQUESTS", 3)?; let limiter_window_seconds = easyenv::get_env_num("LIMITER_WINDOW_SECONDS", 10)?; - let limiter = Limiter::build(&redis_connection_string) + let limiter = Limiter::build(&common_env.redis_0_connection_string) .limit(limiter_max_requests) .period(Duration::from_secs(limiter_window_seconds)) .finish()?; diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index 7379b7ec19..a2fcdf5172 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -161,7 +161,7 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to redis..."); let redis_manager = - RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index b85dd2ee63..fc4f18c1a9 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -227,7 +227,7 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to redis..."); let redis_manager = - RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs index c40627d30b..3504012dc1 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/storyteller_web/src/w2l_download_job.rs @@ -208,7 +208,7 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to redis..."); let redis_manager = - RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs index 794f49660c..c42302626f 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/storyteller_web/src/w2l_inference_job.rs @@ -207,7 +207,7 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to redis..."); let redis_manager = - RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?;