Two Redis indices. Twitch/OBS will be on n=1

This will require updating deployment yamls.

crates/lib/config/src/shared_constants.rs

@@ -7,7 +7,11 @@ pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=war
 //const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // but sometimes we want to debug
 
 /// The default Redis connection string for use in development
-pub const DEFAULT_REDIS_CONNECTION_STRING: &'static str = "redis://localhost";
+pub const DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING: &'static str = "redis://localhost/0";
+
+/// The default Redis connection string for use in development
+/// Database 1
+pub const DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING: &'static str = "redis://localhost/1";
 
 /// The default MySql connection string for use in development
 pub const DEFAULT_MYSQL_CONNECTION_STRING: &'static str = "mysql://storyteller:password@localhost/storyteller";

crates/service/obs_gateway/src/main.rs

@@ -23,7 +23,7 @@ use actix_http::http;
 use actix_web::middleware::{Logger, DefaultHeaders};
 use actix_web::{HttpServer, web, HttpResponse, App};
 use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
+use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
 use crate::endpoints::oauth_begin::oauth_begin_enroll;
 use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect;
@@ -187,11 +187,6 @@ async fn main() -> AnyhowResult<()> {
 //        "MYSQL_URL",
 //        DEFAULT_MYSQL_CONNECTION_STRING);
 //
-//  let redis_connection_string =
-//      easyenv::get_env_string_or_default(
-//        "REDIS_URL",
-//        DEFAULT_REDIS_CONNECTION_STRING);
-//
 //  let pool = MySqlPoolOptions::new()
 //      .max_connections(5)
 //      .connect(&db_connection_string)
@@ -227,16 +222,29 @@ async fn main() -> AnyhowResult<()> {
         "MYSQL_URL",
         DEFAULT_MYSQL_CONNECTION_STRING);
 
-  //let redis_connection_string =
-  //    easyenv::get_env_string_or_default(
-  //      "REDIS_URL",
-  //      DEFAULT_REDIS_CONNECTION_STRING);
+  let redis_connection_string =
+      easyenv::get_env_string_or_default(
+        "REDIS_1_URL",
+        DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING);
+
+  info!("Connecting to mysql...");
 
   let pool = MySqlPoolOptions::new()
       .max_connections(5)
       .connect(&db_connection_string)
       .await?;
 
+  info!("Connecting to redis...");
+
+  let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?;
+
+  let redis_pool = r2d2::Pool::builder()
+      .build(redis_manager)?;
+
+  // NB: Compiler can't figure out throwaway return type
+  let key = "hi-database-1".to_string();
+  let _r : String = redis_pool.get().unwrap().set(&key, "bar").unwrap();
+
   let server_state = ObsGatewayServerState {
     env_config: EnvConfig {
       num_workers,
@@ -259,6 +267,7 @@ async fn main() -> AnyhowResult<()> {
     hostname: server_hostname,
     backends: BackendsConfig {
       mysql_pool: pool,
+      redis_pool,
     }
   };
 

crates/service/obs_gateway/src/server_state.rs

@@ -1,3 +1,4 @@
+use r2d2_redis::{r2d2, RedisConnectionManager};
 use sqlx::MySqlPool;
 use twitch_oauth2::{ClientId, ClientSecret};
 
@@ -18,6 +19,7 @@ pub struct ObsGatewayServerState {
 #[derive(Clone)]
 pub struct BackendsConfig {
   pub mysql_pool: MySqlPool,
+  pub redis_pool: r2d2::Pool<RedisConnectionManager>,
 }
 
 // TODO: Many of these do not need to be passed around past server init.

crates/service/storyteller_web/src/common_env.rs

@@ -1,4 +1,5 @@
-use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
+use config::shared_constants::DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING;
+use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING;
 use container_common::anyhow_result::AnyhowResult;
 
 // TODO: Move more shared configs here.
@@ -9,8 +10,12 @@ use container_common::anyhow_result::AnyhowResult;
 pub struct CommonEnv {
 
   /// The Redis connection string.
-  /// Currently we only have a single Redis that serves all purposes.
-  pub redis_connection_string: String,
+  /// This has FakeYou voice stats, rate limiters, etc.
+  pub redis_0_connection_string: String,
+
+  /// The Redis connection string.
+  /// This has OBS/Twitch related functionality
+  pub redis_1_connection_string: String,
 
   /// The amount of time to wait between job batches (not individual jobs).
   /// This prevents the outer loop of querying batches from flooding the DB.
@@ -37,8 +42,10 @@ impl CommonEnv {
 
   pub fn read_from_env() -> AnyhowResult<Self> {
     Ok(Self {
-      redis_connection_string: easyenv::get_env_string_or_default("REDIS_URL",
-        DEFAULT_REDIS_CONNECTION_STRING),
+      redis_0_connection_string: easyenv::get_env_string_or_default("REDIS_0_URL",
+        DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING),
+      redis_1_connection_string: easyenv::get_env_string_or_default("REDIS_1_URL",
+                                                                  DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING),
       job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?,
       job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?,
       job_batch_size: easyenv::get_env_num("JOB_BATCH_SIZE", 10)?,

crates/service/storyteller_web/src/main.rs

@@ -17,6 +17,7 @@ pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/rese
 pub const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt");
 
 pub mod database;
+pub mod common_env;
 pub mod http_clients;
 pub mod http_server;
 pub mod routes;
@@ -40,9 +41,9 @@ use actix_http::http;
 use actix_web::middleware::{Logger, DefaultHeaders};
 use actix_web::{HttpServer, web, HttpResponse, App};
 use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
 use container_common::anyhow_result::AnyhowResult;
+use crate::common_env::CommonEnv;
 use crate::database::mediators::badge_granter::BadgeGranter;
 use crate::database::mediators::firehose_publisher::FirehosePublisher;
 use crate::http_server::endpoints::events::list_events::list_events_handler;
@@ -162,6 +163,8 @@ async fn main() -> AnyhowResult<()> {
   // This file should only contain *development* secrets, never production.
   let _ = dotenv::from_filename(".env-secrets").ok();
 
+  let common_env = CommonEnv::read_from_env()?;
+
   info!("Obtaining hostname...");
 
   let server_hostname = hostname::get()
@@ -178,11 +181,6 @@ async fn main() -> AnyhowResult<()> {
       "MYSQL_URL",
       DEFAULT_MYSQL_CONNECTION_STRING);
 
-  let redis_connection_string =
-      easyenv::get_env_string_or_default(
-        "REDIS_URL",
-        DEFAULT_REDIS_CONNECTION_STRING);
-
   let pool = MySqlPoolOptions::new()
     .max_connections(5)
     .connect(&db_connection_string)
@@ -197,7 +195,8 @@ async fn main() -> AnyhowResult<()> {
     firehose_publisher: firehose_publisher.clone(), // NB: Also safe
   };
 
-  let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?;
+  let redis_manager = RedisConnectionManager::new(
+    common_env.redis_0_connection_string.clone())?;
 
   let redis_pool = r2d2::Pool::builder()
       .build(redis_manager)?;
@@ -206,7 +205,7 @@ async fn main() -> AnyhowResult<()> {
   let limiter_max_requests = easyenv::get_env_num("LIMITER_MAX_REQUESTS", 3)?;
   let limiter_window_seconds = easyenv::get_env_num("LIMITER_WINDOW_SECONDS", 10)?;
 
-  let limiter = Limiter::build(&redis_connection_string)
+  let limiter = Limiter::build(&common_env.redis_0_connection_string)
       .limit(limiter_max_requests)
       .period(Duration::from_secs(limiter_window_seconds))
       .finish()?;

crates/service/storyteller_web/src/tts_download_job.rs

@@ -161,7 +161,7 @@ async fn main() -> AnyhowResult<()> {
   info!("Connecting to redis...");
 
   let redis_manager =
-      RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
+      RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?;
 
   let redis_pool = r2d2::Pool::builder()
       .build(redis_manager)?;

crates/service/storyteller_web/src/tts_inference_job.rs

@@ -227,7 +227,7 @@ async fn main() -> AnyhowResult<()> {
   info!("Connecting to redis...");
 
   let redis_manager =
-      RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
+      RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?;
 
   let redis_pool = r2d2::Pool::builder()
       .build(redis_manager)?;

crates/service/storyteller_web/src/w2l_download_job.rs

@@ -208,7 +208,7 @@ async fn main() -> AnyhowResult<()> {
   info!("Connecting to redis...");
 
   let redis_manager =
-      RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
+      RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?;
 
   let redis_pool = r2d2::Pool::builder()
       .build(redis_manager)?;

crates/service/storyteller_web/src/w2l_inference_job.rs

@@ -207,7 +207,7 @@ async fn main() -> AnyhowResult<()> {
   info!("Connecting to redis...");
 
   let redis_manager =
-      RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
+      RedisConnectionManager::new(common_env.redis_0_connection_string.deref())?;
 
   let redis_pool = r2d2::Pool::builder()
       .build(redis_manager)?;