diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs index 559f479818..f5f66e64b2 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs @@ -7,6 +7,7 @@ use actix_web::Error; use pager::client::pager::Pager; use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::state::flags::paging_flags::PagingFlags; use super::check_common_web_error::check_common_web_error; use super::check_status_code_fallback::check_status_code_fallback; @@ -18,16 +19,20 @@ use super::check_status_code_fallback::check_status_code_fallback; /// 1. **Typed error matching** via downcast (e.g. `CommonWebError::ServerError`) /// 2. **Status code fallback** for untyped 500s that slip through /// +/// Both `PagingFlags.is_paging_enabled` and `PagingFlags.is_paging_for_500s_enabled` +/// must be true for any alerting to fire. Otherwise the middleware is a passthrough. +/// /// To add new error types, create a new `check_*.rs` matcher module and /// add a downcast branch in `check_ok_response_for_alerts` / `check_err_for_alerts`. #[derive(Clone)] pub struct ErrorAlertingMiddleware { pager: Pager, + paging_flags: PagingFlags, } impl ErrorAlertingMiddleware { - pub fn new(pager: Pager) -> Self { - Self { pager } + pub fn new(pager: Pager, paging_flags: PagingFlags) -> Self { + Self { pager, paging_flags } } } @@ -47,6 +52,7 @@ impl Transform for ErrorAlertingMiddleware ready(Ok(ErrorAlertingService { service, pager: self.pager.clone(), + paging_flags: self.paging_flags.clone(), })) } } @@ -56,6 +62,7 @@ impl Transform for ErrorAlertingMiddleware pub struct ErrorAlertingService { service: S, pager: Pager, + paging_flags: PagingFlags, } impl Service for ErrorAlertingService @@ -71,6 +78,11 @@ impl Service for ErrorAlertingService actix_service::forward_ready!(service); fn call(&self, req: ServiceRequest) -> Self::Future { + // Fast path: if paging is disabled, skip all inspection. + if !self.paging_flags.is_paging_enabled || !self.paging_flags.is_paging_for_500s_enabled { + return Box::pin(self.service.call(req)); + } + let method = req.method().to_string(); let path = req.path().to_string(); let pager = self.pager.clone(); diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 3ead8f9828..6851cb9803 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -339,7 +339,7 @@ async fn main() -> AnyhowResult<()> { ServerEnvironment::Development => server_environment::ServerEnvironment::Development, }; - let (pager, pager_worker) = build_pager(server_environment_typed); + let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed); info!("Spawning pager worker thread."); @@ -392,6 +392,9 @@ async fn main() -> AnyhowResult<()> { // Disable voice features disable_tts: easyenv::get_env_bool_or_default("DISABLE_TTS", false), disable_voice_conversion: easyenv::get_env_bool_or_default("DISABLE_VOICE_CONVERSION", false), + + // Paging + paging: paging_flags, }; let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment); @@ -626,11 +629,13 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> ServerEnvironment::Production => server_environment::ServerEnvironment::Production, }; - let enable_error_alerting = easyenv::get_env_bool_or_default("ENABLE_ERROR_ALERTING_MIDDLEWARE", false); + let paging_flags_for_middleware = server_state.flags.paging.clone(); let pager_for_middleware = server_state.pager.clone(); + let enable_error_alerting = paging_flags_for_middleware.is_paging_enabled + && paging_flags_for_middleware.is_paging_for_500s_enabled; if enable_error_alerting { - info!("Error alerting middleware is ENABLED."); + info!("Error alerting middleware is ENABLED (ENABLE_PAGING=true, ENABLE_PAGING_FOR_500S=true)."); } let server_state_arc = web::Data::new(Arc::new(server_state)); @@ -692,7 +697,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .header("X-Build-Sha", server_state_arc.server_info.build_sha.clone())) .wrap(middleware::Condition::new( enable_error_alerting, - ErrorAlertingMiddleware::new(pager_for_middleware.clone()), + ErrorAlertingMiddleware::new(pager_for_middleware.clone(), paging_flags_for_middleware.clone()), )) .wrap(PushbackFilter::new(&server_state_arc.flags.clone())) .wrap(DisabledEndpointFilter::new(disabled_endpoints.clone())) diff --git a/crates/service/web/storyteller_web/src/startup/build_pager.rs b/crates/service/web/storyteller_web/src/startup/build_pager.rs index 33db0f42a2..47e423f17b 100644 --- a/crates/service/web/storyteller_web/src/startup/build_pager.rs +++ b/crates/service/web/storyteller_web/src/startup/build_pager.rs @@ -4,20 +4,38 @@ use pager::client::pager_builder::PagerBuilder; use pager::worker::pager_worker::PagerWorker; use rootly_client::creds::rootly_api_key::RootlyApiKey; -pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> (Pager, PagerWorker) { +use crate::state::flags::paging_flags::PagingFlags; + +pub fn build_pager( + server_environment: server_environment::ServerEnvironment, +) -> (Pager, PagerWorker, PagingFlags) { + let paging_flags = PagingFlags::from_env(); + + info!("Paging flags: {:?}", paging_flags); + let environment = if server_environment.is_deployed_in_production() { "production" } else { "development" }; + // If paging is globally disabled, use a NoOp pager regardless of API key. + if !paging_flags.is_paging_enabled { + warn!("ENABLE_PAGING is false. Pager will be NoOp."); + let (pager, worker) = PagerBuilder::new() + .application_name("storyteller-web".to_string()) + .environment(environment.to_string()) + .build_with_worker(); + return (pager, worker, paging_flags); + } + let maybe_api_key = easyenv::get_env_string_optional("ROOTLY_API_KEY"); let builder = PagerBuilder::new() .application_name("storyteller-web".to_string()) .environment(environment.to_string()); - match maybe_api_key { + let (pager, worker) = match maybe_api_key { Some(api_key) => { info!("Rootly API key found. Configuring pager with Rootly backend."); build_rootly_pager(builder, api_key) @@ -26,10 +44,12 @@ pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> warn!("ROOTLY_API_KEY not set. Pager will not send real pages."); builder.build_with_worker() } - } + }; + + (pager, worker, paging_flags) } -fn build_rootly_pager(mut builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) { +fn build_rootly_pager(builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) { let mut rootly_builder = builder .rootly(RootlyApiKey::new(api_key)); diff --git a/crates/service/web/storyteller_web/src/state/flags/mod.rs b/crates/service/web/storyteller_web/src/state/flags/mod.rs new file mode 100644 index 0000000000..8c71e814be --- /dev/null +++ b/crates/service/web/storyteller_web/src/state/flags/mod.rs @@ -0,0 +1 @@ +pub mod paging_flags; diff --git a/crates/service/web/storyteller_web/src/state/flags/paging_flags.rs b/crates/service/web/storyteller_web/src/state/flags/paging_flags.rs new file mode 100644 index 0000000000..ff901853ed --- /dev/null +++ b/crates/service/web/storyteller_web/src/state/flags/paging_flags.rs @@ -0,0 +1,21 @@ +/// Flags controlling the pager system. +#[derive(Clone, Debug)] +pub struct PagingFlags { + /// Master switch for paging. If false, no pages are sent. + /// Env var: ENABLE_PAGING (default: false) + pub is_paging_enabled: bool, + + /// If true, the error alerting middleware will enqueue pages for HTTP 500s. + /// Requires `is_paging_enabled` to also be true. + /// Env var: ENABLE_PAGING_FOR_500S (default: false) + pub is_paging_for_500s_enabled: bool, +} + +impl PagingFlags { + pub fn from_env() -> Self { + Self { + is_paging_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING", false), + is_paging_for_500s_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING_FOR_500S", false), + } + } +} diff --git a/crates/service/web/storyteller_web/src/state/mod.rs b/crates/service/web/storyteller_web/src/state/mod.rs index ec90ef7a12..ac27af42c1 100644 --- a/crates/service/web/storyteller_web/src/state/mod.rs +++ b/crates/service/web/storyteller_web/src/state/mod.rs @@ -1,4 +1,5 @@ pub mod cached_queries; +pub mod flags; pub mod certs; pub mod memory_cache; pub mod server_state; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/state/server_state.rs b/crates/service/web/storyteller_web/src/state/server_state.rs index 1bf0a4b452..0bf8e52377 100644 --- a/crates/service/web/storyteller_web/src/state/server_state.rs +++ b/crates/service/web/storyteller_web/src/state/server_state.rs @@ -33,6 +33,7 @@ use crate::http_server::endpoints::stats::result_transformer::CacheableQueueStat use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse; use crate::http_server::endpoints::voice_conversion::list_voice_conversion_models_handler::VoiceConversionModel; use crate::http_server::session::session_checker::SessionChecker; +use crate::state::flags::paging_flags::PagingFlags; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator; use crate::state::certs::google_sign_in_cert::GoogleSignInCert; @@ -298,6 +299,9 @@ pub struct StaticFeatureFlags { /// Disable voice conversion endpoints /// If true, respond with 429. pub disable_voice_conversion: bool, + + /// Paging system flags. + pub paging: PagingFlags, } /// Instead of top level service denial, these are bans against entities that instead return