nice control flags

This commit is contained in:
Brandon Thomas committed 2026-03-25 23:35:00 -04:00
1 parent 14ef9cb2b8
commit bf02bf7bbc
7 files changed
+74 -10

No files matched your search

@@ -7,6 +7,7 @@ use actix_web::Error;
use pager::client::pager::Pager;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::state::flags::paging_flags::PagingFlags;
use super::check_common_web_error::check_common_web_error;
use super::check_status_code_fallback::check_status_code_fallback;
@@ -18,16 +19,20 @@ use super::check_status_code_fallback::check_status_code_fallback;
/// 1. **Typed error matching** via downcast (e.g. `CommonWebError::ServerError`)
/// 2. **Status code fallback** for untyped 500s that slip through
///
/// Both `PagingFlags.is_paging_enabled` and `PagingFlags.is_paging_for_500s_enabled`
/// must be true for any alerting to fire. Otherwise the middleware is a passthrough.
///
/// To add new error types, create a new `check_*.rs` matcher module and
/// add a downcast branch in `check_ok_response_for_alerts` / `check_err_for_alerts`.
#[derive(Clone)]
pub struct ErrorAlertingMiddleware {
pager: Pager,
paging_flags: PagingFlags,
}
impl ErrorAlertingMiddleware {
pub fn new(pager: Pager) -> Self {
Self { pager }
pub fn new(pager: Pager, paging_flags: PagingFlags) -> Self {
Self { pager, paging_flags }
}
}
@@ -47,6 +52,7 @@ impl<S, B> Transform<S, ServiceRequest> for ErrorAlertingMiddleware
ready(Ok(ErrorAlertingService {
service,
pager: self.pager.clone(),
paging_flags: self.paging_flags.clone(),
}))
}
}
@@ -56,6 +62,7 @@ impl<S, B> Transform<S, ServiceRequest> for ErrorAlertingMiddleware
pub struct ErrorAlertingService<S> {
service: S,
pager: Pager,
paging_flags: PagingFlags,
}
impl<S, B> Service<ServiceRequest> for ErrorAlertingService<S>
@@ -71,6 +78,11 @@ impl<S, B> Service<ServiceRequest> for ErrorAlertingService<S>
actix_service::forward_ready!(service);
fn call(&self, req: ServiceRequest) -> Self::Future {
// Fast path: if paging is disabled, skip all inspection.
if !self.paging_flags.is_paging_enabled || !self.paging_flags.is_paging_for_500s_enabled {
return Box::pin(self.service.call(req));
}
let method = req.method().to_string();
let path = req.path().to_string();
let pager = self.pager.clone();
@@ -339,7 +339,7 @@ async fn main() -> AnyhowResult<()> {
ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
};
let (pager, pager_worker) = build_pager(server_environment_typed);
let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed);
info!("Spawning pager worker thread.");
@@ -392,6 +392,9 @@ async fn main() -> AnyhowResult<()> {
// Disable voice features
disable_tts: easyenv::get_env_bool_or_default("DISABLE_TTS", false),
disable_voice_conversion: easyenv::get_env_bool_or_default("DISABLE_VOICE_CONVERSION", false),
// Paging
paging: paging_flags,
};
let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment);
@@ -626,11 +629,13 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
};
let enable_error_alerting = easyenv::get_env_bool_or_default("ENABLE_ERROR_ALERTING_MIDDLEWARE", false);
let paging_flags_for_middleware = server_state.flags.paging.clone();
let pager_for_middleware = server_state.pager.clone();
let enable_error_alerting = paging_flags_for_middleware.is_paging_enabled
&& paging_flags_for_middleware.is_paging_for_500s_enabled;
if enable_error_alerting {
info!("Error alerting middleware is ENABLED.");
info!("Error alerting middleware is ENABLED (ENABLE_PAGING=true, ENABLE_PAGING_FOR_500S=true).");
}
let server_state_arc = web::Data::new(Arc::new(server_state));
@@ -692,7 +697,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
.header("X-Build-Sha", server_state_arc.server_info.build_sha.clone()))
.wrap(middleware::Condition::new(
enable_error_alerting,
ErrorAlertingMiddleware::new(pager_for_middleware.clone()),
ErrorAlertingMiddleware::new(pager_for_middleware.clone(), paging_flags_for_middleware.clone()),
))
.wrap(PushbackFilter::new(&server_state_arc.flags.clone()))
.wrap(DisabledEndpointFilter::new(disabled_endpoints.clone()))
@@ -4,20 +4,38 @@ use pager::client::pager_builder::PagerBuilder;
use pager::worker::pager_worker::PagerWorker;
use rootly_client::creds::rootly_api_key::RootlyApiKey;
pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> (Pager, PagerWorker) {
use crate::state::flags::paging_flags::PagingFlags;
pub fn build_pager(
server_environment: server_environment::ServerEnvironment,
) -> (Pager, PagerWorker, PagingFlags) {
let paging_flags = PagingFlags::from_env();
info!("Paging flags: {:?}", paging_flags);
let environment = if server_environment.is_deployed_in_production() {
"production"
} else {
"development"
};
// If paging is globally disabled, use a NoOp pager regardless of API key.
if !paging_flags.is_paging_enabled {
warn!("ENABLE_PAGING is false. Pager will be NoOp.");
let (pager, worker) = PagerBuilder::new()
.application_name("storyteller-web".to_string())
.environment(environment.to_string())
.build_with_worker();
return (pager, worker, paging_flags);
}
let maybe_api_key = easyenv::get_env_string_optional("ROOTLY_API_KEY");
let builder = PagerBuilder::new()
.application_name("storyteller-web".to_string())
.environment(environment.to_string());
match maybe_api_key {
let (pager, worker) = match maybe_api_key {
Some(api_key) => {
info!("Rootly API key found. Configuring pager with Rootly backend.");
build_rootly_pager(builder, api_key)
@@ -26,10 +44,12 @@ pub fn build_pager(server_environment: server_environment::ServerEnvironment) ->
warn!("ROOTLY_API_KEY not set. Pager will not send real pages.");
builder.build_with_worker()
}
}
};
(pager, worker, paging_flags)
}
fn build_rootly_pager(mut builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) {
fn build_rootly_pager(builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) {
let mut rootly_builder = builder
.rootly(RootlyApiKey::new(api_key));
@@ -0,0 +1 @@
pub mod paging_flags;
@@ -0,0 +1,21 @@
/// Flags controlling the pager system.
#[derive(Clone, Debug)]
pub struct PagingFlags {
/// Master switch for paging. If false, no pages are sent.
/// Env var: ENABLE_PAGING (default: false)
pub is_paging_enabled: bool,
/// If true, the error alerting middleware will enqueue pages for HTTP 500s.
/// Requires `is_paging_enabled` to also be true.
/// Env var: ENABLE_PAGING_FOR_500S (default: false)
pub is_paging_for_500s_enabled: bool,
}
impl PagingFlags {
pub fn from_env() -> Self {
Self {
is_paging_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING", false),
is_paging_for_500s_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING_FOR_500S", false),
}
}
}
@@ -1,4 +1,5 @@
pub mod cached_queries;
pub mod flags;
pub mod certs;
pub mod memory_cache;
pub mod server_state;
@@ -33,6 +33,7 @@ use crate::http_server::endpoints::stats::result_transformer::CacheableQueueStat
use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse;
use crate::http_server::endpoints::voice_conversion::list_voice_conversion_models_handler::VoiceConversionModel;
use crate::http_server::session::session_checker::SessionChecker;
use crate::state::flags::paging_flags::PagingFlags;
use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator;
use crate::state::certs::google_sign_in_cert::GoogleSignInCert;
@@ -298,6 +299,9 @@ pub struct StaticFeatureFlags {
/// Disable voice conversion endpoints
/// If true, respond with 429.
pub disable_voice_conversion: bool,
/// Paging system flags.
pub paging: PagingFlags,
}
/// Instead of top level service denial, these are bans against entities that instead return