From ed1ce11dc01dd9db6bd75eb659bc5982e6e19574 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Mar 2026 19:05:05 -0400 Subject: [PATCH] add paging to enqueue failures (oncall support) (#1228) It's SRE time! --- Cargo.lock | 2 + .../src/moderation/alerts/mod.rs | 1 + .../alerts/moderation_send_alert.rs | 18 ++++++ .../artcraft_api_defs/src/moderation/mod.rs | 1 + crates/lib/pager/src/client/pager.rs | 27 ++++---- .../get_user_session_by_token.rs | 4 +- crates/service/web/storyteller_web/Cargo.toml | 2 + .../web/storyteller_web/src/api_doc_cli.rs | 1 + .../web/storyteller_web/src/api_doc_server.rs | 1 + .../web/storyteller_web/src/docs/api_doc.rs | 4 ++ ...ce_2p0_multi_function_video_gen_handler.rs | 11 ++++ .../endpoints/moderation/alerts/mod.rs | 1 + .../alerts/moderation_send_alert_handler.rs | 58 ++++++++++++++++++ .../http_server/endpoints/moderation/mod.rs | 1 + .../application_routes/moderation_routes.rs | 7 +++ .../user_session/require_moderator.rs | 2 +- .../service/web/storyteller_web/src/main.rs | 29 ++++++--- .../src/startup/build_pager.rs | 61 +++++++++++++++++++ .../web/storyteller_web/src/startup/mod.rs | 1 + .../storyteller_web/src/state/server_state.rs | 3 + 20 files changed, 210 insertions(+), 25 deletions(-) create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/mod.rs create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs create mode 100644 crates/service/web/storyteller_web/src/startup/build_pager.rs create mode 100644 crates/service/web/storyteller_web/src/startup/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9b2856fa65..f391e2915e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11263,6 +11263,7 @@ dependencies = [ "mimetypes", "mysql_queries", "once_cell", + "pager", "password", "primitives", "prompt_engineering", @@ -11275,6 +11276,7 @@ dependencies = [ "regex", "resend-rs", "reusable_types", + "rootly_client", "rust-s3", "rustls 0.23.36", "seedance2pro_client", diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/mod.rs new file mode 100644 index 0000000000..4970ea6299 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/mod.rs @@ -0,0 +1 @@ +pub mod moderation_send_alert; diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs new file mode 100644 index 0000000000..36831a6da5 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs @@ -0,0 +1,18 @@ +use serde_derive::{Deserialize, Serialize}; +use utoipa::ToSchema; + +pub const MODERATION_SEND_ALERT_PATH: &str = "/v1/moderation/alerts/send"; + +#[derive(Deserialize, ToSchema)] +pub struct ModerationSendAlertRequest { + /// Optional title for the alert. Defaults to "Test Moderation Alert". + pub title: Option, + + /// Optional description for the alert. Defaults to "This is a test moderation alert." + pub description: Option, +} + +#[derive(Serialize, ToSchema)] +pub struct ModerationSendAlertResponse { + pub success: bool, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/mod.rs index 43f96bb232..10ed47c7f6 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/mod.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/mod.rs @@ -1,3 +1,4 @@ +pub mod alerts; pub mod jobs; pub mod user; pub mod wallet_ledger_entries; diff --git a/crates/lib/pager/src/client/pager.rs b/crates/lib/pager/src/client/pager.rs index eefae2bb3b..335f3a749b 100644 --- a/crates/lib/pager/src/client/pager.rs +++ b/crates/lib/pager/src/client/pager.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use log::info; +use log::{info, warn}; use crate::client::pager_client::{PageSentResult, PagerClient}; use crate::error::pager_error::PagerError; @@ -15,20 +15,9 @@ use crate::worker::pager_worker_message_queue::PagerWorkerMessageQueue; /// - **Immediate**: `send_page_immediately()` sends inline (blocks until API responds). /// - **Queued**: `enqueue_page()` pushes to a background worker (non-blocking). /// -/// Build an instance via `PagerBuilder`: -/// ```ignore -/// // Without worker: -/// let pager = PagerBuilder::new() -/// .application_name("my-service".to_string()) -/// .rootly(api_key) -/// .build()?; +/// Build an instance via `PagerBuilder`. /// -/// // With worker: -/// let (pager, worker) = PagerBuilder::new() -/// .application_name("my-service".to_string()) -/// .rootly(api_key) -/// .build_with_worker()?; -/// ``` +#[derive(Clone)] pub struct Pager { client: PagerClient, queue: Option>, @@ -62,6 +51,16 @@ impl Pager { self.client.send_page(¬ification).await } + /// Send a page immediately, blocking until the API responds. + pub async fn send_page_immediately_infallible( + &self, + notification: NotificationDetails, + ) { + if let Err(err) = self.send_page_immediately(notification).await { + warn!("Failure sending page: {:?}", err); + } + } + /// Enqueue a page to be sent by the background worker thread. /// /// Returns an error if the worker is not configured (use `build_with_worker()`). diff --git a/crates/schema/database/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs b/crates/schema/database/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs index 86466fa45e..2934ad1e25 100644 --- a/crates/schema/database/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs +++ b/crates/schema/database/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs @@ -13,6 +13,8 @@ use tokens::tokens::users::UserToken; use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool_default_false, nullable_i8_to_optional_bool}; +const ADMIN_ROLE_SLUG: &str = "admin"; + #[derive(Clone, Serialize, Deserialize)] pub struct SessionUserRecord { pub user_token: UserToken, @@ -90,7 +92,7 @@ impl SessionUserRecord { } pub fn is_mod(&self) -> bool { - self.can_ban_users + self.can_ban_users || self.user_role_slug == ADMIN_ROLE_SLUG } } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 46a5301dd9..f485e216d6 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -65,7 +65,9 @@ memory_caching = { path = "../../../lib/caching/memory_caching" } migration = { path = "../../../schema/database/migration" } mimetypes.workspace = true mysql_queries.workspace = true +pager.workspace = true password = { path = "../../../lib/password" } +rootly_client.workspace = true primitives = { path = "../../../lib/data/primitives" } prompt_engineering = { path = "../../../lib/deprecated/machine_learning/prompt_engineering" } redis_caching = { path = "../../../lib/caching/redis_caching" } diff --git a/crates/service/web/storyteller_web/src/api_doc_cli.rs b/crates/service/web/storyteller_web/src/api_doc_cli.rs index d766aa6c55..5aba19e7b0 100644 --- a/crates/service/web/storyteller_web/src/api_doc_cli.rs +++ b/crates/service/web/storyteller_web/src/api_doc_cli.rs @@ -32,6 +32,7 @@ pub mod docs; pub mod email; pub mod error; pub mod http_server; +pub mod startup; pub mod state; pub mod threads; pub mod util; diff --git a/crates/service/web/storyteller_web/src/api_doc_server.rs b/crates/service/web/storyteller_web/src/api_doc_server.rs index f1e3425f2b..73c0949ad3 100644 --- a/crates/service/web/storyteller_web/src/api_doc_server.rs +++ b/crates/service/web/storyteller_web/src/api_doc_server.rs @@ -34,6 +34,7 @@ pub mod docs; pub mod email; pub mod error; pub mod http_server; +pub mod startup; pub mod state; pub mod threads; pub mod util; diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 8aea0de780..13d568f0a8 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -278,6 +278,7 @@ use artcraft_api_defs::subscriptions::get_session_subscription::*; use artcraft_api_defs::media_file::list_batch_generated_media_files::*; // Handler modules with locally-defined types use crate::http_server::endpoints::moderation::info::moderator_token_info_handler::*; +use artcraft_api_defs::moderation::alerts::moderation_send_alert::*; use artcraft_api_defs::moderation::user::list_subscribing_users_by_signup_date::*; use artcraft_api_defs::moderation::user::list_users_by_signup_date::*; use artcraft_api_defs::moderation::user::user_lookup::*; @@ -467,6 +468,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux // TTS crate::http_server::endpoints::tts::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler, // Moderation + crate::http_server::endpoints::moderation::alerts::moderation_send_alert_handler::moderation_send_alert_handler, crate::http_server::endpoints::moderation::info::moderator_token_info_handler::moderator_get_token_info_handler, crate::http_server::endpoints::moderation::user::moderator_list_subscribing_users_by_signup_date::moderator_list_subscribing_users_by_signup_date_handler, crate::http_server::endpoints::moderation::user::moderator_list_users_by_signup_date::moderator_list_users_by_signup_date_handler, @@ -661,6 +663,8 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux EditUserFeatureFlagsError, EditUserFeatureFlagsOption, EditUserFeatureFlagsRequest, + ModerationSendAlertRequest, + ModerationSendAlertResponse, ModeratorListSubscribingUsersBySignupDateRequest, ModeratorListSubscribingUsersBySignupDateResponse, ModeratorListSubscribingUsersBySignupDateEntry, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs index 22572f787a..f906fa57de 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs @@ -46,6 +46,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; use url::Url; use url_utils::extension::extract_extension_from_url::{extract_extension_from_url, ExtractExtensions}; +use pager::notification::notification_details::NotificationDetails; use crate::http_server::session::lookup::user_session_feature_flags::UserSessionFeatureFlags; // ======================== Result of a successful generation ======================== @@ -219,6 +220,11 @@ pub async fn seedance_2p0_multi_function_video_gen_handler( } Err(err) => { warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err); + if let Err(page_err) = server_state.pager.enqueue_page( + NotificationDetails::from_error_with_context("Seedance 2.0 generation failed (whitelist + fallback)", &err) + ) { + warn!("Failed to enqueue pager alert: {:?}", page_err); + } refund_wallet_after_api_failure(&deduction_result.ledger_entry_token, &mut mysql_connection).await?; return Err(CommonWebError::ServerError); } @@ -245,6 +251,11 @@ pub async fn seedance_2p0_multi_function_video_gen_handler( Ok(result) => result, Err(err) => { warn!("Error calling seedance2pro generate_video: {:?}", err); + if let Err(page_err) = server_state.pager.enqueue_page( + NotificationDetails::from_error_with_context("Seedance 2.0 generation failed", &err) + ) { + warn!("Failed to enqueue pager alert: {:?}", page_err); + } refund_wallet_after_api_failure(&deduction_result.ledger_entry_token, &mut mysql_connection).await?; return Err(CommonWebError::ServerError); } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/mod.rs new file mode 100644 index 0000000000..9524f92550 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/mod.rs @@ -0,0 +1 @@ +pub mod moderation_send_alert_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs new file mode 100644 index 0000000000..77f5e60d56 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs @@ -0,0 +1,58 @@ +use std::sync::Arc; + +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::warn; + +use artcraft_api_defs::moderation::alerts::moderation_send_alert::{ + ModerationSendAlertRequest, + ModerationSendAlertResponse, +}; +use pager::client::pager::Pager; +use pager::notification::notification_details::NotificationDetails; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::web_utils::user_session::require_moderator::{require_moderator, UseDatabase}; +use crate::state::server_state::ServerState; + +/// Send a test alert via the pager system. Moderators only. +#[utoipa::path( + post, + tag = "Moderation", + path = "/v1/moderation/alerts/send", + request_body = ModerationSendAlertRequest, + responses( + (status = 200, description = "Success", body = ModerationSendAlertResponse), + (status = 401, description = "Unauthorized"), + (status = 500, description = "Server error"), + ), +)] +pub async fn moderation_send_alert_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data>, +) -> Result, CommonWebError> { + + let _user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection) + .await + .map_err(|_| CommonWebError::NotAuthorized)?; + + let title = request.title.clone() + .unwrap_or_else(|| "Test Moderation Alert".to_string()); + + let description = request.description.clone() + .unwrap_or_else(|| "This is a test moderation alert.".to_string()); + + let notification = NotificationDetails::with_summary_and_description(title, description); + + server_state.pager + .enqueue_page(notification) + .map_err(|err| { + warn!("moderation_send_alert error: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(ModerationSendAlertResponse { + success: true, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/mod.rs index 312833231b..784d6f2da0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/mod.rs @@ -1,3 +1,4 @@ +pub mod alerts; pub mod info; pub mod ip_bans; pub mod jobs; diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/moderation_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/moderation_routes.rs index 750618bf76..9617cabe44 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/moderation_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/moderation_routes.rs @@ -35,6 +35,7 @@ use crate::http_server::endpoints::moderation::user::moderator_list_subscribing_ use crate::http_server::endpoints::moderation::user::moderator_list_users_by_signup_date::moderator_list_users_by_signup_date_handler; use crate::http_server::endpoints::moderation::user::moderator_user_lookup_by_stripe_customer_id_handler::moderator_user_lookup_by_stripe_customer_id_handler; use crate::http_server::endpoints::moderation::user::moderator_user_lookup_handler::moderator_user_lookup_handler; +use crate::http_server::endpoints::moderation::alerts::moderation_send_alert_handler::moderation_send_alert_handler; use crate::http_server::endpoints::moderation::user_feature_flags::edit_user_feature_flags_handler::edit_user_feature_flags_handler; pub fn add_moderator_routes (app: App) -> App @@ -49,6 +50,12 @@ pub fn add_moderator_routes (app: App) -> App >, { app.service(web::scope("/v1/moderation") + .service(web::scope("/alerts") + .service(web::resource("/send") + .route(web::post().to(moderation_send_alert_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(web::resource("/user_feature_flags/{username_or_token}") .route(web::post().to(edit_user_feature_flags_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/crates/service/web/storyteller_web/src/http_server/web_utils/user_session/require_moderator.rs b/crates/service/web/storyteller_web/src/http_server/web_utils/user_session/require_moderator.rs index 51a8beaf93..7605361929 100644 --- a/crates/service/web/storyteller_web/src/http_server/web_utils/user_session/require_moderator.rs +++ b/crates/service/web/storyteller_web/src/http_server/web_utils/user_session/require_moderator.rs @@ -75,7 +75,7 @@ pub async fn require_moderator( } }; - if !user_session.can_ban_users { + if !user_session.is_mod() { warn!("user is not a moderator: {:?}", user_session.user_token.as_str()); return Err(RequireModeratorError::NotAuthorized); } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 19522e9314..57eef61ace 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -52,7 +52,7 @@ use elasticsearch::http::transport::Transport; use elasticsearch::Elasticsearch; use errors::AnyhowResult; use fal_client::creds::fal_api_key::FalApiKey; -use log::info; +use log::{info, warn}; use memory_caching::arc_ttl_sieve::ArcTtlSieve; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; @@ -79,6 +79,7 @@ use crate::http_server::web_utils::handle_multipart_error::handle_multipart_erro use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator; use crate::state::certs::google_sign_in_cert::GoogleSignInCert; use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; +use crate::startup::build_pager::build_pager; use crate::state::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, WorldLabsData}; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; @@ -93,6 +94,7 @@ pub mod configs; pub mod email; pub mod error; pub mod http_server; +pub mod startup; pub mod state; pub mod threads; pub mod util; @@ -328,6 +330,21 @@ async fn main() -> AnyhowResult<()> { poll_model_token_info_thread(model_token_info_cache2, mysql_pool5).await; }); + let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) + .ok_or(anyhow!("invalid server environment"))?; + + let server_environment_typed = match server_environment { + ServerEnvironment::Production => server_environment::ServerEnvironment::Production, + ServerEnvironment::Development => server_environment::ServerEnvironment::Development, + }; + + let (pager, pager_worker) = build_pager(server_environment_typed); + + info!("Spawning pager worker thread."); + + tokio_runtime.spawn(async move { + pager_worker.run().await; + }); let stripe_configs = StripeConfig { checkout: StripeCheckoutConfigs { @@ -345,9 +362,6 @@ async fn main() -> AnyhowResult<()> { }, }; - let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) - .ok_or(anyhow!("invalid server environment"))?; - let service_feature_flags = StaticFeatureFlags { // Permanent (control plane / safety) flags : messaging maybe_status_alert_category: easyenv::get_env_string_optional("FF_STATUS_ALERT_CATEGORY"), @@ -391,11 +405,6 @@ async fn main() -> AnyhowResult<()> { .unwrap_or(String::from("unknown")) .trim() .to_string(); - - let server_environment_typed = match server_environment { - ServerEnvironment::Production => server_environment::ServerEnvironment::Production, - ServerEnvironment::Development => server_environment::ServerEnvironment::Development, - }; let fal_api_key = FalApiKey::new(easyenv::get_env_string_required("FAL_API_KEY")?); let fal_webhook_url = easyenv::get_env_string_required("FAL_WEBHOOK_URL")?; @@ -473,6 +482,7 @@ async fn main() -> AnyhowResult<()> { worldlabs: WorldLabsData { api_key: worldlabs_api_key, }, + pager, caches: InMemoryCaches { durable: DurableInMemoryCaches { model_token_info: model_token_info_cache, @@ -652,6 +662,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .app_data(web::Data::new(server_state_arc.stripe.clone().client.clone())) .app_data(web::Data::new(server_state_arc.third_party_url_redirector)) .app_data(web::Data::new(server_state_arc.google_sign_in_cert.clone())) + .app_data(web::Data::new(server_state_arc.pager.clone())) .app_data(web::Data::new(old_server_environment)) .app_data(web::Data::new(new_server_environment)) .app_data(web::Data::from(product_lookup)) // NB: Data::from(Arc) for dynamic dispatch diff --git a/crates/service/web/storyteller_web/src/startup/build_pager.rs b/crates/service/web/storyteller_web/src/startup/build_pager.rs new file mode 100644 index 0000000000..cc1bfcbd0e --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/build_pager.rs @@ -0,0 +1,61 @@ +use log::{info, warn}; +use pager::client::pager::Pager; +use pager::client::pager_builder::PagerBuilder; +use pager::worker::pager_worker::PagerWorker; +use rootly_client::creds::rootly_api_key::RootlyApiKey; + +pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> (Pager, PagerWorker) { + let environment = if server_environment.is_deployed_in_production() { + "production" + } else { + "development" + }; + + let maybe_api_key = easyenv::get_env_string_optional("ROOTLY_API_KEY"); + + match maybe_api_key { + Some(api_key) => { + info!("Rootly API key found. Configuring pager with Rootly backend."); + build_rootly_pager(environment, api_key) + } + None => { + warn!("ROOTLY_API_KEY not set. Pager will not send real pages."); + build_fallback_pager(environment) + } + } +} + +fn build_rootly_pager(environment: &str, api_key: String) -> (Pager, PagerWorker) { + let mut rootly_builder = PagerBuilder::new() + .application_name("storyteller-web".to_string()) + .environment(environment.to_string()) + .rootly(RootlyApiKey::new(api_key)); + + if let Some(urgency_id) = easyenv::get_env_string_optional("ROOTLY_ALERT_URGENCY_ID") { + rootly_builder = rootly_builder.alert_urgency_id(urgency_id); + } + + let target_type = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_TYPE"); + let target_id = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_ID"); + + if let (Some(t_type), Some(t_id)) = (target_type, target_id) { + rootly_builder = rootly_builder.notification_target(t_type, t_id); + } + + match rootly_builder.build_with_worker() { + Ok((pager, worker)) => (pager, worker), + Err(err) => { + warn!("Failed to build pager with Rootly backend: {}. Pages will not be sent.", err); + build_fallback_pager(environment) + } + } +} + +fn build_fallback_pager(environment: &str) -> (Pager, PagerWorker) { + PagerBuilder::new() + .application_name("storyteller-web".to_string()) + .environment(environment.to_string()) + .rootly(RootlyApiKey::new(String::new())) + .build_with_worker() + .expect("fallback pager build should not fail") +} diff --git a/crates/service/web/storyteller_web/src/startup/mod.rs b/crates/service/web/storyteller_web/src/startup/mod.rs new file mode 100644 index 0000000000..d074c6c0ce --- /dev/null +++ b/crates/service/web/storyteller_web/src/startup/mod.rs @@ -0,0 +1 @@ +pub mod build_pager; diff --git a/crates/service/web/storyteller_web/src/state/server_state.rs b/crates/service/web/storyteller_web/src/state/server_state.rs index 278f122d99..1bf0a4b452 100644 --- a/crates/service/web/storyteller_web/src/state/server_state.rs +++ b/crates/service/web/storyteller_web/src/state/server_state.rs @@ -9,6 +9,7 @@ use elasticsearch::Elasticsearch; use fal_client::creds::fal_api_key::FalApiKey; use memory_caching::arc_ttl_sieve::ArcTtlSieve; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use pager::client::pager::Pager; use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult; @@ -100,6 +101,8 @@ pub struct ServerState { pub worldlabs: WorldLabsData, + pub pager: Pager, + /// Where to store audio uploads for w2l pub audio_uploads_bucket_root: String,