From 7e2d7048b14fde2bd7842584203e199e2a819f9e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 31 Mar 2026 01:22:28 -0400 Subject: [PATCH] health check page and health check thread page, mod pages have message + priority (#1262) --- .../alerts/moderation_send_alert.rs | 11 ++++++ crates/lib/pager/src/client/pager.rs | 4 +- crates/lib/pager/src/client/pager_client.rs | 6 +-- .../generate_deduplication_key.rs | 14 +++---- .../src/notification/notification_details.rs | 34 ++++++++-------- .../notification_details_builder.rs | 14 +++---- crates/lib/pager/src/worker/pager_worker.rs | 6 +-- .../src/worker/pager_worker_message_queue.rs | 4 +- .../web/storyteller_web/src/docs/api_doc.rs | 1 + ...ce_2p0_multi_function_video_gen_handler.rs | 4 +- .../alerts/moderation_send_alert_handler.rs | 39 ++++++++++++++----- .../endpoints/service/health_check_handler.rs | 29 ++++++++++++++ .../check_common_web_error.rs | 2 +- .../check_status_code_fallback.rs | 2 +- .../service/web/storyteller_web/src/main.rs | 35 +++++++++-------- .../db_health_checker_thread.rs | 18 ++++++++- 16 files changed, 150 insertions(+), 73 deletions(-) diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs index 36831a6da5..ca5255878a 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs @@ -10,6 +10,17 @@ pub struct ModerationSendAlertRequest { /// Optional description for the alert. Defaults to "This is a test moderation alert." pub description: Option, + + /// Optional urgency of the page. + pub urgency: Option, +} + +#[derive(Deserialize, ToSchema, Copy, Clone, Debug)] +#[serde(rename_all = "snake_case")] +pub enum ModerationSendAlertUrgency { + High, + Medium, + Low, } #[derive(Serialize, ToSchema)] diff --git a/crates/lib/pager/src/client/pager.rs b/crates/lib/pager/src/client/pager.rs index 482c499020..f1dea9f6eb 100644 --- a/crates/lib/pager/src/client/pager.rs +++ b/crates/lib/pager/src/client/pager.rs @@ -77,7 +77,7 @@ impl Pager { notification: NotificationDetails, ) -> Result<(), PagerError> { if self.client.is_noop() { - debug!("Pager no-op: would have enqueued page: {}", notification.summary); + debug!("Pager no-op: would have enqueued page: {}", notification.title); return Ok(()); } @@ -87,7 +87,7 @@ impl Pager { let dropped = queue.push(notification)?; if let Some(dropped) = dropped { - log::warn!("Pager queue overflow: dropped '{}'", dropped.summary); + log::warn!("Pager queue overflow: dropped '{}'", dropped.title); } Ok(()) diff --git a/crates/lib/pager/src/client/pager_client.rs b/crates/lib/pager/src/client/pager_client.rs index e563acfc46..3f6895b225 100644 --- a/crates/lib/pager/src/client/pager_client.rs +++ b/crates/lib/pager/src/client/pager_client.rs @@ -85,7 +85,7 @@ impl PagerClient { pub async fn send_page(&self, notification: &NotificationDetails) -> Result, PagerError> { match &self.client_config { PagerClientConfig::NoOp => { - debug!("Pager no-op: would have sent page: {}", notification.summary); + debug!("Pager no-op: would have sent page: {}", notification.title); Ok(None) } PagerClientConfig::Rootly { .. } => { @@ -113,7 +113,7 @@ impl PagerClient { .clone() .unwrap_or_else(|| "unknown".to_string()); - debug!("Sending page via Rootly (source={}): {}", source, notification.summary); + debug!("Sending page via Rootly (source={}): {}", source, notification.title); let mut labels: Vec<(String, String)> = Vec::new(); @@ -190,7 +190,7 @@ impl PagerClient { let result = create_alert(CreateAlertArgs { api_key: api_key.clone(), source: "api".to_string(), - summary: notification.summary.clone(), + summary: notification.title.clone(), description, status: Some("triggered".to_string()), service_ids: self.service_id diff --git a/crates/lib/pager/src/notification/generate_deduplication_key.rs b/crates/lib/pager/src/notification/generate_deduplication_key.rs index 0f4d055a0e..b340d36485 100644 --- a/crates/lib/pager/src/notification/generate_deduplication_key.rs +++ b/crates/lib/pager/src/notification/generate_deduplication_key.rs @@ -14,13 +14,13 @@ fn hours_since_epoch(time: DateTime) -> i64 { /// The key is a hex-encoded SHA-256 hash of the notification's identifying fields. /// /// When both `http_method` and `endpoint_path` are present, they are included in the -/// hash so that different endpoints with the same summary produce distinct keys. +/// hash so that different endpoints with the same title produce distinct keys. /// The `hours_since_epoch` of the event time is included so that the same notification /// firing in different hours produces a different key (allowing re-alerting). pub(crate) fn generate_deduplication_key(details: &NotificationDetails) -> String { let mut hasher = Sha256::new(); - hasher.update(details.summary.as_bytes()); + hasher.update(details.title.as_bytes()); hasher.update(if details.is_from_error { b"1" } else { b"0" }); if let (Some(method), Some(path)) = (&details.http_method, &details.http_path) { @@ -44,14 +44,14 @@ mod tests { use chrono::TimeZone; fn make_details( - summary: &str, + title: &str, is_from_error: bool, http_method: Option<&str>, http_path: Option<&str>, event_time: DateTime, ) -> NotificationDetails { NotificationDetails { - summary: summary.to_string(), + title: title.to_string(), description: None, event_time, http_method: http_method.map(|s| s.to_string()), @@ -154,7 +154,7 @@ mod tests { } #[test] - fn different_summary_different_key() { + fn different_title_different_key() { let time = Utc.with_ymd_and_hms(2026, 3, 30, 14, 0, 0).unwrap(); let a = make_details("error A", true, None, None, time); @@ -198,7 +198,7 @@ mod tests { use super::*; fn make_details_with_status( - summary: &str, + title: &str, is_from_error: bool, http_method: Option<&str>, http_path: Option<&str>, @@ -206,7 +206,7 @@ mod tests { event_time: DateTime, ) -> NotificationDetails { NotificationDetails { - summary: summary.to_string(), + title: title.to_string(), description: None, event_time, http_method: http_method.map(|s| s.to_string()), diff --git a/crates/lib/pager/src/notification/notification_details.rs b/crates/lib/pager/src/notification/notification_details.rs index 4086dd2cd3..2ea3c19410 100644 --- a/crates/lib/pager/src/notification/notification_details.rs +++ b/crates/lib/pager/src/notification/notification_details.rs @@ -8,7 +8,7 @@ use crate::notification::notification_urgency::NotificationUrgency; #[derive(Clone, Debug)] pub struct NotificationDetails { /// Title or summary of the alert. - pub summary: String, + pub title: String, /// Full details for the alert. pub description: Option, @@ -37,10 +37,10 @@ impl NotificationDetails { generate_deduplication_key(self) } - /// Create a notification with a summary and description. - pub fn with_summary_and_description(summary: String, description: String) -> Self { + /// Create a notification with a title and description. + pub fn with_title_and_description(title: String, description: String) -> Self { Self { - summary, + title, description: Some(description), event_time: Utc::now(), http_method: None, @@ -51,10 +51,10 @@ impl NotificationDetails { } } - /// Create a notification with just a summary. - pub fn with_summary(summary: String) -> Self { + /// Create a notification with just a title. + pub fn with_title(title: String) -> Self { Self { - summary, + title, description: None, event_time: Utc::now(), http_method: None, @@ -73,13 +73,13 @@ impl NotificationDetails { /// - A backtrace (if available via `std::backtrace`) /// - The timestamp of the event pub fn from_error(error: &E) -> Self { - let summary = format!("{}", error); + let title = format!("{}", error); - // Truncate summary to a reasonable length for alert titles. - let summary = if summary.len() > 200 { - format!("{}...", &summary[..197]) + // Truncate the title to a reasonable length for alert titles. + let title = if title.len() > 200 { + format!("{}...", &title[..197]) } else { - summary + title }; let event_time = Utc::now(); @@ -115,7 +115,7 @@ impl NotificationDetails { let description = description_parts.join("\n"); Self { - summary, + title, description: Some(description), event_time, http_method: None, @@ -126,12 +126,12 @@ impl NotificationDetails { } } - /// Create a notification from an error, with a custom summary prefix. + /// Create a notification from an error, with a custom title prefix. pub fn from_error_with_context(context: &str, error: &E) -> Self { let mut notification = Self::from_error(error); - notification.summary = format!("{}: {}", context, notification.summary); - if notification.summary.len() > 200 { - notification.summary = format!("{}...", ¬ification.summary[..197]); + notification.title = format!("{}: {}", context, notification.title); + if notification.title.len() > 200 { + notification.title = format!("{}...", ¬ification.title[..197]); } notification } diff --git a/crates/lib/pager/src/notification/notification_details_builder.rs b/crates/lib/pager/src/notification/notification_details_builder.rs index fd3fda28fa..72106efbe5 100644 --- a/crates/lib/pager/src/notification/notification_details_builder.rs +++ b/crates/lib/pager/src/notification/notification_details_builder.rs @@ -5,7 +5,7 @@ use crate::notification::notification_details::NotificationDetails; use crate::notification::notification_urgency::NotificationUrgency; pub struct NotificationDetailsBuilder { - summary: String, + title: String, description: Option, event_time: DateTime, http_method: Option, @@ -17,9 +17,9 @@ pub struct NotificationDetailsBuilder { impl NotificationDetailsBuilder { /// Create a builder from a summary string. - pub fn from_summary(summary: String) -> Self { + pub fn from_title(title: String) -> Self { Self { - summary, + title, description: None, event_time: Utc::now(), http_method: None, @@ -38,7 +38,7 @@ impl NotificationDetailsBuilder { // TODO(bt,2026-03-30): Clean this up let details = NotificationDetails::from_error(error); Self { - summary: details.summary, + title: details.title, description: details.description, event_time: details.event_time, http_method: None, @@ -49,8 +49,8 @@ impl NotificationDetailsBuilder { } } - pub fn set_summary(mut self, summary: String) -> Self { - self.summary = summary; + pub fn set_title(mut self, title: String) -> Self { + self.title = title; self } @@ -81,7 +81,7 @@ impl NotificationDetailsBuilder { pub fn build(self) -> NotificationDetails { NotificationDetails { - summary: self.summary, + title: self.title, description: self.description, event_time: self.event_time, http_method: self.http_method, diff --git a/crates/lib/pager/src/worker/pager_worker.rs b/crates/lib/pager/src/worker/pager_worker.rs index 7105da5af7..7f9e43fac5 100644 --- a/crates/lib/pager/src/worker/pager_worker.rs +++ b/crates/lib/pager/src/worker/pager_worker.rs @@ -78,8 +78,8 @@ impl PagerWorker { match self.client.send_page(notification).await { Ok(Some(success)) => { debug!( - "Pager worker sent page: id={:?}, summary={}", - success.id, notification.summary + "Pager worker sent page: id={:?}, title={}", + success.id, notification.title ); } Ok(None) => { @@ -88,7 +88,7 @@ impl PagerWorker { Err(err) => { error!( "Pager worker failed to send page for '{}': {}", - notification.summary, err + notification.title, err ); // Don't kill the thread on errors — keep processing. } diff --git a/crates/lib/pager/src/worker/pager_worker_message_queue.rs b/crates/lib/pager/src/worker/pager_worker_message_queue.rs index a8e8f2ace2..4ebd5c30b1 100644 --- a/crates/lib/pager/src/worker/pager_worker_message_queue.rs +++ b/crates/lib/pager/src/worker/pager_worker_message_queue.rs @@ -35,7 +35,7 @@ impl PagerWorkerMessageQueue { /// Push a notification onto the queue. /// /// If the queue is full, the oldest item is dropped and a warning is logged. - /// Returns `Ok(Some(summary))` if an old item was evicted, `Ok(None)` otherwise. + /// Returns `Ok(Some(notification))` if an old item was evicted, `Ok(None)` otherwise. pub fn push(&self, notification: NotificationDetails) -> Result, PagerError> { let mut queue = self.inner.lock() .map_err(|e| PagerSystemError::MutexPoisoned(format!("push: {}", e)))?; @@ -43,7 +43,7 @@ impl PagerWorkerMessageQueue { let dropped = if queue.len() >= self.max_size { let old = queue.pop_front(); if let Some(ref n) = old { - warn!("Pager queue full (max={}). Dropped oldest: {}", self.max_size, n.summary); + warn!("Pager queue full (max={}). Dropped oldest: {}", self.max_size, n.title); } old } else { diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 43b2d184a5..c43e78d5ca 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -691,6 +691,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux EditUserFeatureFlagsRequest, ModerationSendAlertRequest, ModerationSendAlertResponse, + ModerationSendAlertUrgency, ModeratorListSubscribingUsersBySignupDateEntry, ModeratorListSubscribingUsersBySignupDateRequest, ModeratorListSubscribingUsersBySignupDateResponse, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs index de8aa9f9b2..8aa94d3cd2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs @@ -223,7 +223,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler( warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err); if let Err(page_err) = server_state.pager.enqueue_page( NotificationDetailsBuilder::from_error(&err) - .set_summary("Seedance 2.0 generation failed (whitelist + fallback)".to_string()) + .set_title("Seedance 2.0 generation failed (whitelist + fallback)".to_string()) .set_urgency(Some(NotificationUrgency::High)) .build() ) { @@ -257,7 +257,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler( warn!("Error calling seedance2pro generate_video: {:?}", err); if let Err(page_err) = server_state.pager.enqueue_page( NotificationDetailsBuilder::from_error(&err) - .set_summary("Seedance 2.0 generation failed".to_string()) + .set_title("Seedance 2.0 generation failed".to_string()) .set_urgency(Some(NotificationUrgency::High)) .build() ) { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs index 77f5e60d56..214caefbc9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs @@ -4,13 +4,11 @@ use actix_web::web::Json; use actix_web::{web, HttpRequest}; use log::warn; -use artcraft_api_defs::moderation::alerts::moderation_send_alert::{ - ModerationSendAlertRequest, - ModerationSendAlertResponse, -}; +use artcraft_api_defs::moderation::alerts::moderation_send_alert::{ModerationSendAlertRequest, ModerationSendAlertResponse, ModerationSendAlertUrgency}; use pager::client::pager::Pager; use pager::notification::notification_details::NotificationDetails; - +use pager::notification::notification_details_builder::NotificationDetailsBuilder; +use pager::notification::notification_urgency::NotificationUrgency; use crate::http_server::common_responses::common_web_error::CommonWebError; use crate::http_server::web_utils::user_session::require_moderator::{require_moderator, UseDatabase}; use crate::state::server_state::ServerState; @@ -33,17 +31,38 @@ pub async fn moderation_send_alert_handler( server_state: web::Data>, ) -> Result, CommonWebError> { - let _user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection) + let user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection) .await .map_err(|_| CommonWebError::NotAuthorized)?; let title = request.title.clone() - .unwrap_or_else(|| "Test Moderation Alert".to_string()); + .unwrap_or_else(|| "Moderation Alert".to_string()); - let description = request.description.clone() - .unwrap_or_else(|| "This is a test moderation alert.".to_string()); + let description = { + let description = request.description.clone() + .unwrap_or_else(|| "This moderation alert does not have a description.".to_string()); - let notification = NotificationDetails::with_summary_and_description(title, description); + let description = vec![ + description, + format!("Sent by: {}", user_session.username), + ]; + + description.join("\n\n") + }; + + let urgency = request.urgency + .map(|urgency| match urgency { + ModerationSendAlertUrgency::High => NotificationUrgency::High, + ModerationSendAlertUrgency::Medium => NotificationUrgency::Medium, + ModerationSendAlertUrgency::Low => NotificationUrgency::Low, + }); + + let notification = NotificationDetailsBuilder::from_title(title) + .set_description(Some(description)) + .set_http_method(Some(http_request.method().to_string())) + .set_http_path(Some(http_request.path().to_string())) + .set_urgency(urgency) + .build(); server_state.pager .enqueue_page(notification) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs index 9c227df29a..468f6e64b3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs @@ -10,6 +10,9 @@ use log::error; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use pager::notification::notification_details_builder::NotificationDetailsBuilder; +use pager::notification::notification_urgency::NotificationUrgency; + use crate::state::server_state::ServerState; // =============== Success Response =============== @@ -73,6 +76,32 @@ pub async fn get_health_check_handler( .num_seconds() .unsigned_abs(); + if !is_healthy { + let notification = NotificationDetailsBuilder::from_title( + format!("Health check unhealthy on {}", server_state.hostname)) + .set_description(Some(format!( + "Health check returned unhealthy.\n\n\ + Hostname: {}\n\ + Build SHA: {}\n\ + Seconds since startup: {}\n\ + Unhealthy consecutive count: {:?}\n\ + Last DB time: {:?}", + server_state.hostname, + server_state.server_info.build_sha, + seconds_since_startup, + health_check_status.unhealthy_check_consecutive_count, + health_check_status.last_db_time, + ))) + .set_urgency(Some(NotificationUrgency::High)) + .set_http_method(Some(http_request.method().to_string())) + .set_http_path(Some(http_request.path().to_string())) + .build(); + + if let Err(err) = server_state.pager.enqueue_page(notification) { + error!("Failed to enqueue health check alert: {:?}", err); + } + } + let response = HealthCheckResponse { success: true, is_healthy: health_check_status.is_healthy, diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs index bed2340b51..0b959d1ac4 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs @@ -26,7 +26,7 @@ pub(super) fn check_common_web_error( chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"), ); - let notification = NotificationDetailsBuilder::from_summary(summary) + let notification = NotificationDetailsBuilder::from_title(summary) .set_description(Some(description)) .set_http_method(Some(method.to_string())) .set_http_path(Some(path.to_string())) diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs index a10a870e34..5bed6d6980 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs @@ -22,7 +22,7 @@ pub(super) fn check_status_code_fallback( chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"), ); - let notification = NotificationDetailsBuilder::from_summary(summary) + let notification = NotificationDetailsBuilder::from_title(summary) .set_description(Some(description)) .set_http_method(Some(method.to_string())) .set_http_path(Some(path.to_string())) diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index e07d9c45b9..d2cd08f5cb 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -307,15 +307,34 @@ async fn main() -> AnyhowResult<()> { let mysql_pool4 = pool.clone(); let mysql_pool5 = pool.clone(); + let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) + .ok_or(anyhow!("invalid server environment"))?; + + let server_environment_typed = match server_environment { + ServerEnvironment::Production => server_environment::ServerEnvironment::Production, + ServerEnvironment::Development => server_environment::ServerEnvironment::Development, + }; + + let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, &server_hostname); + let tokio_runtime = Runtime::new()?; + info!("Spawning pager worker thread."); + + tokio_runtime.spawn(async move { + pager_worker.run().await; + }); + info!("Spawning DB health checker thread."); + let health_checker_pager = pager.clone(); + tokio_runtime.spawn(async move { db_health_checker_thread( health_check_status2, mysql_pool3, health_check_interval, + health_checker_pager, ).await; }); @@ -331,22 +350,6 @@ async fn main() -> AnyhowResult<()> { poll_model_token_info_thread(model_token_info_cache2, mysql_pool5).await; }); - let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?) - .ok_or(anyhow!("invalid server environment"))?; - - let server_environment_typed = match server_environment { - ServerEnvironment::Production => server_environment::ServerEnvironment::Production, - ServerEnvironment::Development => server_environment::ServerEnvironment::Development, - }; - - let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, &server_hostname); - - info!("Spawning pager worker thread."); - - tokio_runtime.spawn(async move { - pager_worker.run().await; - }); - let stripe_configs = StripeConfig { checkout: StripeCheckoutConfigs { success_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_SUCCESS_URL_PATH")?), diff --git a/crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs b/crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs index ce2c7e8bb5..f61400205d 100644 --- a/crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs +++ b/crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs @@ -4,6 +4,9 @@ use log::debug; use log::error; use log::warn; use mysql_queries::queries::health_check::health_check_query::health_check_db; +use pager::client::pager::Pager; +use pager::notification::notification_details_builder::NotificationDetailsBuilder; +use pager::notification::notification_urgency::NotificationUrgency; use sqlx::MySqlPool; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; @@ -12,6 +15,7 @@ pub async fn db_health_checker_thread( health_check_status: HealthCheckStatus, mysql_pool: MySqlPool, check_duration: Duration, + pager: Pager, ) { loop { debug!("Checking DB health..."); @@ -23,12 +27,22 @@ pub async fn db_health_checker_thread( Ok(_) => {}, } } - Err(e) => { - error!("Problem health checking database: {:?}", e); + Err(database_error) => { + error!("Problem health checking database: {:?}", database_error); + match health_check_status.record_ping_failure() { Err(e) => error!("Problem updating application health checks!"), Ok(_) => {}, } + + let notification = NotificationDetailsBuilder::from_error(&database_error) + .set_title("DB health check thread failed check".to_string()) + .set_urgency(Some(NotificationUrgency::High)) + .build(); + + if let Err(page_err) = pager.enqueue_page(notification) { + error!("Failed to enqueue DB health check alert: {:?}", page_err); + } } }