From 62b01bd9d8177da98eca987c909ecbea2b49776f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 3 Jul 2026 01:45:20 -0400 Subject: [PATCH] trace-id --- crates/lib/easyenv/Cargo.toml | 1 + crates/lib/easyenv/src/logging.rs | 33 +++- crates/lib/logging/Cargo.toml | 1 + crates/lib/logging/src/lib.rs | 33 +++- crates/lib/pager/Cargo.toml | 1 + .../src/notification/notification_details.rs | 7 + .../notification_details_builder.rs | 16 ++ crates/schema/trace_id/Cargo.toml | 25 +++ crates/schema/trace_id/src/lib.rs | 156 ++++++++++++++++++ crates/service/web/storyteller_web/Cargo.toml | 1 + .../handlers/check_common_web_error.rs | 1 + .../request_debugging_metadata.rs | 10 ++ .../src/http_server/middleware/mod.rs | 1 + .../middleware/trace_id_middleware/mod.rs | 1 + .../trace_id_middleware.rs | 95 +++++++++++ .../service/web/storyteller_web/src/main.rs | 6 +- 16 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 crates/schema/trace_id/Cargo.toml create mode 100644 crates/schema/trace_id/src/lib.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/trace_id_middleware.rs diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index 22cd80300b..2651496c1b 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -25,6 +25,7 @@ workspace-build-acceleration = { version = "0.1", path = "../../workspace-build- # External dotenv.workspace = true env_logger = "0.11.3" +trace_id.workspace = true log = "0.4.14" [dev-dependencies] diff --git a/crates/lib/easyenv/src/logging.rs b/crates/lib/easyenv/src/logging.rs index ae4ce8c07d..5d389f9e9b 100644 --- a/crates/lib/easyenv/src/logging.rs +++ b/crates/lib/easyenv/src/logging.rs @@ -29,6 +29,37 @@ pub fn init_env_logger(default_if_absent: Option<&str>) { env::set_var(ENV_RUST_LOG, default_log_level); } - env_logger::init(); + // Custom format so that log lines emitted while serving an HTTP request + // automatically carry the request's trace id (see the `trace_id` crate and + // the `TraceIdMiddleware` in storyteller-web). Outside of a request scope + // the trace segment is simply omitted. + env_logger::Builder::from_env(env_logger::Env::default()) + .format(|buf, record| { + use std::io::Write; + + let timestamp = buf.timestamp(); + let level_style = buf.default_level_style(record.level()); + + match trace_id::current_trace_id() { + Some(trace) => writeln!( + buf, + "[{} {level_style}{}{level_style:#} {} {}] {}", + timestamp, + record.level(), + record.target(), + trace, + record.args(), + ), + None => writeln!( + buf, + "[{} {level_style}{}{level_style:#} {}] {}", + timestamp, + record.level(), + record.target(), + record.args(), + ), + } + }) + .init(); } diff --git a/crates/lib/logging/Cargo.toml b/crates/lib/logging/Cargo.toml index 9f285920ad..ebb2a7960e 100644 --- a/crates/lib/logging/Cargo.toml +++ b/crates/lib/logging/Cargo.toml @@ -25,6 +25,7 @@ workspace-build-acceleration = { workspace = true } # External env_logger = "0.11.3" +trace_id.workspace = true log = { workspace = true } [dev-dependencies] diff --git a/crates/lib/logging/src/lib.rs b/crates/lib/logging/src/lib.rs index 7743fc0080..ee4b136422 100644 --- a/crates/lib/logging/src/lib.rs +++ b/crates/lib/logging/src/lib.rs @@ -68,6 +68,37 @@ pub fn init_env_logger(default_if_absent: Option<&str>) { env::set_var(ENV_RUST_LOG, default_log_level); } - env_logger::init(); + // Custom format so that log lines emitted while serving an HTTP request + // automatically carry the request's trace id (see the `trace_id` crate and + // the `TraceIdMiddleware` in storyteller-web). Outside of a request scope + // the trace segment is simply omitted. + env_logger::Builder::from_env(env_logger::Env::default()) + .format(|buf, record| { + use std::io::Write; + + let timestamp = buf.timestamp(); + let level_style = buf.default_level_style(record.level()); + + match trace_id::current_trace_id() { + Some(trace) => writeln!( + buf, + "[{} {level_style}{}{level_style:#} {} {}] {}", + timestamp, + record.level(), + record.target(), + trace, + record.args(), + ), + None => writeln!( + buf, + "[{} {level_style}{}{level_style:#} {}] {}", + timestamp, + record.level(), + record.target(), + record.args(), + ), + } + }) + .init(); } diff --git a/crates/lib/pager/Cargo.toml b/crates/lib/pager/Cargo.toml index 74e2f302e1..0133da1189 100644 --- a/crates/lib/pager/Cargo.toml +++ b/crates/lib/pager/Cargo.toml @@ -24,6 +24,7 @@ chrono.workspace = true sha2.workspace = true log.workspace = true tokio.workspace = true +trace_id.workspace = true [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/crates/lib/pager/src/notification/notification_details.rs b/crates/lib/pager/src/notification/notification_details.rs index 7c4ca3404b..c7f057dc89 100644 --- a/crates/lib/pager/src/notification/notification_details.rs +++ b/crates/lib/pager/src/notification/notification_details.rs @@ -49,6 +49,9 @@ pub struct NotificationDetails { /// Third-party identifier associated with the event, if any. pub(crate) third_party_id: Option, + /// The request-scoped trace id associated with the event, if any. + pub(crate) trace_id: Option, + /// IP address of the request that triggered this notification, if any. pub(crate) request_ip_address: Option, @@ -189,6 +192,10 @@ impl NotificationDetails { parts.push(format!("Request IP: {}", ip)); } + if let Some(trace_id) = &self.trace_id { + parts.push(format!("Trace ID: {}", trace_id)); + } + if parts.is_empty() { None } else { Some(parts.join("\n")) } } diff --git a/crates/lib/pager/src/notification/notification_details_builder.rs b/crates/lib/pager/src/notification/notification_details_builder.rs index b538c77366..a45ae9d28d 100644 --- a/crates/lib/pager/src/notification/notification_details_builder.rs +++ b/crates/lib/pager/src/notification/notification_details_builder.rs @@ -31,6 +31,10 @@ pub struct NotificationDetailsBuilder { pub(crate) avt_cookie_token: Option, pub(crate) session_token: Option, pub(crate) session_user_token: Option, + + /// The request-scoped trace id, if any. Auto-filled from the tokio + /// task-local at `build()` time when not explicitly set. + pub(crate) trace_id: Option, } impl NotificationDetailsBuilder { @@ -57,6 +61,7 @@ impl NotificationDetailsBuilder { avt_cookie_token: None, session_token: None, session_user_token: None, + trace_id: None, } } @@ -87,6 +92,7 @@ impl NotificationDetailsBuilder { avt_cookie_token: None, session_token: None, session_user_token: None, + trace_id: None, } } @@ -167,6 +173,11 @@ impl NotificationDetailsBuilder { self } + pub fn set_trace_id(mut self, trace_id: Option) -> Self { + self.trace_id = trace_id; + self + } + pub fn set_session_user_token(mut self, session_user_token: Option) -> Self { self.session_user_token = session_user_token; self @@ -199,6 +210,11 @@ impl NotificationDetailsBuilder { avt_cookie_token: self.avt_cookie_token, session_token: self.session_token, session_user_token: self.session_user_token, + // Auto-fill from the request's task-local so any page fired while + // serving a request is correlated even if the caller never set it. + trace_id: self.trace_id.or_else(|| { + trace_id::current_trace_id().map(|t| t.to_string()) + }), } } diff --git a/crates/schema/trace_id/Cargo.toml b/crates/schema/trace_id/Cargo.toml new file mode 100644 index 0000000000..c51ab81160 --- /dev/null +++ b/crates/schema/trace_id/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "trace_id" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-build-acceleration"] + +[lib] +name = "trace_id" +path = "src/lib.rs" + +[dependencies] +# Workspace hack (faster compile times) +workspace-build-acceleration.workspace = true + +rand = { version = "0.8.5", features = ["std_rng"] } +serde.workspace = true +serde_derive.workspace = true +tokio.workspace = true diff --git a/crates/schema/trace_id/src/lib.rs b/crates/schema/trace_id/src/lib.rs new file mode 100644 index 0000000000..3abca3c34d --- /dev/null +++ b/crates/schema/trace_id/src/lib.rs @@ -0,0 +1,156 @@ +//! trace_id +//! +//! Request-scoped trace identifiers for observability. +//! +//! A `TraceId` is generated for every inbound HTTP request (see the +//! `trace_id_middleware` in `storyteller-web`), stored in a tokio task-local, +//! and emitted with every log line and pager notification produced while +//! serving that request. + +use std::fmt; + +use serde_derive::{Deserialize, Serialize}; + +/// The string prefix for trace ids. +pub const TRACE_ID_PREFIX: &str = "trace_"; + +/// Number of Crockford base32 characters after the prefix. +/// 26 characters × 5 bits = 130 bits of entropy (≥ the 128-bit industry +/// standard for trace ids). Total length: 6 + 26 = 32 characters. +pub const TRACE_ID_ENTROPY_CHARS: usize = 26; + +/// Total serialized length of a trace id ("trace_" + entropy). +pub const TRACE_ID_TOTAL_LENGTH: usize = 32; + +/// Crockford base32 alphabet, lowercase (no i, l, o, u). +const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz"; + +/// A unique-per-request trace identifier, eg. `trace_1h3x9k2m4p7q8r5t6v0w2y3z4a`. +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct TraceId(String); + +impl TraceId { + /// Generate a new random trace id. Infallible. + #[inline] + pub fn generate() -> Self { + use rand::Rng; + + let mut rng = rand::thread_rng(); + + let entropy: String = (0..TRACE_ID_ENTROPY_CHARS) + .map(|_| { + let idx = rng.gen_range(0..CROCKFORD_LOWERCASE_CHARSET.len()); + CROCKFORD_LOWERCASE_CHARSET[idx] as char + }) + .collect(); + + TraceId(format!("{}{}", TRACE_ID_PREFIX, entropy)) + } + + #[inline] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for TraceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl fmt::Debug for TraceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +tokio::task_local! { + /// The trace id for the currently-executing request, if any. + /// + /// Set by the HTTP middleware via [`TRACE_ID_TASK_LOCAL.scope(...)`], and + /// readable from anywhere within the request's task — including the + /// logger's format closure and the pager — via [`current_trace_id`]. + pub static TRACE_ID_TASK_LOCAL: TraceId; +} + +/// The trace id of the currently-executing request, if one is in scope. +/// +/// Returns `None` outside of a request context (startup, background threads, +/// detached `tokio::spawn` tasks that weren't explicitly re-scoped). +#[inline] +pub fn current_trace_id() -> Option { + TRACE_ID_TASK_LOCAL.try_with(|trace_id| trace_id.clone()).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + mod generation { + use super::*; + + #[test] + fn total_length_is_32() { + assert_eq!(TraceId::generate().as_str().len(), TRACE_ID_TOTAL_LENGTH); + } + + #[test] + fn has_prefix() { + assert!(TraceId::generate().as_str().starts_with(TRACE_ID_PREFIX)); + } + + #[test] + fn entropy_uses_crockford_lowercase_only() { + let trace_id = TraceId::generate(); + let entropy = &trace_id.as_str()[TRACE_ID_PREFIX.len()..]; + assert_eq!(entropy.len(), TRACE_ID_ENTROPY_CHARS); + for c in entropy.chars() { + assert!( + CROCKFORD_LOWERCASE_CHARSET.contains(&(c as u8)), + "unexpected character: {}", + c + ); + } + } + + #[test] + fn ids_are_unique() { + let a = TraceId::generate(); + let b = TraceId::generate(); + assert_ne!(a, b); + } + } + + mod task_local { + use super::*; + + #[tokio::test] + async fn absent_outside_scope() { + assert!(current_trace_id().is_none()); + } + + #[tokio::test] + async fn present_inside_scope() { + let trace_id = TraceId::generate(); + let expected = trace_id.clone(); + TRACE_ID_TASK_LOCAL + .scope(trace_id, async move { + assert_eq!(current_trace_id(), Some(expected)); + }) + .await; + } + + #[tokio::test] + async fn survives_await_points() { + let trace_id = TraceId::generate(); + let expected = trace_id.clone(); + TRACE_ID_TASK_LOCAL + .scope(trace_id, async move { + tokio::task::yield_now().await; + assert_eq!(current_trace_id(), Some(expected)); + }) + .await; + } + } +} diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index dc5f806e8a..30fb9a4c93 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -89,6 +89,7 @@ seedance2pro_client.workspace = true server_environment = { path = "../../../lib/server_environment" } thumbnail_generator = { path = "../../../api_clients/thumbnail_generator" } tokens.workspace = true +trace_id.workspace = true tts_common = { path = "../../../lib/deprecated/tts_common" } url_config = { path = "../../../lib/url_config" } url_utils = { path = "../../../lib/url_utils" } diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/handlers/check_common_web_error.rs b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/handlers/check_common_web_error.rs index be67168dd4..aa07a4b425 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/handlers/check_common_web_error.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/handlers/check_common_web_error.rs @@ -44,6 +44,7 @@ pub(crate) fn check_common_web_error( .set_urgency(Some(NotificationUrgency::Medium)) .set_http_method(Some(method.to_string())) .set_http_path(Some(path.to_string())) + .set_trace_id(metadata.trace_id.clone()) .set_http_status_code(Some(500)) .set_request_ip_address(metadata.request_ip_address.clone()) .set_avt_cookie_token(metadata.avt_cookie_token.clone()) diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/request_debugging_metadata.rs b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/request_debugging_metadata.rs index fa6edfb3eb..951c40e41b 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/request_debugging_metadata.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/request_debugging_metadata.rs @@ -2,9 +2,12 @@ use std::sync::Arc; use actix_web::dev::ServiceRequest; use actix_web::web; +use actix_web::HttpMessage; use http_server_common::request::get_request_ip::get_request_ip; use log::debug; +use trace_id::TraceId; + use crate::state::server_state::ServerState; /// Optional per-request context extracted by the error alerting middleware @@ -16,6 +19,8 @@ pub(crate) struct RequestDebuggingMetadata { pub avt_cookie_token: Option, pub session_token: Option, pub session_user_token: Option, + /// Set by `TraceIdMiddleware` (outermost middleware) on every request. + pub trace_id: Option, } impl RequestDebuggingMetadata { @@ -26,6 +31,10 @@ impl RequestDebuggingMetadata { let request_ip_address = Some(get_request_ip(http_request)); + let trace_id = http_request.extensions() + .get::() + .map(|t| t.to_string()); + let server_state = req.app_data::>>(); let avt_cookie_token = server_state @@ -58,6 +67,7 @@ impl RequestDebuggingMetadata { avt_cookie_token, session_token, session_user_token, + trace_id, } } } diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/mod.rs b/crates/service/web/storyteller_web/src/http_server/middleware/mod.rs index 56fa763e93..ae5c6e18b6 100644 --- a/crates/service/web/storyteller_web/src/http_server/middleware/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/middleware/mod.rs @@ -1,3 +1,4 @@ pub mod error_alerting_middleware; pub mod metrics_middleware; pub mod pushback_filter_middleware; +pub mod trace_id_middleware; diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/mod.rs b/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/mod.rs new file mode 100644 index 0000000000..0a47160164 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/mod.rs @@ -0,0 +1 @@ +pub mod trace_id_middleware; diff --git a/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/trace_id_middleware.rs b/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/trace_id_middleware.rs new file mode 100644 index 0000000000..0fe39160ca --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/middleware/trace_id_middleware/trace_id_middleware.rs @@ -0,0 +1,95 @@ +//! Per-request trace id middleware. +//! +//! Generates a unique [`TraceId`] for every inbound HTTP request and makes it +//! available three ways: +//! +//! 1. **Request extensions** — downstream middleware and handlers can read +//! it via `req.extensions().get::()`. +//! 2. **Tokio task-local** — the whole request runs inside +//! [`trace_id::TRACE_ID_TASK_LOCAL`], so `trace_id::current_trace_id()` +//! works from anywhere in the request's task. The env-logger format +//! hook uses this to stamp every log line, and the pager uses it to tag +//! notifications. +//! 3. **`x-trace-id` response header** — so clients and support tickets can +//! quote the id. +//! +//! Register this middleware LAST in the actix `.wrap(...)` chain so it runs +//! FIRST (actix middleware executes in reverse registration order) and every +//! other middleware sees the trace id. +//! +//! NB: Detached `tokio::spawn` tasks do NOT inherit the task-local. Wrap +//! spawned futures in `TRACE_ID_TASK_LOCAL.scope(trace_id, fut)` if they +//! should stay correlated. + +use std::future::{Future, Ready, ready}; +use std::pin::Pin; + +use actix_web::Error; +use actix_web::HttpMessage; +use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; +use actix_web::http::header::{HeaderName, HeaderValue}; + +use trace_id::{TraceId, TRACE_ID_TASK_LOCAL}; + +/// Response header carrying the request's trace id. +pub const TRACE_ID_HEADER: &str = "x-trace-id"; + +#[derive(Clone, Default)] +pub struct TraceIdMiddleware; + +impl Transform for TraceIdMiddleware +where + S: Service, Error = Error> + 'static, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = TraceIdService; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ready(Ok(TraceIdService { service })) + } +} + +pub struct TraceIdService { + service: S, +} + +impl Service for TraceIdService +where + S: Service, Error = Error> + 'static, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Future = Pin>>>; + + actix_service::forward_ready!(service); + + fn call(&self, req: ServiceRequest) -> Self::Future { + let trace_id = TraceId::generate(); + + // Make the trace id readable from the request itself. + req.extensions_mut().insert(trace_id.clone()); + + let response_trace_id = trace_id.clone(); + let fut = self.service.call(req); + + Box::pin(TRACE_ID_TASK_LOCAL.scope(trace_id, async move { + let mut response = fut.await?; + + if let Ok(header_value) = HeaderValue::from_str(response_trace_id.as_str()) { + response.headers_mut().insert( + HeaderName::from_static(TRACE_ID_HEADER), + header_value, + ); + } + + Ok(response) + })) + } +} diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 0a6d2cb445..50627aa863 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -46,6 +46,7 @@ use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupIm use crate::http_server::middleware::error_alerting_middleware::error_alerting_middleware::ErrorAlertingMiddleware; use crate::http_server::middleware::metrics_middleware::metrics_middleware::MetricsMiddleware; use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter; +use crate::http_server::middleware::trace_id_middleware::trace_id_middleware::TraceIdMiddleware; use crate::http_server::routes::add_routes::add_routes; use crate::http_server::web_utils::handle_multipart_error::handle_multipart_error; use crate::startup::build_dependencies::setup_dependencies; @@ -271,7 +272,10 @@ pub async fn serve( .wrap(Logger::new(LOG_FORMAT) .exclude("/liveness") .exclude("/readiness")) - .wrap(middleware::Compress::default()); + .wrap(middleware::Compress::default()) + // NB: Registered last => runs FIRST. Every request gets a trace id + // before any other middleware or handler executes. + .wrap(TraceIdMiddleware); add_routes(app, server_environment) })