diff --git a/.gitignore b/.gitignore index 27017f83ad..752bc2096f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ env-secrets pubkeys.txt secrets.toml secrets.yaml +secrets/ twitch_secrets.toml # Old env secrets diff --git a/_metrics/datadog/dashboards/storyteller_web_http_overview.json b/_metrics/datadog/dashboards/storyteller_web_http_overview.json new file mode 100644 index 0000000000..78ab873e60 --- /dev/null +++ b/_metrics/datadog/dashboards/storyteller_web_http_overview.json @@ -0,0 +1,437 @@ +{ + "title": "Storyteller-web — HTTP overview", + "description": "Per-request HTTP metrics emitted by the storyteller-web metrics middleware (`http.server.request.count` counter, `http.server.request.duration_ms` distribution). Filter by `env` / `service` / `route` using the template variables at the top.\n\nMetrics emitted from `crates/lib/metrics` + `crates/service/web/storyteller_web/src/http_server/middleware/metrics_middleware/`. Source of truth for this dashboard JSON is `_metrics/datadog/dashboards/storyteller_web_http_overview.json` in the storyteller monorepo. Apply changes with `script/metrics/apply_dashboards.sh`.", + "layout_type": "ordered", + "notify_list": [], + "template_variables": [ + { + "name": "env", + "prefix": "env", + "available_values": [], + "default": "*" + }, + { + "name": "service", + "prefix": "service", + "available_values": [], + "default": "storyteller-web" + }, + { + "name": "route", + "prefix": "route", + "available_values": [], + "default": "*" + } + ], + "reflow_type": "fixed", + "widgets": [ + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Top-line health", + "show_title": true, + "widgets": [ + { + "definition": { + "type": "query_value", + "title": "Request rate (req/s)", + "title_size": "16", + "title_align": "left", + "precision": 1, + "autoscale": true, + "requests": [ + { + "q": "sum:http.server.request.count{$env,$service,$route}.as_rate()", + "aggregator": "avg" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "type": "query_value", + "title": "5xx error rate (%)", + "title_size": "16", + "title_align": "left", + "precision": 2, + "autoscale": false, + "custom_unit": "%", + "requests": [ + { + "q": "(sum:http.server.request.count{$env,$service,$route,status_class:5xx}.as_count() / sum:http.server.request.count{$env,$service,$route}.as_count()) * 100", + "aggregator": "avg", + "conditional_formats": [ + { + "comparator": ">", + "value": 5.0, + "palette": "white_on_red" + }, + { + "comparator": ">", + "value": 1.0, + "palette": "white_on_yellow" + }, + { + "comparator": "<=", + "value": 1.0, + "palette": "white_on_green" + } + ] + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "type": "query_value", + "title": "Latency p95 (ms)", + "title_size": "16", + "title_align": "left", + "precision": 0, + "autoscale": true, + "custom_unit": "ms", + "requests": [ + { + "q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}", + "aggregator": "avg", + "conditional_formats": [ + { + "comparator": ">", + "value": 1000, + "palette": "white_on_red" + }, + { + "comparator": ">", + "value": 250, + "palette": "white_on_yellow" + }, + { + "comparator": "<=", + "value": 250, + "palette": "white_on_green" + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "type": "query_value", + "title": "Latency p99 (ms)", + "title_size": "16", + "title_align": "left", + "precision": 0, + "autoscale": true, + "custom_unit": "ms", + "requests": [ + { + "q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}", + "aggregator": "avg" + } + ] + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 3 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Traffic shape", + "show_title": true, + "widgets": [ + { + "definition": { + "type": "timeseries", + "title": "Requests/sec by status class (stacked)", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "max", + "value", + "sum" + ], + "requests": [ + { + "q": "sum:http.server.request.count{$env,$service,$route} by {status_class}.as_rate()", + "display_type": "bars", + "style": { + "palette": "semantic", + "line_type": "solid", + "line_width": "normal" + } + } + ], + "yaxis": { + "scale": "linear", + "include_zero": true + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "definition": { + "type": "timeseries", + "title": "Latency percentiles over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "requests": [ + { + "q": "avg:http.server.request.duration_ms.p50{$env,$service,$route}", + "display_type": "line", + "style": { + "palette": "cool", + "line_type": "solid", + "line_width": "normal" + } + }, + { + "q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}", + "display_type": "line", + "style": { + "palette": "warm", + "line_type": "solid", + "line_width": "normal" + } + }, + { + "q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}", + "display_type": "line", + "style": { + "palette": "warm", + "line_type": "dashed", + "line_width": "normal" + } + } + ], + "yaxis": { + "scale": "linear", + "include_zero": true, + "label": "ms" + } + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 5 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Per-route breakdown", + "show_title": true, + "widgets": [ + { + "definition": { + "type": "toplist", + "title": "Top 10 routes by request volume", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "q": "top(sum:http.server.request.count{$env,$service,$route} by {route}.as_count(), 10, 'sum', 'desc')" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "definition": { + "type": "toplist", + "title": "Top 10 routes by p95 latency", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "q": "top(avg:http.server.request.duration_ms.p95{$env,$service,$route} by {route}, 10, 'mean', 'desc')" + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "definition": { + "type": "timeseries", + "title": "5xx error count by route", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "requests": [ + { + "q": "sum:http.server.request.count{$env,$service,$route,status_class:5xx} by {route}.as_count()", + "display_type": "bars" + } + ], + "yaxis": { + "scale": "linear", + "include_zero": true + } + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 4 + } + }, + { + "definition": { + "type": "timeseries", + "title": "4xx error count by route", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "requests": [ + { + "q": "sum:http.server.request.count{$env,$service,$route,status_class:4xx} by {route}.as_count()", + "display_type": "bars" + } + ], + "yaxis": { + "scale": "linear", + "include_zero": true + } + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 13 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Methods and status codes", + "show_title": true, + "widgets": [ + { + "definition": { + "type": "timeseries", + "title": "Requests/sec by HTTP method", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "requests": [ + { + "q": "sum:http.server.request.count{$env,$service,$route} by {method}.as_rate()", + "display_type": "bars" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "definition": { + "type": "toplist", + "title": "Status code distribution", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "q": "top(sum:http.server.request.count{$env,$service,$route} by {status_code}.as_count(), 25, 'sum', 'desc')" + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 21, + "width": 12, + "height": 5 + } + } + ], + "id": "thp-c27-vnv" +} diff --git a/_metrics/datadog/monitors/storyteller_web_5xx_error_rate.json b/_metrics/datadog/monitors/storyteller_web_5xx_error_rate.json new file mode 100644 index 0000000000..b653e1eb3e --- /dev/null +++ b/_metrics/datadog/monitors/storyteller_web_5xx_error_rate.json @@ -0,0 +1,27 @@ +{ + "name": "[storyteller-web] HTTP 5xx error rate is elevated", + "type": "query alert", + "tags": [ + "service:storyteller-web", + "team:backend", + "metric:http_5xx" + ], + "priority": 2, + "message": "5xx error rate on storyteller-web is elevated.\n\nDashboard: https://app.datadoghq.com/dashboard/thp-c27-vnv/storyteller-web--http-overview\n\nMetric: `http.server.request.count{status_class:5xx} / http.server.request.count` over the last 5 minutes.\n\nFirst response steps:\n- Open the dashboard linked above. Check the **5xx error count by route** widget to identify which routes are failing.\n- Check the **Latency percentiles over time** widget — a latency spike often co-occurs with a 5xx spike (e.g. DB or upstream-provider stall).\n- Tail the production logs for `WARN`/`ERROR` lines from the affected route(s).\n\nThis monitor's source-of-truth JSON is at `_metrics/datadog/monitors/storyteller_web_5xx_error_rate.json`. Update with `script/metrics/apply_monitors.sh`.\n\n{{#is_alert}}🔥 5xx error rate is **{{value}}%**, above the **5%** critical threshold.{{/is_alert}}\n{{#is_warning}}⚠️ 5xx error rate is **{{value}}%**, above the **1%** warning threshold.{{/is_warning}}\n{{#is_recovery}}✅ 5xx error rate recovered to **{{value}}%**.{{/is_recovery}}\n\n@slack-storyteller-pages", + "query": "sum(last_5m):(sum:http.server.request.count{service:storyteller-web,status_class:5xx,env:production}.as_count() / sum:http.server.request.count{service:storyteller-web,env:production}.as_count()) * 100 > 5", + "options": { + "thresholds": { + "critical": 5, + "warning": 1 + }, + "notify_audit": false, + "require_full_window": false, + "notify_no_data": false, + "renotify_interval": 30, + "evaluation_delay": 60, + "new_host_delay": 60, + "include_tags": true, + "escalation_message": "5xx error rate on storyteller-web is **still elevated** — please ack or fix." + }, + "id": 286427558 +} diff --git a/crates/api_clients/datadog_client/src/test_utils/get_test_api_key.rs b/crates/api_clients/datadog_client/src/test_utils/get_test_api_key.rs index 6aaff349f7..b72277fa5c 100644 --- a/crates/api_clients/datadog_client/src/test_utils/get_test_api_key.rs +++ b/crates/api_clients/datadog_client/src/test_utils/get_test_api_key.rs @@ -3,38 +3,43 @@ use std::fs::read_to_string; use std::path::{Path, PathBuf}; -/// Reads the Datadog API key from `datadog_key.txt` at the workspace root. -/// Used only by `#[ignore]` smoke tests that hit the real API. +/// Reads the Datadog API key from `secrets/datadog_key.txt` (relative to +/// the workspace root). Used only by `#[ignore]`d smoke tests that hit the +/// real API. pub(crate) fn get_test_api_key() -> Result { let candidates = candidate_paths(); for path in &candidates { if let Ok(contents) = read_to_string(path) { let trimmed = contents.trim().to_string(); if trimmed.is_empty() { - return Err(format!("datadog_key.txt at {} is empty", path.display())); + return Err(format!("{} is empty", path.display())); } return Ok(trimmed); } } Err(format!( - "Could not locate datadog_key.txt. Tried: {}", + "Could not locate Datadog API key. Tried: {}", candidates.iter().map(|p| p.display().to_string()).collect::>().join(", "), )) } -/// Walk upward from the test's CWD looking for `datadog_key.txt`. Tests run -/// from the crate dir (`crates/api_clients/datadog_client`), so we need to -/// climb a few levels to reach the workspace root. +/// Walk upward from the test's CWD looking for the key. Tests run from +/// the crate dir (`crates/api_clients/datadog_client`), so we need to +/// climb a few levels to reach the workspace root. We try the new +/// `secrets/datadog_key.txt` location first and fall back to the older +/// root-level `datadog_key.txt` for any stale checkouts. fn candidate_paths() -> Vec { let mut out = Vec::new(); let mut current: PathBuf = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); for _ in 0..6 { + out.push(current.join("secrets/datadog_key.txt")); out.push(current.join("datadog_key.txt")); if !current.pop() { break; } } - // Last-resort hardcoded path matching this repo's known layout. + // Last-resort hardcoded paths matching this repo's known layout. + out.push(Path::new("/Users/bt/dev/storyteller/artcraft/secrets/datadog_key.txt").to_path_buf()); out.push(Path::new("/Users/bt/dev/storyteller/artcraft/datadog_key.txt").to_path_buf()); out } diff --git a/script/metrics/apply_dashboards.sh b/script/metrics/apply_dashboards.sh new file mode 100755 index 0000000000..d2ac923786 --- /dev/null +++ b/script/metrics/apply_dashboards.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# apply_dashboards.sh +# +# Idempotently applies every dashboard JSON under `_metrics/datadog/dashboards/` +# to our Datadog account. +# +# Behavior, per file: +# - If the file already has a top-level `"id": ""`, PUT +# the dashboard to `/api/v1/dashboard/` (in-place update). +# - Otherwise, POST to `/api/v1/dashboard` (create) and write the assigned +# id back into the local JSON so the next run becomes an update. +# +# Secrets: +# secrets/datadog_key.txt — DD-API-KEY (required) +# secrets/datadog_app_key.txt — DD-APPLICATION-KEY (required; +# dashboard endpoints reject API-key-only +# calls with 403). +# +# Both files should contain just the key, optionally with trailing whitespace. +# +# Flags: +# --dry-run Print what would happen; don't call the API. +# --site Datadog site host. Default: api.datadoghq.com. +# US3 = api.us3.datadoghq.com, EU = api.datadoghq.eu, etc. +# --dir Override dashboards directory. +# Default: _metrics/datadog/dashboards +# +# Examples: +# script/metrics/apply_dashboards.sh +# script/metrics/apply_dashboards.sh --dry-run +# script/metrics/apply_dashboards.sh --site api.datadoghq.eu + +set -euo pipefail + +# -------- defaults -------- +DD_SITE="api.datadoghq.com" +DASH_DIR="_metrics/datadog/dashboards" +API_KEY_FILE="secrets/datadog_key.txt" +APP_KEY_FILE="secrets/datadog_app_key.txt" +DRY_RUN=0 + +# -------- args -------- +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --site) DD_SITE="$2"; shift 2 ;; + --dir) DASH_DIR="$2"; shift 2 ;; + -h|--help) + sed -n '2,28p' "$0"; exit 0 ;; + *) + echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +BASE="https://${DD_SITE}" + +# -------- preflight -------- +command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; } +command -v curl >/dev/null 2>&1 || { echo "curl is required" >&2; exit 1; } + +if [ ! -s "$API_KEY_FILE" ]; then + echo "missing or empty API key file: $API_KEY_FILE" >&2 + exit 1 +fi +if [ ! -s "$APP_KEY_FILE" ]; then + cat >&2 <&2 + exit 1 +fi + +shopt -s nullglob +files=("$DASH_DIR"/*.json) +if [ ${#files[@]} -eq 0 ]; then + echo "no dashboard JSON files in $DASH_DIR" >&2 + exit 0 +fi + +# -------- per-file apply -------- +exit_code=0 + +for file in "${files[@]}"; do + echo + echo "==> $file" + + # Validate JSON before sending it anywhere. + if ! jq empty "$file" 2>/dev/null; then + echo " invalid JSON, skipping" >&2 + exit_code=1 + continue + fi + + existing_id=$(jq -r '.id // empty' "$file") + + if [ -n "$existing_id" ]; then + method="PUT" + url="${BASE}/api/v1/dashboard/${existing_id}" + echo " updating existing dashboard id=${existing_id}" + else + method="POST" + url="${BASE}/api/v1/dashboard" + echo " creating new dashboard" + fi + + if [ "$DRY_RUN" -eq 1 ]; then + echo " [dry-run] would ${method} ${url}" + continue + fi + + http_status_file=$(mktemp) + response_file=$(mktemp) + # Drop the local-only `id` field before sending — Datadog returns it on + # POST and accepts it (or ignores it) on PUT, but keeping it out of the + # request body avoids any chance of confusing the API. + payload=$(jq 'del(.id)' "$file") + + curl_status=$(curl -sS -o "$response_file" -w "%{http_code}" \ + -X "$method" "$url" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: ${DD_API_KEY}" \ + -H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \ + --data-binary "$payload" || echo "000") + echo "$curl_status" > "$http_status_file" + + if [ "$curl_status" -lt 200 ] || [ "$curl_status" -ge 300 ]; then + echo " ✗ HTTP $curl_status" + sed 's/^/ /' "$response_file" >&2 + exit_code=1 + else + if [ "$method" = "POST" ]; then + new_id=$(jq -r '.id // empty' "$response_file") + if [ -n "$new_id" ]; then + tmp=$(mktemp) + jq --arg id "$new_id" '. + {id: $id}' "$file" > "$tmp" && mv "$tmp" "$file" + echo " ✓ created (id=${new_id}), id written back to $file" + else + echo " ✓ created but response had no id (??); body:" + sed 's/^/ /' "$response_file" + exit_code=1 + fi + else + echo " ✓ updated" + fi + fi + + rm -f "$http_status_file" "$response_file" +done + +echo +if [ "$exit_code" -eq 0 ]; then + echo "All dashboards applied." +else + echo "Finished with errors. See output above." +fi +exit "$exit_code" diff --git a/script/metrics/apply_monitors.sh b/script/metrics/apply_monitors.sh new file mode 100755 index 0000000000..be2e5076d2 --- /dev/null +++ b/script/metrics/apply_monitors.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# apply_monitors.sh +# +# Idempotently applies every monitor JSON under `_metrics/datadog/monitors/` +# to our Datadog account. Sibling to `apply_dashboards.sh`. +# +# Behavior, per file: +# - If the file already has a top-level `"id": `, PUT the +# monitor to `/api/v1/monitor/` (in-place update). +# - Otherwise, POST to `/api/v1/monitor` (create) and write the assigned +# id back into the local JSON so the next run becomes an update. +# +# Note: Datadog dashboard ids are strings (`thp-c27-vnv`); monitor ids are +# integers. The script handles both — `jq` writes the value the API +# returns, whatever its type. +# +# Secrets: +# secrets/datadog_key.txt — DD-API-KEY (required) +# secrets/datadog_app_key.txt — DD-APPLICATION-KEY (required) +# +# Flags: +# --dry-run Print what would happen; don't call the API. +# --site Datadog site host. Default: api.datadoghq.com. +# --dir Override monitors directory. +# Default: _metrics/datadog/monitors + +set -euo pipefail + +# -------- defaults -------- +DD_SITE="api.datadoghq.com" +MON_DIR="_metrics/datadog/monitors" +API_KEY_FILE="secrets/datadog_key.txt" +APP_KEY_FILE="secrets/datadog_app_key.txt" +DRY_RUN=0 + +# -------- args -------- +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --site) DD_SITE="$2"; shift 2 ;; + --dir) MON_DIR="$2"; shift 2 ;; + -h|--help) + sed -n '2,24p' "$0"; exit 0 ;; + *) + echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +BASE="https://${DD_SITE}" + +# -------- preflight -------- +command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; } +command -v curl >/dev/null 2>&1 || { echo "curl is required" >&2; exit 1; } + +if [ ! -s "$API_KEY_FILE" ]; then + echo "missing or empty API key file: $API_KEY_FILE" >&2 + exit 1 +fi +if [ ! -s "$APP_KEY_FILE" ]; then + cat >&2 <&2 + exit 1 +fi + +shopt -s nullglob +files=("$MON_DIR"/*.json) +if [ ${#files[@]} -eq 0 ]; then + echo "no monitor JSON files in $MON_DIR" >&2 + exit 0 +fi + +exit_code=0 + +for file in "${files[@]}"; do + echo + echo "==> $file" + + if ! jq empty "$file" 2>/dev/null; then + echo " invalid JSON, skipping" >&2 + exit_code=1 + continue + fi + + existing_id=$(jq -r '.id // empty' "$file") + + if [ -n "$existing_id" ]; then + method="PUT" + url="${BASE}/api/v1/monitor/${existing_id}" + echo " updating existing monitor id=${existing_id}" + else + method="POST" + url="${BASE}/api/v1/monitor" + echo " creating new monitor" + fi + + if [ "$DRY_RUN" -eq 1 ]; then + echo " [dry-run] would ${method} ${url}" + continue + fi + + response_file=$(mktemp) + payload=$(jq 'del(.id)' "$file") + + curl_status=$(curl -sS -o "$response_file" -w "%{http_code}" \ + -X "$method" "$url" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: ${DD_API_KEY}" \ + -H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \ + --data-binary "$payload" || echo "000") + + if [ "$curl_status" -lt 200 ] || [ "$curl_status" -ge 300 ]; then + echo " ✗ HTTP $curl_status" + sed 's/^/ /' "$response_file" >&2 + exit_code=1 + else + if [ "$method" = "POST" ]; then + # Monitor ids are integers; preserve type when writing back. + new_id=$(jq -r '.id // empty' "$response_file") + if [ -n "$new_id" ]; then + tmp=$(mktemp) + jq --argjson id "$new_id" '. + {id: $id}' "$file" > "$tmp" && mv "$tmp" "$file" + echo " ✓ created (id=${new_id}), id written back to $file" + else + echo " ✓ created but response had no id (??); body:" + sed 's/^/ /' "$response_file" + exit_code=1 + fi + else + echo " ✓ updated" + fi + fi + + rm -f "$response_file" +done + +echo +if [ "$exit_code" -eq 0 ]; then + echo "All monitors applied." +else + echo "Finished with errors. See output above." +fi +exit "$exit_code" diff --git a/secrets/.gitkeep b/secrets/.gitkeep new file mode 100644 index 0000000000..e69de29bb2