From 86c86efd4c9dc0f1901ff4b7ee4f242800187db3 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Tue, 15 Jul 2025 23:56:28 -0400
Subject: [PATCH] classify cloudflare errors

---
 .../storyteller_client/src/error/api_error.rs |  5 ++
 .../src/utils/filter_bad_response.rs          | 59 +++++++++++--------
 2 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/crates/api_clients/storyteller_client/src/error/api_error.rs b/crates/api_clients/storyteller_client/src/error/api_error.rs
index d0bf2c6b61..8895f7e97d 100644
--- a/crates/api_clients/storyteller_client/src/error/api_error.rs
+++ b/crates/api_clients/storyteller_client/src/error/api_error.rs
@@ -26,6 +26,9 @@ pub enum ApiError {
     backend_hostname: Option<String>,
   },
 
+  /// 504. Cloudflare timed out waiting for the backend server.
+  Cloudflare504Timeout(String),
+
   /// A deserialization error with the response.
   DeserializationError(serde_json::Error),
 
@@ -59,6 +62,8 @@ impl Display for ApiError {
       ApiError::InternalServerError {body, backend_hostname} => 
         write!(f, "Internal Server Error; backend hostname: {:?} ; body: {}; ", backend_hostname, body),
       // Server response handling errors
+      ApiError::Cloudflare504Timeout(body) =>
+        write!(f, "Cloudflare 504 Timeout; body: {}", body),
       ApiError::DeserializationError(error) => write!(f, "Deserialization error: {}", error),
       // Network errors
       ApiError::Timeout(msg) => write!(f, "Timeout: {}", msg),
diff --git a/crates/api_clients/storyteller_client/src/utils/filter_bad_response.rs b/crates/api_clients/storyteller_client/src/utils/filter_bad_response.rs
index c446a31582..a052d96f39 100644
--- a/crates/api_clients/storyteller_client/src/utils/filter_bad_response.rs
+++ b/crates/api_clients/storyteller_client/src/utils/filter_bad_response.rs
@@ -13,30 +13,43 @@ use log::error;
 pub async fn filter_bad_response(response: reqwest::Response) -> Result<reqwest::Response, ApiError> {
   let status = response.status();
 
-  if !status.is_success() {
-    // The host in the fleet that handled the request.
-    let backend_hostname= response.headers()
-        .get("x-backend-hostname")
-        .map(|header| header.to_str().unwrap_or_else(|err| {
-          error!("Failed to parse x-backend-hostname header: {:?}", err);
-          "unknown"
-        }))
-        .map(|s| s.to_string());
-    
-    let response_body = match response.text().await {
-      Ok(text) => text.to_string(),
-      Err(err) => format!("Could not read response body: {:?}", err),
-    };
+  if status.is_success() {
+    return Ok(response);
+  }
 
-    return match status.as_u16() {
-      STATUS_401_UNAUTHORIZED => Err(ApiError::Unauthorized(response_body)),
-      STATUS_403_FORBIDDEN => Err(ApiError::Forbidden(response_body)),
-      STATUS_404_NOT_FOUND => Err(ApiError::NotFound(response_body)),
-      STATUS_429_TOO_MANY_REQUESTS => Err(ApiError::TooManyRequests(response_body)),
-      STATUS_500_INTERNAL_SERVER_ERROR => Err(ApiError::InternalServerError { body: response_body, backend_hostname }),
-      _ => Err(ApiError::Other(anyhow!("Bad status code: {}; message: {:?}", status, response_body))),
+  // The host in the fleet that handled the request.
+  let backend_hostname= response.headers()
+      .get("x-backend-hostname")
+      .map(|header| header.to_str().unwrap_or_else(|err| {
+        error!("Failed to parse x-backend-hostname header: {:?}", err);
+        "unknown"
+      }))
+      .map(|s| s.to_string());
+
+  let response_body = match response.text().await {
+    Ok(text) => text.to_string(),
+    Err(err) => format!("Could not read response body: {:?}", err),
+  };
+
+  let is_cloudflare = response_body.contains("cloudflare.com")
+      || response_body.contains("Cloudflare Ray ID");
+
+  if is_cloudflare {
+    if status.as_u16() == 504
+        || response_body.contains("errorcode_504")
+        || response_body.contains("Gateway time-out")
+        || response_body.contains("Error code 504")
+    {
+      return Err(ApiError::Cloudflare504Timeout(response_body));
     }
   }
 
-  Ok(response)
-}
\ No newline at end of file
+  match status.as_u16() {
+    STATUS_401_UNAUTHORIZED => Err(ApiError::Unauthorized(response_body)),
+    STATUS_403_FORBIDDEN => Err(ApiError::Forbidden(response_body)),
+    STATUS_404_NOT_FOUND => Err(ApiError::NotFound(response_body)),
+    STATUS_429_TOO_MANY_REQUESTS => Err(ApiError::TooManyRequests(response_body)),
+    STATUS_500_INTERNAL_SERVER_ERROR => Err(ApiError::InternalServerError { body: response_body, backend_hostname }),
+    _ => Err(ApiError::Other(anyhow!("Bad status code: {}; message: {:?}", status, response_body))),
+  }
+}