From a08bc3fa644e166c8be3b73830ee87ad944eed95 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Wed, 22 Feb 2023 07:51:53 -0500
Subject: [PATCH] include outputs in table, cleanup bugs

---
 .../process_single_item.rs                    | 23 +++++++++++++++++-
 .../process_single_item.rs                    | 24 ++++++++++++++++++-
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs
index eee70f2d1a..0fc9562a17 100644
--- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs
+++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs
@@ -64,10 +64,31 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A
     Ok(rendition_data) => rendition_data,
   };
 
+  let categorization = rendition_data.response.trim();
+
+  // NB: GPT often completes the prompt we used itself, so we grab the last line (presumably output)
+  let categorization = categorization.split("\n")
+      .map(|line| line.trim().to_string())
+      .filter(|line| line.len() > 5)
+      .last()
+      .unwrap_or("".to_string());
+
+  if categorization.len() < 5 {
+    update_news_story_llm_categorization_status(Args {
+      news_story_token: &target.news_story_token,
+      is_successful: false,
+      maybe_categorization: None,
+      llm_categorization_attempts,
+      sqlite_pool: &job_state.sqlite_pool,
+    }).await?; // NB: If these queries fail, we could get stuck in retry hell.
+
+    return Err(anyhow!("too short"));
+  }
+
   update_news_story_llm_categorization_status(Args {
     news_story_token: &target.news_story_token,
     is_successful: true,
-    maybe_categorization: None,
+    maybe_categorization: Some(categorization.to_string()),
     llm_categorization_attempts,
     sqlite_pool: &job_state.sqlite_pool,
   }).await?; // NB: If these queries fail, we could get stuck in retry hell.
diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs
index 5d1da1f422..a500474e4d 100644
--- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs
+++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs
@@ -68,10 +68,32 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A
     Ok(rendition_data) => rendition_data,
   };
 
+  let title_summary = rendition_data.response.trim();
+
+  // NB: GPT often completes the prompt we used itself, so we grab the last line (presumably output)
+  let title_summary = title_summary.split("\n")
+      .map(|line| line.trim().to_string())
+      .filter(|line| line.len() > 5)
+      .last()
+      .unwrap_or("".to_string());
+
+  // NB: GPT often includes quotes in the output.
+  let title_summary = title_summary.replace("\"", "");
+
+  if title_summary.len() < 5 {
+    update_news_story_llm_title_summary_status(Args {
+      news_story_token: &target.news_story_token,
+      is_successful: false,
+      maybe_title_summary: None,
+      llm_title_summary_attempts,
+      sqlite_pool: &job_state.sqlite_pool,
+    }).await?; // NB: If these queries fail, we could get stuck in retry hell.
+  }
+
   update_news_story_llm_title_summary_status(Args {
     news_story_token: &target.news_story_token,
     is_successful: true,
-    maybe_title_summary: None,
+    maybe_title_summary: Some(title_summary.to_string()),
     llm_title_summary_attempts,
     sqlite_pool: &job_state.sqlite_pool,
   }).await?; // NB: If these queries fail, we could get stuck in retry hell.