From 95f2ec3e405bfe35325a60e1ce1a9fbbd6ec8461 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Feb 2023 06:02:10 -0500 Subject: [PATCH] Add new status columns to list queries --- ...tions_awaiting_audio_final_verification.rs | 8 ++ ...y_productions_awaiting_audio_generation.rs | 8 ++ ...s_story_productions_awaiting_greenlight.rs | 8 ++ ...y_productions_awaiting_image_generation.rs | 8 ++ ...tory_productions_awaiting_llm_rendition.rs | 8 ++ ..._news_story_productions_ready_for_debut.rs | 8 ++ .../list/news_story_production_item.rs | 10 +- .../up.sql | 6 +- .../2023-02-11-010711_news_stories/up.sql | 4 +- .../src/workers/news_stories/phase_2/mod.rs | 4 +- .../gpt_prompts/mod.rs | 1 + .../gpt_prompts/news_article_prompt.rs | 25 ++++ .../main_loop.rs | 84 +++++++++++ .../news_story_llm_category_summary/mod.rs | 3 + .../process_single_item.rs | 135 ++++++++++++++++++ .../gpt_prompts/mod.rs | 1 + .../gpt_prompts/news_article_prompt.rs | 25 ++++ .../news_story_llm_title_summary/main_loop.rs | 84 +++++++++++ .../news_story_llm_title_summary/mod.rs | 3 + .../process_single_item.rs | 135 ++++++++++++++++++ 20 files changed, 561 insertions(+), 7 deletions(-) create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/mod.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/news_article_prompt.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/main_loop.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/mod.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/mod.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/news_article_prompt.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/main_loop.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/mod.rs create mode 100644 crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs index 9e1d52070b..40910855f0 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs @@ -20,8 +20,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs index 17023041d3..be11da2407 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs @@ -20,8 +20,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs index bd1a363c86..34113c85e9 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs @@ -20,8 +20,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_image_generation.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_image_generation.rs index c346a2e691..9900f70598 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_image_generation.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_image_generation.rs @@ -25,8 +25,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs index 9668c2e79f..47ed8362af 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs @@ -25,8 +25,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs index f3c13e905f..81b9fec113 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs @@ -20,8 +20,16 @@ SELECT web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, + + llm_title_summary_status as `llm_title_summary_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_title_summary_attempts, + + llm_categorization_status as `llm_categorization_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_categorization_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_status as `image_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, image_generation_attempts diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs index d21922e0f8..4493bef1b8 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs @@ -12,10 +12,18 @@ pub struct NewsStoryProductionItem { // Overall production status, which depends on all other statuses pub overall_production_status: AwaitableJobStatus, - // LLM rendition task + // LLM main text rendition task pub llm_rendition_status: AwaitableJobStatus, pub llm_rendition_attempts: i64, + // LLM title summary task + pub llm_title_summary_status: AwaitableJobStatus, + pub llm_title_summary_attempts: i64, + + // LLM categorization task + pub llm_categorization_status: AwaitableJobStatus, + pub llm_categorization_attempts: i64, + // Audio generation tasks (farmed out to 1:n work items) pub audio_generation_status: AwaitableJobStatus, diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql index 31681508ca..3e439f0037 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql @@ -22,7 +22,7 @@ CREATE TABLE news_story_productions ( maybe_summary_news_title TEXT, -- Comma-separated categories for the story. Generated by LLM. - maybe_summary_categories TEXT, + maybe_categorization TEXT, -- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure -- reason, this is why. @@ -49,10 +49,10 @@ CREATE TABLE news_story_productions ( -- Classifying the categories with LLM/GPT -- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done" - llm_category_summary_status TEXT DEFAULT "not_ready" NOT NULL, + llm_categorization_status TEXT DEFAULT "not_ready" NOT NULL, -- Number of LLM/GPT category summary attempts that have been made - llm_category_summary_attempts INT DEFAULT 0 NOT NULL, + llm_categorization_attempts INT DEFAULT 0 NOT NULL, -- TTS for the audio with FakeYou -- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done" diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql index a3e14a7506..ae2f88475e 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql @@ -19,10 +19,10 @@ CREATE TABLE news_stories ( original_news_title TEXT NOT NULL, -- LLM renditioned news title, short. Under 60 characters. - --summary_news_title TEXT NOT NULL, + summary_news_title TEXT NOT NULL, -- Comma-separated categories for the story. Generated by LLM. - --summary_categories TEXT NOT NULL, + llm_categorization TEXT NOT NULL, -- Total number of discrete audio files for this story audio_file_count INTEGER NOT NULL, diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/mod.rs index 8099027312..5eb6ef30c8 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/mod.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/mod.rs @@ -1,2 +1,4 @@ -pub mod news_story_llm_rendition; pub mod news_story_image_generation; +pub mod news_story_llm_category_summary; +pub mod news_story_llm_rendition; +pub mod news_story_llm_title_summary; diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/mod.rs new file mode 100644 index 0000000000..a9e94b3ce3 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/mod.rs @@ -0,0 +1 @@ +pub mod news_article_prompt; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/news_article_prompt.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/news_article_prompt.rs new file mode 100644 index 0000000000..cd2ec80cd4 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/gpt_prompts/news_article_prompt.rs @@ -0,0 +1,25 @@ + +// TODO: We'll add more types of prompt templates in the future. + +pub struct NewsArticlePrompt { + // TODO: Add other details (title, etc.) + paragraphs: Vec, +} + +impl NewsArticlePrompt { + + pub fn new(paragraphs: &Vec) -> Self { + Self { + paragraphs: paragraphs.clone(), + } + } + + // TODO: This will need a lot of work and experimentation. + pub fn make_prompt(&self) -> String { + let start = "The following is a news article:\n\n"; + + let end = "\n\nPlease reword the above article as if you were a newscaster presenting the news. Spell out any abbreviations and numbers"; + + format!("{}{}{}", start, &self.paragraphs.join("\n\n"), end) + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/main_loop.rs new file mode 100644 index 0000000000..165a32f727 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/main_loop.rs @@ -0,0 +1,84 @@ +use crate::shared_state::job_state::JobState; +use crate::workers::news_stories::phase_2::news_story_llm_rendition::process_single_item::process_single_item; +use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus; +use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus; +use log::{debug, error, info}; +use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_llm_rendition::list_news_story_productions_awaiting_llm_rendition; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +/// Project scraped, summarize the categories. +pub async fn news_story_llm_category_summary_main_loop(job_state: Arc) { + loop { + debug!("news_story_llm_category_summary_main_loop main loop"); + + single_job_loop_iteration(&job_state).await; + + debug!("llm_category_summary loop finished; waiting..."); + thread::sleep(Duration::from_secs(1)); + } +} + +// NB: No failures at this level, because we don't want to prevent progress on a stuck feed. +async fn single_job_loop_iteration(job_state: &Arc) { + let statuses = vec![AwaitableJobStatus::ReadyWaiting, AwaitableJobStatus::RetryablyFailed]; + for status in statuses { + rendition_jobs_of_status(status, job_state).await; + } +} + +async fn rendition_jobs_of_status(status: AwaitableJobStatus, job_state: &Arc) { + const BATCH_SIZE : i64 = 10; + + let mut last_id = 0; + let mut failure_count = 0; + + loop { + // NB: Protect sqlite from contention. + thread::sleep(Duration::from_millis(500)); + + debug!("web_rendition querying {:?} targets from id > {} ...", &status, last_id); + + let query_result = list_news_story_productions_awaiting_llm_rendition( + status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await; + + let targets = match query_result { + Ok(targets) => targets, + Err(err) => { + error!("failure querying batch: {:?}", err); + + failure_count += 1; + + // NB: Don't starve progress. + if failure_count > 2 { + failure_count = 0; + last_id += 1; + } else if failure_count > 3 { + return; + } + + continue; + } + }; + + failure_count = 0; + + if targets.is_empty() { + return; // Done with batches. + } + + for target in targets { + debug!("Rendition for target: {:?}", target.original_news_canonical_url); + + let result = process_single_item(&target, job_state).await; + if let Err(err) = result { + error!("Error processing target: {:?}", err); + } + + last_id = target.id; + + thread::sleep(Duration::from_secs(5)); // NB: We probably (?) need to throttle API calls to OpenAI + } + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/mod.rs new file mode 100644 index 0000000000..5c3c9d6cc8 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/mod.rs @@ -0,0 +1,3 @@ +pub mod gpt_prompts; +pub mod main_loop; +pub mod process_single_item; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs new file mode 100644 index 0000000000..c6219a34c0 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_category_summary/process_single_item.rs @@ -0,0 +1,135 @@ +use async_openai::Client; +use async_openai::error::OpenAIError; +use async_openai::types::CreateCompletionRequestArgs; +use crate::persistence::load_scraped_result::load_scraped_result; +use crate::persistence::rendition_data::RenditionData; +use crate::persistence::save_directory::SaveDirectory; +use crate::shared_state::job_state::JobState; +use crate::workers::news_stories::phase_2::news_story_llm_rendition::gpt_prompts::news_article_prompt::NewsArticlePrompt; +use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus; +use errors::{anyhow, AnyhowResult}; +use log::{error, info, warn}; +use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem; +use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_llm_rendition_status::{Args, update_news_story_llm_rendition_status}; +use std::sync::Arc; +use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle; + +pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc) -> AnyhowResult<()> { + + let scraping_result = load_scraped_result( + &target.original_news_canonical_url, + &job_state.save_directory).await?; + + let prompt = NewsArticlePrompt::new(&scraping_result.paragraphs); + let prompt = prompt.make_prompt(); + + let rendition_result = call_openai_gpt( + &target.original_news_canonical_url, + &prompt, + &job_state.openai_client + ).await; + + let llm_rendition_attempts = target.llm_rendition_attempts + 1; + + let rendition_data = match rendition_result { + Err(err) => { + error!("Error using GPT: {:?}", err); + + match &err { + WrappedOpenAiError::Other(_) => {} + WrappedOpenAiError::OpenAiError(inner) => { + match inner { + OpenAIError::Reqwest(e) => { error!("Request error: {:?}", e) } + OpenAIError::ApiError(e) => { error!("API error: {:?}", e) } + OpenAIError::JSONDeserialize(e) => { error!("JSON error: {:?}", e) } + OpenAIError::FileSaveError(e) => { error!("File save error: {:?}", e) } + OpenAIError::FileReadError(e) => { error!("File read error: {:?}", e) } + OpenAIError::StreamError(e) => { error!("Stream error: {:?}", e) } + OpenAIError::InvalidArgument(e) => { error!("Invalid args error: {:?}", e) } + } + } + } + + update_news_story_llm_rendition_status(Args { + news_story_token: &target.news_story_token, + is_successful: false, + llm_rendition_attempts, + sqlite_pool: &job_state.sqlite_pool, + }).await?; // NB: If these queries fail, we could get stuck in retry hell. + + return Err(anyhow!("OpenAI Rendition Error: {:?}", err)); + } + Ok(rendition_data) => rendition_data, + }; + + update_news_story_llm_rendition_status(Args { + news_story_token: &target.news_story_token, + is_successful: true, + llm_rendition_attempts, + sqlite_pool: &job_state.sqlite_pool, + }).await?; // NB: If these queries fail, we could get stuck in retry hell. + +// update_web_rendition_target(Args { +// canonical_url: &target.original_news_canonical_url, +// rendition_status: RenditionStatus::Success, +// rendition_attempts: target.llm_rendition_attempts + 1, +// sqlite_pool: &job_state.sqlite_pool, +// }).await?; // NB: If these queries fail, we could get stuck. + + { + let yaml_filename = job_state.save_directory + .rendition_file_for_webpage_url(&target.original_news_canonical_url)?; + rendition_data.write_to_yaml_file(yaml_filename)?; + } + + Ok(()) +} + +#[derive(Debug)] +pub enum WrappedOpenAiError { + OpenAiError(OpenAIError), + Other(String), +} + +async fn call_openai_gpt(url: &str, prompt: &str, openai_client: &Arc) -> Result { + warn!("Building OpenAI Request..."); + let request = CreateCompletionRequestArgs::default() + .model("text-davinci-003") + .prompt(prompt) + .max_tokens(2500_u16) + .build() + .map_err(|err| { + error!("Error building request: {:?}", err); + WrappedOpenAiError::Other(format!("error: {:?}", err)) + })?; + + warn!("Making OpenAI Request..."); + let response = openai_client + .completions() + .create(request) + .await + .map_err(|err| { + error!("OpenAI Error: {:?}", err); + WrappedOpenAiError::OpenAiError(err) + })?; + + info!("Open AI response: {:?}", response); + + let rendition_text = response.choices.get(0) + .map(|option| option.text.clone()) + .unwrap_or("".to_string()); + + Ok(RenditionData { + original_content_url: url.to_string(), + original_prompt: prompt.to_string(), + response: rendition_text, + }) +} + +//fn next_status(rendition_attempts: i64) -> RenditionStatus { +// if rendition_attempts >= 2 { +// RenditionStatus::PermanentlyFailed +// } else { +// RenditionStatus::Failed +// } +//} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/mod.rs new file mode 100644 index 0000000000..a9e94b3ce3 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/mod.rs @@ -0,0 +1 @@ +pub mod news_article_prompt; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/news_article_prompt.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/news_article_prompt.rs new file mode 100644 index 0000000000..cd2ec80cd4 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/gpt_prompts/news_article_prompt.rs @@ -0,0 +1,25 @@ + +// TODO: We'll add more types of prompt templates in the future. + +pub struct NewsArticlePrompt { + // TODO: Add other details (title, etc.) + paragraphs: Vec, +} + +impl NewsArticlePrompt { + + pub fn new(paragraphs: &Vec) -> Self { + Self { + paragraphs: paragraphs.clone(), + } + } + + // TODO: This will need a lot of work and experimentation. + pub fn make_prompt(&self) -> String { + let start = "The following is a news article:\n\n"; + + let end = "\n\nPlease reword the above article as if you were a newscaster presenting the news. Spell out any abbreviations and numbers"; + + format!("{}{}{}", start, &self.paragraphs.join("\n\n"), end) + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/main_loop.rs new file mode 100644 index 0000000000..b7979d5b62 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/main_loop.rs @@ -0,0 +1,84 @@ +use crate::shared_state::job_state::JobState; +use crate::workers::news_stories::phase_2::news_story_llm_rendition::process_single_item::process_single_item; +use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus; +use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus; +use log::{debug, error, info}; +use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_llm_rendition::list_news_story_productions_awaiting_llm_rendition; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +/// Project scraped, summarize the title. +pub async fn news_story_llm_title_summary_main_loop(job_state: Arc) { + loop { + debug!("news_story_llm_title_summary_main_loop main loop"); + + single_job_loop_iteration(&job_state).await; + + debug!("llm_title_summary loop finished; waiting..."); + thread::sleep(Duration::from_secs(1)); + } +} + +// NB: No failures at this level, because we don't want to prevent progress on a stuck feed. +async fn single_job_loop_iteration(job_state: &Arc) { + let statuses = vec![AwaitableJobStatus::ReadyWaiting, AwaitableJobStatus::RetryablyFailed]; + for status in statuses { + rendition_jobs_of_status(status, job_state).await; + } +} + +async fn rendition_jobs_of_status(status: AwaitableJobStatus, job_state: &Arc) { + const BATCH_SIZE : i64 = 10; + + let mut last_id = 0; + let mut failure_count = 0; + + loop { + // NB: Protect sqlite from contention. + thread::sleep(Duration::from_millis(500)); + + debug!("web_rendition querying {:?} targets from id > {} ...", &status, last_id); + + let query_result = list_news_story_productions_awaiting_llm_rendition( + status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await; + + let targets = match query_result { + Ok(targets) => targets, + Err(err) => { + error!("failure querying batch: {:?}", err); + + failure_count += 1; + + // NB: Don't starve progress. + if failure_count > 2 { + failure_count = 0; + last_id += 1; + } else if failure_count > 3 { + return; + } + + continue; + } + }; + + failure_count = 0; + + if targets.is_empty() { + return; // Done with batches. + } + + for target in targets { + debug!("Rendition for target: {:?}", target.original_news_canonical_url); + + let result = process_single_item(&target, job_state).await; + if let Err(err) = result { + error!("Error processing target: {:?}", err); + } + + last_id = target.id; + + thread::sleep(Duration::from_secs(5)); // NB: We probably (?) need to throttle API calls to OpenAI + } + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/mod.rs new file mode 100644 index 0000000000..5c3c9d6cc8 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/mod.rs @@ -0,0 +1,3 @@ +pub mod gpt_prompts; +pub mod main_loop; +pub mod process_single_item; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs new file mode 100644 index 0000000000..c6219a34c0 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/phase_2/news_story_llm_title_summary/process_single_item.rs @@ -0,0 +1,135 @@ +use async_openai::Client; +use async_openai::error::OpenAIError; +use async_openai::types::CreateCompletionRequestArgs; +use crate::persistence::load_scraped_result::load_scraped_result; +use crate::persistence::rendition_data::RenditionData; +use crate::persistence::save_directory::SaveDirectory; +use crate::shared_state::job_state::JobState; +use crate::workers::news_stories::phase_2::news_story_llm_rendition::gpt_prompts::news_article_prompt::NewsArticlePrompt; +use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus; +use errors::{anyhow, AnyhowResult}; +use log::{error, info, warn}; +use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem; +use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_llm_rendition_status::{Args, update_news_story_llm_rendition_status}; +use std::sync::Arc; +use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle; + +pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc) -> AnyhowResult<()> { + + let scraping_result = load_scraped_result( + &target.original_news_canonical_url, + &job_state.save_directory).await?; + + let prompt = NewsArticlePrompt::new(&scraping_result.paragraphs); + let prompt = prompt.make_prompt(); + + let rendition_result = call_openai_gpt( + &target.original_news_canonical_url, + &prompt, + &job_state.openai_client + ).await; + + let llm_rendition_attempts = target.llm_rendition_attempts + 1; + + let rendition_data = match rendition_result { + Err(err) => { + error!("Error using GPT: {:?}", err); + + match &err { + WrappedOpenAiError::Other(_) => {} + WrappedOpenAiError::OpenAiError(inner) => { + match inner { + OpenAIError::Reqwest(e) => { error!("Request error: {:?}", e) } + OpenAIError::ApiError(e) => { error!("API error: {:?}", e) } + OpenAIError::JSONDeserialize(e) => { error!("JSON error: {:?}", e) } + OpenAIError::FileSaveError(e) => { error!("File save error: {:?}", e) } + OpenAIError::FileReadError(e) => { error!("File read error: {:?}", e) } + OpenAIError::StreamError(e) => { error!("Stream error: {:?}", e) } + OpenAIError::InvalidArgument(e) => { error!("Invalid args error: {:?}", e) } + } + } + } + + update_news_story_llm_rendition_status(Args { + news_story_token: &target.news_story_token, + is_successful: false, + llm_rendition_attempts, + sqlite_pool: &job_state.sqlite_pool, + }).await?; // NB: If these queries fail, we could get stuck in retry hell. + + return Err(anyhow!("OpenAI Rendition Error: {:?}", err)); + } + Ok(rendition_data) => rendition_data, + }; + + update_news_story_llm_rendition_status(Args { + news_story_token: &target.news_story_token, + is_successful: true, + llm_rendition_attempts, + sqlite_pool: &job_state.sqlite_pool, + }).await?; // NB: If these queries fail, we could get stuck in retry hell. + +// update_web_rendition_target(Args { +// canonical_url: &target.original_news_canonical_url, +// rendition_status: RenditionStatus::Success, +// rendition_attempts: target.llm_rendition_attempts + 1, +// sqlite_pool: &job_state.sqlite_pool, +// }).await?; // NB: If these queries fail, we could get stuck. + + { + let yaml_filename = job_state.save_directory + .rendition_file_for_webpage_url(&target.original_news_canonical_url)?; + rendition_data.write_to_yaml_file(yaml_filename)?; + } + + Ok(()) +} + +#[derive(Debug)] +pub enum WrappedOpenAiError { + OpenAiError(OpenAIError), + Other(String), +} + +async fn call_openai_gpt(url: &str, prompt: &str, openai_client: &Arc) -> Result { + warn!("Building OpenAI Request..."); + let request = CreateCompletionRequestArgs::default() + .model("text-davinci-003") + .prompt(prompt) + .max_tokens(2500_u16) + .build() + .map_err(|err| { + error!("Error building request: {:?}", err); + WrappedOpenAiError::Other(format!("error: {:?}", err)) + })?; + + warn!("Making OpenAI Request..."); + let response = openai_client + .completions() + .create(request) + .await + .map_err(|err| { + error!("OpenAI Error: {:?}", err); + WrappedOpenAiError::OpenAiError(err) + })?; + + info!("Open AI response: {:?}", response); + + let rendition_text = response.choices.get(0) + .map(|option| option.text.clone()) + .unwrap_or("".to_string()); + + Ok(RenditionData { + original_content_url: url.to_string(), + original_prompt: prompt.to_string(), + response: rendition_text, + }) +} + +//fn next_status(rendition_attempts: i64) -> RenditionStatus { +// if rendition_attempts >= 2 { +// RenditionStatus::PermanentlyFailed +// } else { +// RenditionStatus::Failed +// } +//}