diff --git a/Cargo.lock b/Cargo.lock index 6d0af7daac..11244c93fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,7 +763,9 @@ dependencies = [ "imgui-glium-renderer", "imgui-winit-support", "log", + "once_cell", "path-absolutize", + "regex", "reqwest", "rss", "serde", diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs new file mode 100644 index 0000000000..9438b26740 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs @@ -0,0 +1,39 @@ +use crate::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem; +use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus; +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use tokens::tokens::news_stories::NewsStoryToken; + +pub async fn list_news_story_productions_awaiting_audio_generation( + last_id: i64, + limit: i64, + sqlite_pool: &SqlitePool, +) -> AnyhowResult> { + + let query = sqlx::query_as!( + NewsStoryProductionItem, + r#" +SELECT + id, + news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, + original_news_canonical_url, + overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, + llm_rendition_attempts, + audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus` +FROM news_story_productions +WHERE + audio_generation_status = "ready_waiting" + AND id > ? +ORDER BY id ASC +LIMIT ? + "#, + last_id, + limit, + ); + + let records = query.fetch_all(sqlite_pool) + .await?; + + Ok(records) +} diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/mod.rs index 064df0860a..ae6e18e530 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/mod.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/mod.rs @@ -1,3 +1,4 @@ +pub mod list_news_story_productions_awaiting_audio_generation; pub mod list_news_story_productions_awaiting_greenlight; pub mod list_news_story_productions_awaiting_llm_rendition; pub mod news_story_production_item; diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index 39b798ddc3..0f222b44cb 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -30,7 +30,9 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat dotenv = "0.15.0" easyenv = "0.0.6" log = "0.4.14" +once_cell = "1.9.0" path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet +regex = "1.5.4" strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc. # SQlite Database diff --git a/crates/programs/aichatbot-sidecar/src/persistence/rendition_data.rs b/crates/programs/aichatbot-sidecar/src/persistence/rendition_data.rs index 258c312409..210fc9c0bb 100644 --- a/crates/programs/aichatbot-sidecar/src/persistence/rendition_data.rs +++ b/crates/programs/aichatbot-sidecar/src/persistence/rendition_data.rs @@ -1,3 +1,5 @@ +use std::path::Path; +use errors::AnyhowResult; /// For serializing OpenAI GPT requests. #[derive(Clone, Debug, Serialize, Deserialize)] @@ -11,3 +13,18 @@ pub struct RenditionData { /// Response from OpenAI pub response: String, } + +impl RenditionData { + + pub fn try_from_yaml_file>(filename: P) -> AnyhowResult { + let file = std::fs::File::open(filename)?; + let object = serde_yaml::from_reader(file)?; + Ok(object) + } + + pub fn write_to_yaml_file>(&self, filename: P) -> AnyhowResult<()> { + let mut file = std::fs::File::create(filename)?; + serde_yaml::to_writer(file, &self)?; + Ok(()) + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/fakeyou_enrichment/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/fakeyou_audio_generation/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/fakeyou_enrichment/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/fakeyou_audio_generation/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/mod.rs index 618bf0ff53..8187348141 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/mod.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/mod.rs @@ -1,4 +1,4 @@ -pub mod fakeyou_enrichment; +pub mod fakeyou_audio_generation; pub mod news_stories; pub mod web_content_scraping; pub mod web_index_ingestion; diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/mod.rs index 12acd763cd..3e35082b1f 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/mod.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/mod.rs @@ -1,2 +1,4 @@ +pub mod news_story_audio_final_verification; +pub mod news_story_audio_preprocessing; pub mod news_story_greenlighting; pub mod news_story_llm_rendition; diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_final_verification/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_final_verification/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/main_loop.rs new file mode 100644 index 0000000000..703ae850f9 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/main_loop.rs @@ -0,0 +1,74 @@ +use crate::shared_state::job_state::JobState; +use crate::workers::news_stories::news_story_audio_preprocessing::process_single_item::process_single_item; +use log::{debug, error}; +use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_audio_generation::list_news_story_productions_awaiting_audio_generation; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +/// Split text into text segments fit for rendering. +pub async fn news_story_audio_preprocessing_main_loop(job_state: Arc) { + loop { + debug!("news_story_audio_preprocessing main loop"); + + query_and_process_all_batches(&job_state).await; + + debug!("news_story_audio_preprocessing loop finished; waiting..."); + thread::sleep(Duration::from_secs(5)); + } +} + +async fn query_and_process_all_batches(job_state: &Arc) { + const BATCH_SIZE : i64 = 10; + + let mut last_id = 0; + let mut failure_count = 0; + + loop { + // NB: Protect sqlite from contention. + thread::sleep(Duration::from_millis(500)); + + debug!("querying targets from id > {} ...", last_id); + + let query_result = list_news_story_productions_awaiting_audio_generation( + last_id, BATCH_SIZE, &job_state.sqlite_pool).await; + + let targets = match query_result { + Ok(targets) => targets, + Err(err) => { + error!("failure querying batch: {:?}", err); + + failure_count += 1; + + // NB: Don't starve progress. + if failure_count > 2 { + failure_count = 0; + last_id += 1; + } else if failure_count > 3 { + return; + } + + continue; + } + }; + + failure_count = 0; + + if targets.is_empty() { + return; // Done with batches. + } + + for target in targets { + debug!("Split audio for target: {:?}", target.original_news_canonical_url); + + let result = process_single_item(&target, job_state).await; + if let Err(err) = result { + error!("Error processing target: {:?}", err); + } + + last_id = target.id; + + thread::sleep(Duration::from_millis(100)); + } + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/mod.rs new file mode 100644 index 0000000000..69b2dcb3e8 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/mod.rs @@ -0,0 +1,2 @@ +pub mod main_loop; +pub mod process_single_item; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/process_single_item.rs new file mode 100644 index 0000000000..96e57d944e --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_audio_preprocessing/process_single_item.rs @@ -0,0 +1,139 @@ +use std::sync::Arc; +use once_cell::sync::Lazy; +use regex::Regex; +use errors::AnyhowResult; +use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem; +use crate::persistence::rendition_data::RenditionData; +use crate::shared_state::job_state::JobState; + +static NEWLINE_REGEX : Lazy = Lazy::new(|| { + Regex::new(r"[\r\n]+").expect("regex should parse") +}); + +static SPACE_REGEX : Lazy = Lazy::new(|| { + Regex::new(r"[\s]+").expect("regex should parse") +}); + +// Regex at current version doesn't support look-around. UGH. +//static SPACE_REGEX : Lazy = Lazy::new(|| { +// Regex::new(r"(?<=.{150,}[\.?])").expect("regex should parse") +//}); + +pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc) -> AnyhowResult<()> { + let rendition_data = { + let yaml_filename = job_state.save_directory + .rendition_file_for_webpage_url(&target.original_news_canonical_url)?; + RenditionData::try_from_yaml_file(yaml_filename)? + }; + + let text = rendition_data.response.trim(); + + let mut speakable_portions = Vec::new(); + + for paragraph in NEWLINE_REGEX.split(text) { + speakable_portions.push(paragraph.to_string()); + + if paragraph.len() > 512 { + + } + } + + + Ok(()) +} + +fn split_into_manageable_portions(text: &str) -> Vec { + let trimmed_text = text.trim(); + + let mut speakable_portions = Vec::new(); + + for paragraph in NEWLINE_REGEX.split(trimmed_text) { + let trimmed_paragraph = paragraph.trim(); + if trimmed_paragraph.is_empty() { + continue; + } + + if trimmed_paragraph.len() > 250 { + speakable_portions.extend(split_paragraph(&trimmed_paragraph)); + } else { + speakable_portions.push(paragraph.to_string()); + } + } + + speakable_portions +} + +fn split_paragraph(text: &str) -> Vec { + let trimmed_text = text.trim(); + + let mut speakable_portions = Vec::new(); + + // TODO: Use a binary search like algorithm instead. Or better, understand the underlying NLP task. + let mut word_buffer = Vec::new(); + let mut word_counter = 0; + let mut character_counter = 0; + let mut should_cut = false; + + for wordlike in SPACE_REGEX.split(trimmed_text) { + let wordlike_trimmed = wordlike.trim(); + + word_counter += 1; + character_counter += wordlike_trimmed.len(); + + word_buffer.push(wordlike_trimmed.to_string()); + + if word_counter > 70 || character_counter > 400 { + should_cut = true; + } + + if should_cut && ( + wordlike_trimmed.ends_with(".") + || wordlike_trimmed.ends_with("?") + || wordlike_trimmed.ends_with("\"") + || wordlike_trimmed.ends_with("!")) { + + speakable_portions.push(word_buffer.join(" ")); + word_buffer.clear(); + + word_counter = 0; + character_counter = 0; + should_cut = false; + } + } + + if word_buffer.len() > 0 { + speakable_portions.push(word_buffer.join(" ")); + } + + speakable_portions +} + +#[cfg(test)] +mod tests { + use crate::workers::news_stories::news_story_audio_preprocessing::process_single_item::split_into_manageable_portions; + + #[test] + fn test_split_long_text() { + let text = concat!( + r#"Arkansas Governor Sarah Huckabee Sanders gave a vehement speech Tuesday night in response to President Joe Biden's State of the Union address, "#, + r#"critically comparing herself to the President. Sanders, who is forty years old, connected her age to Biden's, who is eighty, saying that this shows "#, + r#"a divide between the two in generations. She portrayed Biden as unfit to be commander-in-chief and described the current political condition as a "#, + r#"struggle between "normal and crazy". Prior to her line of criticism, Sanders gave a nod to newly established Republican control of the House and said "#, + r#"that Speaker Kevin McCarthy and Senate Republican Leader Mitch McConnell would hold the Biden administration to its word. Serving as White House press "#, + r#"secretary under President Donald Trump, Sanders talked about her memories of a Christmas trip the two took together to Iraq, how the troops cheered upon "#, + r#"their entrance, and the Unity of Americans in the service of their country. On education policy, Sarah Huckabee Sanders spoke about the need for "#, + r#"instruction and not indoctrination for students of all backgrounds, referring to an executive order she passed last year. In statements released ahead of "#, + r#"the address, McConnell and McCarthy both commended the Youngest Governor in the nation for her leadership skills. The response to the State of the Union "#, + r#"was provided by Palin Huckabee Sanders."# + ); + + // TODO: This test isn't very well thought out. We should make better assertions and have more tests. + let results = split_into_manageable_portions(&text); + + assert_eq!(results.len(), 3); // Total paragraph count + + for paragraph in results.iter() { + assert!(results.len() < 450); + } + } +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_llm_rendition/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_llm_rendition/process_single_item.rs index a7c2ebed76..663e4f63a0 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_llm_rendition/process_single_item.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_llm_rendition/process_single_item.rs @@ -71,8 +71,7 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A { let yaml_filename = job_state.save_directory .rendition_file_for_webpage_url(&target.original_news_canonical_url)?; - let mut file = std::fs::File::create(&yaml_filename)?; - serde_yaml::to_writer(file, &rendition_data)?; + rendition_data.write_to_yaml_file(yaml_filename)?; } Ok(())