text splitting

This commit is contained in:
Brandon Thomas committed 2023-02-13 03:49:01 -05:00
1 parent cc6cf0d6c0
commit 3786a64eb1
13 files changed
+280 -3

No files matched your search

Generated
+2
View File
@@ -763,7 +763,9 @@ dependencies = [
"imgui-glium-renderer",
"imgui-winit-support",
"log",
"once_cell",
"path-absolutize",
"regex",
"reqwest",
"rss",
"serde",
@@ -0,0 +1,39 @@
use crate::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus;
use errors::{anyhow, AnyhowResult};
use sqlx::SqlitePool;
use tokens::tokens::news_stories::NewsStoryToken;
pub async fn list_news_story_productions_awaiting_audio_generation(
last_id: i64,
limit: i64,
sqlite_pool: &SqlitePool,
) -> AnyhowResult<Vec<NewsStoryProductionItem>> {
let query = sqlx::query_as!(
NewsStoryProductionItem,
r#"
SELECT
id,
news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
original_news_canonical_url,
overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
llm_rendition_attempts,
audio_generation_status as `audio_generation_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`
FROM news_story_productions
WHERE
audio_generation_status = "ready_waiting"
AND id > ?
ORDER BY id ASC
LIMIT ?
"#,
last_id,
limit,
);
let records = query.fetch_all(sqlite_pool)
.await?;
Ok(records)
}
@@ -1,3 +1,4 @@
pub mod list_news_story_productions_awaiting_audio_generation;
pub mod list_news_story_productions_awaiting_greenlight;
pub mod list_news_story_productions_awaiting_llm_rendition;
pub mod news_story_production_item;
@@ -30,7 +30,9 @@ clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updat
dotenv = "0.15.0"
easyenv = "0.0.6"
log = "0.4.14"
once_cell = "1.9.0"
path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet
regex = "1.5.4"
strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc.
# SQlite Database
@@ -1,3 +1,5 @@
use std::path::Path;
use errors::AnyhowResult;
/// For serializing OpenAI GPT requests.
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -11,3 +13,18 @@ pub struct RenditionData {
/// Response from OpenAI
pub response: String,
}
impl RenditionData {
pub fn try_from_yaml_file<P: AsRef<Path>>(filename: P) -> AnyhowResult<Self> {
let file = std::fs::File::open(filename)?;
let object = serde_yaml::from_reader(file)?;
Ok(object)
}
pub fn write_to_yaml_file<P: AsRef<Path>>(&self, filename: P) -> AnyhowResult<()> {
let mut file = std::fs::File::create(filename)?;
serde_yaml::to_writer(file, &self)?;
Ok(())
}
}
@@ -1,4 +1,4 @@
pub mod fakeyou_enrichment;
pub mod fakeyou_audio_generation;
pub mod news_stories;
pub mod web_content_scraping;
pub mod web_index_ingestion;
@@ -1,2 +1,4 @@
pub mod news_story_audio_final_verification;
pub mod news_story_audio_preprocessing;
pub mod news_story_greenlighting;
pub mod news_story_llm_rendition;
@@ -0,0 +1,74 @@
use crate::shared_state::job_state::JobState;
use crate::workers::news_stories::news_story_audio_preprocessing::process_single_item::process_single_item;
use log::{debug, error};
use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_audio_generation::list_news_story_productions_awaiting_audio_generation;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
/// Split text into text segments fit for rendering.
pub async fn news_story_audio_preprocessing_main_loop(job_state: Arc<JobState>) {
loop {
debug!("news_story_audio_preprocessing main loop");
query_and_process_all_batches(&job_state).await;
debug!("news_story_audio_preprocessing loop finished; waiting...");
thread::sleep(Duration::from_secs(5));
}
}
async fn query_and_process_all_batches(job_state: &Arc<JobState>) {
const BATCH_SIZE : i64 = 10;
let mut last_id = 0;
let mut failure_count = 0;
loop {
// NB: Protect sqlite from contention.
thread::sleep(Duration::from_millis(500));
debug!("querying targets from id > {} ...", last_id);
let query_result = list_news_story_productions_awaiting_audio_generation(
last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
let targets = match query_result {
Ok(targets) => targets,
Err(err) => {
error!("failure querying batch: {:?}", err);
failure_count += 1;
// NB: Don't starve progress.
if failure_count > 2 {
failure_count = 0;
last_id += 1;
} else if failure_count > 3 {
return;
}
continue;
}
};
failure_count = 0;
if targets.is_empty() {
return; // Done with batches.
}
for target in targets {
debug!("Split audio for target: {:?}", target.original_news_canonical_url);
let result = process_single_item(&target, job_state).await;
if let Err(err) = result {
error!("Error processing target: {:?}", err);
}
last_id = target.id;
thread::sleep(Duration::from_millis(100));
}
}
}
@@ -0,0 +1,2 @@
pub mod main_loop;
pub mod process_single_item;
@@ -0,0 +1,139 @@
use std::sync::Arc;
use once_cell::sync::Lazy;
use regex::Regex;
use errors::AnyhowResult;
use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
use crate::persistence::rendition_data::RenditionData;
use crate::shared_state::job_state::JobState;
static NEWLINE_REGEX : Lazy<Regex> = Lazy::new(|| {
Regex::new(r"[\r\n]+").expect("regex should parse")
});
static SPACE_REGEX : Lazy<Regex> = Lazy::new(|| {
Regex::new(r"[\s]+").expect("regex should parse")
});
// Regex at current version doesn't support look-around. UGH.
//static SPACE_REGEX : Lazy<Regex> = Lazy::new(|| {
// Regex::new(r"(?<=.{150,}[\.?])").expect("regex should parse")
//});
pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc<JobState>) -> AnyhowResult<()> {
let rendition_data = {
let yaml_filename = job_state.save_directory
.rendition_file_for_webpage_url(&target.original_news_canonical_url)?;
RenditionData::try_from_yaml_file(yaml_filename)?
};
let text = rendition_data.response.trim();
let mut speakable_portions = Vec::new();
for paragraph in NEWLINE_REGEX.split(text) {
speakable_portions.push(paragraph.to_string());
if paragraph.len() > 512 {
}
}
Ok(())
}
fn split_into_manageable_portions(text: &str) -> Vec<String> {
let trimmed_text = text.trim();
let mut speakable_portions = Vec::new();
for paragraph in NEWLINE_REGEX.split(trimmed_text) {
let trimmed_paragraph = paragraph.trim();
if trimmed_paragraph.is_empty() {
continue;
}
if trimmed_paragraph.len() > 250 {
speakable_portions.extend(split_paragraph(&trimmed_paragraph));
} else {
speakable_portions.push(paragraph.to_string());
}
}
speakable_portions
}
fn split_paragraph(text: &str) -> Vec<String> {
let trimmed_text = text.trim();
let mut speakable_portions = Vec::new();
// TODO: Use a binary search like algorithm instead. Or better, understand the underlying NLP task.
let mut word_buffer = Vec::new();
let mut word_counter = 0;
let mut character_counter = 0;
let mut should_cut = false;
for wordlike in SPACE_REGEX.split(trimmed_text) {
let wordlike_trimmed = wordlike.trim();
word_counter += 1;
character_counter += wordlike_trimmed.len();
word_buffer.push(wordlike_trimmed.to_string());
if word_counter > 70 || character_counter > 400 {
should_cut = true;
}
if should_cut && (
wordlike_trimmed.ends_with(".")
|| wordlike_trimmed.ends_with("?")
|| wordlike_trimmed.ends_with("\"")
|| wordlike_trimmed.ends_with("!")) {
speakable_portions.push(word_buffer.join(" "));
word_buffer.clear();
word_counter = 0;
character_counter = 0;
should_cut = false;
}
}
if word_buffer.len() > 0 {
speakable_portions.push(word_buffer.join(" "));
}
speakable_portions
}
#[cfg(test)]
mod tests {
use crate::workers::news_stories::news_story_audio_preprocessing::process_single_item::split_into_manageable_portions;
#[test]
fn test_split_long_text() {
let text = concat!(
r#"Arkansas Governor Sarah Huckabee Sanders gave a vehement speech Tuesday night in response to President Joe Biden's State of the Union address, "#,
r#"critically comparing herself to the President. Sanders, who is forty years old, connected her age to Biden's, who is eighty, saying that this shows "#,
r#"a divide between the two in generations. She portrayed Biden as unfit to be commander-in-chief and described the current political condition as a "#,
r#"struggle between "normal and crazy". Prior to her line of criticism, Sanders gave a nod to newly established Republican control of the House and said "#,
r#"that Speaker Kevin McCarthy and Senate Republican Leader Mitch McConnell would hold the Biden administration to its word. Serving as White House press "#,
r#"secretary under President Donald Trump, Sanders talked about her memories of a Christmas trip the two took together to Iraq, how the troops cheered upon "#,
r#"their entrance, and the Unity of Americans in the service of their country. On education policy, Sarah Huckabee Sanders spoke about the need for "#,
r#"instruction and not indoctrination for students of all backgrounds, referring to an executive order she passed last year. In statements released ahead of "#,
r#"the address, McConnell and McCarthy both commended the Youngest Governor in the nation for her leadership skills. The response to the State of the Union "#,
r#"was provided by Palin Huckabee Sanders."#
);
// TODO: This test isn't very well thought out. We should make better assertions and have more tests.
let results = split_into_manageable_portions(&text);
assert_eq!(results.len(), 3); // Total paragraph count
for paragraph in results.iter() {
assert!(results.len() < 450);
}
}
}
@@ -71,8 +71,7 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A
{
let yaml_filename = job_state.save_directory
.rendition_file_for_webpage_url(&target.original_news_canonical_url)?;
let mut file = std::fs::File::create(&yaml_filename)?;
serde_yaml::to_writer(file, &rendition_data)?;
rendition_data.write_to_yaml_file(yaml_filename)?;
}
Ok(())