track extra metadata throughout production

crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs

@@ -7,6 +7,10 @@ pub struct Args <'a> {
   pub news_story_token: &'a NewsStoryToken,
 
   pub original_news_canonical_url: &'a str,
+  pub original_news_title: &'a str,
+
+  pub audio_file_count: i64,
+  pub audio_total_duration_seconds: i64,
 
   pub replayable_until: DateTime<Utc>,
 
@@ -21,10 +25,16 @@ pub async fn insert_news_story(args: Args<'_>) -> AnyhowResult<()> {
 INSERT INTO news_stories (
   news_story_token,
   original_news_canonical_url,
+  original_news_title,
+  audio_file_count,
+  audio_total_duration_seconds,
   replayable_until,
   is_playable
 )
 VALUES (
+  ?,
+  ?,
+  ?,
   ?,
   ?,
   ?,
@@ -33,6 +43,9 @@ VALUES (
         "#,
         news_story_token,
         args.original_news_canonical_url,
+        args.original_news_title,
+        args.audio_file_count,
+        args.audio_total_duration_seconds,
         args.replayable_until
     );
 

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs

@@ -6,6 +6,7 @@ pub struct Args <'a> {
   pub news_story_token: &'a NewsStoryToken,
 
   pub original_news_canonical_url: &'a str,
+  pub original_news_title: &'a str,
 
   pub sqlite_pool: &'a SqlitePool,
 }
@@ -18,15 +19,18 @@ pub async fn insert_news_story_production(args: Args<'_>) -> AnyhowResult<()> {
 INSERT INTO news_story_productions (
   news_story_token,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status
 )
 VALUES (
+  ?,
   ?,
   ?,
   "ready_waiting"
 )
         "#,
         news_story_token,
+        args.original_news_title,
         args.original_news_canonical_url
     );
 

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_attempts,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_attempts,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_attempts,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs

@@ -22,6 +22,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_attempts,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_attempts,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs

@@ -5,6 +5,7 @@ pub struct NewsStoryProductionItem {
   pub id: i64,
   pub news_story_token: NewsStoryToken,
   pub original_news_canonical_url: String,
+  pub original_news_title: String,
   pub overall_production_status: AwaitableJobStatus,
   pub llm_rendition_status: AwaitableJobStatus,
   pub llm_rendition_attempts: i64,

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql

@@ -12,6 +12,9 @@ CREATE TABLE news_story_productions (
   -- The original URL for the news item
   original_news_canonical_url TEXT NOT NULL UNIQUE,
 
+  -- Original article title (not LLM renditioned)
+  original_news_title TEXT NOT NULL,
+
   -- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure
   -- reason, this is why.
   -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", etc.

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql

@@ -12,6 +12,15 @@ CREATE TABLE news_stories (
   -- The original URL for the news item
   original_news_canonical_url TEXT NOT NULL UNIQUE,
 
+  -- Original article title (not LLM renditioned)
+  original_news_title TEXT NOT NULL,
+
+  -- Total number of discrete audio files for this story
+  audio_file_count INTEGER NOT NULL,
+
+  -- Total amount of time in the audio files.
+  audio_total_duration_seconds INTEGER NOT NULL,
+
   -- When the story should stop being scheduled.
   replayable_until DATETIME NOT NULL,
 

crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs

@@ -13,6 +13,7 @@ use sqlite_queries::queries::by_table::news_story_productions::update::update_ne
 use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_post_production_complete::update_news_story_post_production_complete;
 use std::ops::Add;
 use std::sync::Arc;
+use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_for_story_token::list_tts_render_tasks_for_story_token;
 
 /// Stop playing stories after 24 hours elapses
 static STORY_FRESHNESS_THRESHOLD : Lazy<Duration> = Lazy::new(|| {
@@ -21,11 +22,26 @@ static STORY_FRESHNESS_THRESHOLD : Lazy<Duration> = Lazy::new(|| {
 
 pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc<JobState>) -> AnyhowResult<()> {
 
+  let story_type = "news_story";
+  let story_token = target.news_story_token.to_string();
+
+  let tts_render_tasks = list_tts_render_tasks_for_story_token(
+    story_type,
+    &story_token,
+    &job_state.sqlite_pool).await?;
+
+  // TODO: Include new column "tts_render_tasks.sequence_length" to verify. Math.max() it.
+  let audio_file_count = tts_render_tasks.len() as i64;
+  let audio_total_duration_seconds = 0; // TODO
+
   let replayable_until = Utc::now().add(*STORY_FRESHNESS_THRESHOLD);
 
   insert_news_story(InsertArgs {
     news_story_token: &target.news_story_token,
     original_news_canonical_url: &target.original_news_canonical_url,
+    original_news_title: &target.original_news_title,
+    audio_file_count,
+    audio_total_duration_seconds,
     replayable_until,
     sqlite_pool: &job_state.sqlite_pool,
   }).await?;

crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs

@@ -70,6 +70,7 @@ pub async fn process_target_record(target: &WebScrapingTargetRecord, job_state:
         news_story_token: &news_story_token,
         original_news_canonical_url: &target.canonical_url,
         // maybe_skip_reason, // TODO: Should I do this?
+        original_news_title: target.maybe_title.as_deref().unwrap_or(""),
         sqlite_pool: &job_state.sqlite_pool,
       }).await?;