diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs index 2a97d91c47..5df1187148 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs @@ -7,6 +7,10 @@ pub struct Args <'a> { pub news_story_token: &'a NewsStoryToken, pub original_news_canonical_url: &'a str, + pub original_news_title: &'a str, + + pub audio_file_count: i64, + pub audio_total_duration_seconds: i64, pub replayable_until: DateTime, @@ -21,10 +25,16 @@ pub async fn insert_news_story(args: Args<'_>) -> AnyhowResult<()> { INSERT INTO news_stories ( news_story_token, original_news_canonical_url, + original_news_title, + audio_file_count, + audio_total_duration_seconds, replayable_until, is_playable ) VALUES ( + ?, + ?, + ?, ?, ?, ?, @@ -33,6 +43,9 @@ VALUES ( "#, news_story_token, args.original_news_canonical_url, + args.original_news_title, + args.audio_file_count, + args.audio_total_duration_seconds, args.replayable_until ); diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs index 7942a2a3c9..02d75cd82d 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs @@ -6,6 +6,7 @@ pub struct Args <'a> { pub news_story_token: &'a NewsStoryToken, pub original_news_canonical_url: &'a str, + pub original_news_title: &'a str, pub sqlite_pool: &'a SqlitePool, } @@ -18,15 +19,18 @@ pub async fn insert_news_story_production(args: Args<'_>) -> AnyhowResult<()> { INSERT INTO news_story_productions ( news_story_token, original_news_canonical_url, + original_news_title, overall_production_status ) VALUES ( + ?, ?, ?, "ready_waiting" ) "#, news_story_token, + args.original_news_title, args.original_news_canonical_url ); diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs index ebf5e34a98..f56b980435 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs index 9438b26740..63f20ae19e 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs index 3431ef1048..e0e515207c 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs index 327a4b0123..924c63060f 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs @@ -22,6 +22,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs index dd3ad8fe4d..4fe1fb3312 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_attempts, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs index 2d696515f2..28b72a83f9 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs @@ -5,6 +5,7 @@ pub struct NewsStoryProductionItem { pub id: i64, pub news_story_token: NewsStoryToken, pub original_news_canonical_url: String, + pub original_news_title: String, pub overall_production_status: AwaitableJobStatus, pub llm_rendition_status: AwaitableJobStatus, pub llm_rendition_attempts: i64, diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql index 6e7a8771e5..af179a4ed8 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql @@ -12,6 +12,9 @@ CREATE TABLE news_story_productions ( -- The original URL for the news item original_news_canonical_url TEXT NOT NULL UNIQUE, + -- Original article title (not LLM renditioned) + original_news_title TEXT NOT NULL, + -- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure -- reason, this is why. -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", etc. diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql index e087c10335..3f38b97b40 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql @@ -12,6 +12,15 @@ CREATE TABLE news_stories ( -- The original URL for the news item original_news_canonical_url TEXT NOT NULL UNIQUE, + -- Original article title (not LLM renditioned) + original_news_title TEXT NOT NULL, + + -- Total number of discrete audio files for this story + audio_file_count INTEGER NOT NULL, + + -- Total amount of time in the audio files. + audio_total_duration_seconds INTEGER NOT NULL, + -- When the story should stop being scheduled. replayable_until DATETIME NOT NULL, diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs index ab9a6bffb2..248194b013 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs @@ -13,6 +13,7 @@ use sqlite_queries::queries::by_table::news_story_productions::update::update_ne use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_post_production_complete::update_news_story_post_production_complete; use std::ops::Add; use std::sync::Arc; +use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_for_story_token::list_tts_render_tasks_for_story_token; /// Stop playing stories after 24 hours elapses static STORY_FRESHNESS_THRESHOLD : Lazy = Lazy::new(|| { @@ -21,11 +22,26 @@ static STORY_FRESHNESS_THRESHOLD : Lazy = Lazy::new(|| { pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc) -> AnyhowResult<()> { + let story_type = "news_story"; + let story_token = target.news_story_token.to_string(); + + let tts_render_tasks = list_tts_render_tasks_for_story_token( + story_type, + &story_token, + &job_state.sqlite_pool).await?; + + // TODO: Include new column "tts_render_tasks.sequence_length" to verify. Math.max() it. + let audio_file_count = tts_render_tasks.len() as i64; + let audio_total_duration_seconds = 0; // TODO + let replayable_until = Utc::now().add(*STORY_FRESHNESS_THRESHOLD); insert_news_story(InsertArgs { news_story_token: &target.news_story_token, original_news_canonical_url: &target.original_news_canonical_url, + original_news_title: &target.original_news_title, + audio_file_count, + audio_total_duration_seconds, replayable_until, sqlite_pool: &job_state.sqlite_pool, }).await?; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs index 11cf45bbc5..a95ba8c47d 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs @@ -70,6 +70,7 @@ pub async fn process_target_record(target: &WebScrapingTargetRecord, job_state: news_story_token: &news_story_token, original_news_canonical_url: &target.canonical_url, // maybe_skip_reason, // TODO: Should I do this? + original_news_title: target.maybe_title.as_deref().unwrap_or(""), sqlite_pool: &job_state.sqlite_pool, }).await?;