Add web_content_type to news_stories and news_story_productions

crates/_types/enums/src/by_table/tts_render_tasks/tts_render_status.rs

@@ -62,8 +62,8 @@ impl TtsRenderStatus {
 
 #[cfg(test)]
 mod tests {
-  use crate::by_table::tts_render_targets::tts_render_status::TtsRenderStatus;
   use crate::test_helpers::assert_serialization;
+  use crate::by_table::tts_render_tasks::tts_render_status::TtsRenderStatus;
 
   mod serde {
     use super::*;

crates/_types/enums/src/by_table/web_scraping_targets/mod.rs

@@ -1,2 +1 @@
 pub mod scraping_status;
-pub mod web_content_type;

crates/_types/enums/src/common/sqlite/mod.rs

@@ -1,3 +1,4 @@
 pub mod awaitable_job_status;
 pub mod chatbot_job_status;
-pub mod skip_reason;
+pub mod skip_reason;
+pub mod web_content_type;

crates/_types/enums/src/common/sqlite/web_content_type.rs

@@ -7,6 +7,8 @@ use strum::EnumCount;
 use strum::EnumIter;
 
 /// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `web_content_type`.
+/// Used in the SqLite `news_story_productions` table in a `TEXT` field named `web_content_type`.
+/// Used in the SqLite `news_stories` table in a `TEXT` field named `web_content_type`.
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
 pub enum WebContentType {
@@ -66,8 +68,8 @@ impl WebContentType {
 
 #[cfg(test)]
 mod tests {
-  use crate::by_table::web_scraping_targets::web_content_type::WebContentType;
   use crate::test_helpers::assert_serialization;
+  use crate::common::sqlite::web_content_type::WebContentType;
 
   mod serde {
     use super::*;

crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs

@@ -1,4 +1,5 @@
 use chrono::{DateTime, Utc};
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
 use tokens::tokens::news_stories::NewsStoryToken;
@@ -7,6 +8,7 @@ pub struct Args <'a> {
   pub news_story_token: &'a NewsStoryToken,
 
   pub original_news_canonical_url: &'a str,
+  pub web_content_type: WebContentType,
   pub original_news_title: &'a str,
 
   pub audio_file_count: i64,
@@ -19,12 +21,14 @@ pub struct Args <'a> {
 
 pub async fn insert_news_story(args: Args<'_>) -> AnyhowResult<()> {
   let news_story_token = args.news_story_token.as_str();
+  let web_content_type = args.web_content_type.to_str();
 
   let query = sqlx::query!(
         r#"
 INSERT INTO news_stories (
   news_story_token,
   original_news_canonical_url,
+  web_content_type,
   original_news_title,
   audio_file_count,
   audio_total_duration_seconds,
@@ -38,11 +42,13 @@ VALUES (
   ?,
   ?,
   ?,
+  ?,
   true
 )
         "#,
         news_story_token,
         args.original_news_canonical_url,
+        web_content_type,
         args.original_news_title,
         args.audio_file_count,
         args.audio_total_duration_seconds,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs

@@ -1,3 +1,4 @@
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
 use tokens::tokens::news_stories::NewsStoryToken;
@@ -6,6 +7,7 @@ pub struct Args <'a> {
   pub news_story_token: &'a NewsStoryToken,
 
   pub original_news_canonical_url: &'a str,
+  pub web_content_type: WebContentType,
   pub original_news_title: &'a str,
 
   pub sqlite_pool: &'a SqlitePool,
@@ -13,12 +15,14 @@ pub struct Args <'a> {
 
 pub async fn insert_news_story_production(args: Args<'_>) -> AnyhowResult<()> {
   let news_story_token = args.news_story_token.as_str();
+  let web_content_type = args.web_content_type.to_str();
 
   let query = sqlx::query!(
         r#"
 INSERT INTO news_story_productions (
   news_story_token,
   original_news_canonical_url,
+  web_content_type,
   original_news_title,
   overall_production_status
 )
@@ -26,11 +30,13 @@ VALUES (
   ?,
   ?,
   ?,
+  ?,
   "ready_waiting"
 )
         "#,
         news_story_token,
         args.original_news_canonical_url,
+        web_content_type,
         args.original_news_title,
     );
 

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs

@@ -22,6 +22,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs

@@ -17,6 +17,7 @@ SELECT
   id,
   news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
   original_news_canonical_url,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   original_news_title,
   overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,
   llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`,

crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs

@@ -1,9 +1,11 @@
 use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus;
+use enums::common::sqlite::web_content_type::WebContentType;
 use tokens::tokens::news_stories::NewsStoryToken;
 
 pub struct NewsStoryProductionItem {
   pub id: i64,
   pub news_story_token: NewsStoryToken,
+  pub web_content_type: WebContentType,
   pub original_news_canonical_url: String,
   pub original_news_title: String,
   pub overall_production_status: AwaitableJobStatus,

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs

@@ -1,8 +1,8 @@
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
 use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
-use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
 
 pub struct Args <'a> {
   pub canonical_url: &'a str,

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs

@@ -1,7 +1,7 @@
 use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 
 pub struct WebScrapingTarget {
   pub id: i64,
@@ -31,7 +31,7 @@ pub async fn list_web_scraping_targets(
 SELECT
   id,
   canonical_url,
-  web_content_type as `web_content_type: enums::by_table::web_scraping_targets::web_content_type::WebContentType`,
+  web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`,
   maybe_title,
   maybe_article_full_image_url,
   maybe_article_thumbnail_image_url,

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/update_web_scraping_target.rs

@@ -1,7 +1,6 @@
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
-use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
 
 pub struct Args <'a> {
   pub canonical_url: &'a str,

crates/lib/web_scrapers/src/payloads/web_scraping_result.rs

@@ -1,4 +1,4 @@
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 
 // TODO: Rename this to not confuse with `Result<T, E>`.
 #[derive(Clone, Debug, Serialize, Deserialize)]

crates/lib/web_scrapers/src/payloads/web_scraping_target.rs

@@ -1,5 +1,5 @@
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
 
 // TODO: Include the article publish date(!)
 

crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs

@@ -2,7 +2,7 @@ use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
 use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use log::{error, warn};
 use once_cell::sync::Lazy;

crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs

@@ -1,9 +1,9 @@
 use crate::payloads::web_scraping_target::WebScrapingTarget;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;
-use enums::common::sqlite::skip_reason::SkipReason;
 
 // NB: Contains nearly 70 items
 const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss";

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs

@@ -2,7 +2,7 @@ use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
 use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use log::{error, warn};
 use once_cell::sync::Lazy;

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs

@@ -1,6 +1,6 @@
 use bytes::Bytes;
 use crate::payloads::web_scraping_target::WebScrapingTarget;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;

crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs

@@ -1,5 +1,5 @@
 use crate::payloads::web_scraping_target::WebScrapingTarget;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql

@@ -12,12 +12,15 @@ CREATE TABLE news_story_productions (
   -- The original URL for the news item
   original_news_canonical_url TEXT NOT NULL UNIQUE,
 
+  -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
+  web_content_type TEXT NOT NULL,
+
   -- Original article title (not LLM renditioned)
   original_news_title TEXT NOT NULL,
 
   -- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure
   -- reason, this is why.
-  -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", etc.
+  -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", "advertisement", etc.
   maybe_skip_reason TEXT,
 
   -- The overall status of the news story.

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql

@@ -12,6 +12,9 @@ CREATE TABLE news_stories (
   -- The original URL for the news item
   original_news_canonical_url TEXT NOT NULL UNIQUE,
 
+  -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
+  web_content_type TEXT NOT NULL,
+
   -- Original article title (not LLM renditioned)
   original_news_title TEXT NOT NULL,
 

crates/programs/aichatbot-sidecar/src/main.rs

@@ -21,14 +21,20 @@ use crate::shared_state::app_control_state::AppControlState;
 use crate::shared_state::job_state::JobState;
 use crate::startup_args::get_startup_args;
 use crate::web_server::launch_web_server::{launch_web_server, LaunchWebServerArgs};
+use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create_main_loop;
+use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop;
+use crate::workers::news_stories::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
 use crate::workers::news_stories::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop;
 use crate::workers::news_stories::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop;
 use crate::workers::news_stories::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop;
+use crate::workers::news_stories::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;
 use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop;
 use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save;
 use crate::workers::web::web_index_ingestion::main_loop::web_index_ingestion_main_loop;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
+use fakeyou_client::credentials::FakeYouCredentials;
+use fakeyou_client::fakeyou_api_client::FakeYouApiClient;
 use log::info;
 use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
 use sqlx::sqlite::SqlitePoolOptions;
@@ -36,15 +42,9 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use tokio::runtime::{Builder, Runtime};
-use fakeyou_client::credentials::FakeYouCredentials;
-use fakeyou_client::fakeyou_api_client::FakeYouApiClient;
 use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper;
 use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper;
 use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test;
-use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create_main_loop;
-use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop;
-use crate::workers::news_stories::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
-use crate::workers::news_stories::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;
 
 //#[tokio::main]
 //pub async fn main() -> AnyhowResult<()> {

crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs

@@ -44,6 +44,7 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A
   insert_news_story(InsertArgs {
     news_story_token: &target.news_story_token,
     original_news_canonical_url: &target.original_news_canonical_url,
+    web_content_type: target.web_content_type,
     original_news_title: &target.original_news_title,
     audio_file_count,
     audio_total_duration_seconds,

crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs

@@ -1,5 +1,5 @@
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use web_scrapers::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult};
 

crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs

@@ -1,5 +1,5 @@
 use crate::persistence::save_directory::SaveDirectory;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use std::io::Write;
 use web_scrapers::payloads::web_scraping_result::WebScrapingResult;

crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs

@@ -69,8 +69,9 @@ pub async fn process_target_record(target: &WebScrapingTargetRecord, job_state:
       insert_news_story_production(ProductionArgs {
         news_story_token: &news_story_token,
         original_news_canonical_url: &target.canonical_url,
-        // maybe_skip_reason, // TODO: Should I do this?
+        web_content_type: target.web_content_type,
         original_news_title: target.maybe_title.as_deref().unwrap_or(""),
+        // maybe_skip_reason, // TODO: Should I do this?
         sqlite_pool: &job_state.sqlite_pool,
       }).await?;