diff --git a/crates/_types/enums/src/by_table/tts_render_tasks/tts_render_status.rs b/crates/_types/enums/src/by_table/tts_render_tasks/tts_render_status.rs index 940e33a258..4406e531f8 100644 --- a/crates/_types/enums/src/by_table/tts_render_tasks/tts_render_status.rs +++ b/crates/_types/enums/src/by_table/tts_render_tasks/tts_render_status.rs @@ -62,8 +62,8 @@ impl TtsRenderStatus { #[cfg(test)] mod tests { - use crate::by_table::tts_render_targets::tts_render_status::TtsRenderStatus; use crate::test_helpers::assert_serialization; + use crate::by_table::tts_render_tasks::tts_render_status::TtsRenderStatus; mod serde { use super::*; diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs index d66a3a21bb..2dea4d4509 100644 --- a/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs +++ b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs @@ -1,2 +1 @@ pub mod scraping_status; -pub mod web_content_type; \ No newline at end of file diff --git a/crates/_types/enums/src/common/sqlite/mod.rs b/crates/_types/enums/src/common/sqlite/mod.rs index 2f5c464139..7659a1e044 100644 --- a/crates/_types/enums/src/common/sqlite/mod.rs +++ b/crates/_types/enums/src/common/sqlite/mod.rs @@ -1,3 +1,4 @@ pub mod awaitable_job_status; pub mod chatbot_job_status; -pub mod skip_reason; \ No newline at end of file +pub mod skip_reason; +pub mod web_content_type; diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs b/crates/_types/enums/src/common/sqlite/web_content_type.rs similarity index 94% rename from crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs rename to crates/_types/enums/src/common/sqlite/web_content_type.rs index a4e8abb63f..c1dc66d55a 100644 --- a/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs +++ b/crates/_types/enums/src/common/sqlite/web_content_type.rs @@ -7,6 +7,8 @@ use strum::EnumCount; use strum::EnumIter; /// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `web_content_type`. +/// Used in the SqLite `news_story_productions` table in a `TEXT` field named `web_content_type`. +/// Used in the SqLite `news_stories` table in a `TEXT` field named `web_content_type`. #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] pub enum WebContentType { @@ -66,8 +68,8 @@ impl WebContentType { #[cfg(test)] mod tests { - use crate::by_table::web_scraping_targets::web_content_type::WebContentType; use crate::test_helpers::assert_serialization; + use crate::common::sqlite::web_content_type::WebContentType; mod serde { use super::*; diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs index 5df1187148..5140582124 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_stories/insert_news_story.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; use tokens::tokens::news_stories::NewsStoryToken; @@ -7,6 +8,7 @@ pub struct Args <'a> { pub news_story_token: &'a NewsStoryToken, pub original_news_canonical_url: &'a str, + pub web_content_type: WebContentType, pub original_news_title: &'a str, pub audio_file_count: i64, @@ -19,12 +21,14 @@ pub struct Args <'a> { pub async fn insert_news_story(args: Args<'_>) -> AnyhowResult<()> { let news_story_token = args.news_story_token.as_str(); + let web_content_type = args.web_content_type.to_str(); let query = sqlx::query!( r#" INSERT INTO news_stories ( news_story_token, original_news_canonical_url, + web_content_type, original_news_title, audio_file_count, audio_total_duration_seconds, @@ -38,11 +42,13 @@ VALUES ( ?, ?, ?, + ?, true ) "#, news_story_token, args.original_news_canonical_url, + web_content_type, args.original_news_title, args.audio_file_count, args.audio_total_duration_seconds, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs index fb1615bd10..899b22c782 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/insert_news_story_production.rs @@ -1,3 +1,4 @@ +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; use tokens::tokens::news_stories::NewsStoryToken; @@ -6,6 +7,7 @@ pub struct Args <'a> { pub news_story_token: &'a NewsStoryToken, pub original_news_canonical_url: &'a str, + pub web_content_type: WebContentType, pub original_news_title: &'a str, pub sqlite_pool: &'a SqlitePool, @@ -13,12 +15,14 @@ pub struct Args <'a> { pub async fn insert_news_story_production(args: Args<'_>) -> AnyhowResult<()> { let news_story_token = args.news_story_token.as_str(); + let web_content_type = args.web_content_type.to_str(); let query = sqlx::query!( r#" INSERT INTO news_story_productions ( news_story_token, original_news_canonical_url, + web_content_type, original_news_title, overall_production_status ) @@ -26,11 +30,13 @@ VALUES ( ?, ?, ?, + ?, "ready_waiting" ) "#, news_story_token, args.original_news_canonical_url, + web_content_type, args.original_news_title, ); diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs index f56b980435..fe7cf1a65c 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_final_verification.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs index 63f20ae19e..55ab074811 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_audio_generation.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs index e0e515207c..df2348486b 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_greenlight.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs index 924c63060f..763320623d 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_awaiting_llm_rendition.rs @@ -22,6 +22,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs index 4fe1fb3312..7911d3c0ee 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/list_news_story_productions_ready_for_debut.rs @@ -17,6 +17,7 @@ SELECT id, news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, original_news_canonical_url, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, original_news_title, overall_production_status as `overall_production_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, llm_rendition_status as `llm_rendition_status: enums::common::sqlite::awaitable_job_status::AwaitableJobStatus`, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs index 28b72a83f9..216b45835c 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_productions/list/news_story_production_item.rs @@ -1,9 +1,11 @@ use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus; +use enums::common::sqlite::web_content_type::WebContentType; use tokens::tokens::news_stories::NewsStoryToken; pub struct NewsStoryProductionItem { pub id: i64, pub news_story_token: NewsStoryToken, + pub web_content_type: WebContentType, pub original_news_canonical_url: String, pub original_news_title: String, pub overall_production_status: AwaitableJobStatus, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs index 4929e67819..a3ff34c944 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs @@ -1,8 +1,8 @@ -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; -use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; pub struct Args <'a> { pub canonical_url: &'a str, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs index e8e48bf345..b47ef3543f 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs @@ -1,7 +1,7 @@ use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; pub struct WebScrapingTarget { pub id: i64, @@ -31,7 +31,7 @@ pub async fn list_web_scraping_targets( SELECT id, canonical_url, - web_content_type as `web_content_type: enums::by_table::web_scraping_targets::web_content_type::WebContentType`, + web_content_type as `web_content_type: enums::common::sqlite::web_content_type::WebContentType`, maybe_title, maybe_article_full_image_url, maybe_article_thumbnail_image_url, diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/update_web_scraping_target.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/update_web_scraping_target.rs index 4aa6e55294..529b5a57bd 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/update_web_scraping_target.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/update_web_scraping_target.rs @@ -1,7 +1,6 @@ -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; -use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; pub struct Args <'a> { pub canonical_url: &'a str, diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs index b8ff57e35e..b147cba1d0 100644 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs @@ -1,4 +1,4 @@ -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; // TODO: Rename this to not confuse with `Result`. #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs index 75548e0158..f84fb1943d 100644 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs @@ -1,5 +1,5 @@ -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; // TODO: Include the article publish date(!) diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs index fb7f4e0eef..1902c491f0 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs @@ -2,7 +2,7 @@ use crate::common_extractors::extract_featured_image::extract_featured_image; use crate::common_extractors::extract_title::extract_title; use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle}; use crate::payloads::web_scraping_target::WebScrapingTarget; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use log::{error, warn}; use once_cell::sync::Lazy; diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs index 6c3894caf7..f3d4a4accd 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs @@ -1,9 +1,9 @@ use crate::payloads::web_scraping_target::WebScrapingTarget; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; use log::warn; use rss::Channel; -use enums::common::sqlite::skip_reason::SkipReason; // NB: Contains nearly 70 items const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss"; diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs index a3198dcbc1..cc1acf0a1c 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs @@ -2,7 +2,7 @@ use crate::common_extractors::extract_featured_image::extract_featured_image; use crate::common_extractors::extract_title::extract_title; use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle}; use crate::payloads::web_scraping_target::WebScrapingTarget; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use log::{error, warn}; use once_cell::sync::Lazy; diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs index ff4bfe348c..c5d77d1f27 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use crate::payloads::web_scraping_target::WebScrapingTarget; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; use log::warn; use rss::Channel; diff --git a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs index ac7f98b7ff..09d8d99b8f 100644 --- a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs @@ -1,5 +1,5 @@ use crate::payloads::web_scraping_target::WebScrapingTarget; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; use log::warn; use rss::Channel; diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql index af179a4ed8..b5ec2361b0 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010705_news_story_productions/up.sql @@ -12,12 +12,15 @@ CREATE TABLE news_story_productions ( -- The original URL for the news item original_news_canonical_url TEXT NOT NULL UNIQUE, + -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc. + web_content_type TEXT NOT NULL, + -- Original article title (not LLM renditioned) original_news_title TEXT NOT NULL, -- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure -- reason, this is why. - -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", etc. + -- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", "advertisement", etc. maybe_skip_reason TEXT, -- The overall status of the news story. diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql index fd53db6d31..878c2a80fd 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-11-010711_news_stories/up.sql @@ -12,6 +12,9 @@ CREATE TABLE news_stories ( -- The original URL for the news item original_news_canonical_url TEXT NOT NULL UNIQUE, + -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc. + web_content_type TEXT NOT NULL, + -- Original article title (not LLM renditioned) original_news_title TEXT NOT NULL, diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 100de52959..64324b51a2 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -21,14 +21,20 @@ use crate::shared_state::app_control_state::AppControlState; use crate::shared_state::job_state::JobState; use crate::startup_args::get_startup_args; use crate::web_server::launch_web_server::{launch_web_server, LaunchWebServerArgs}; +use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create_main_loop; +use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop; +use crate::workers::news_stories::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop; use crate::workers::news_stories::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop; use crate::workers::news_stories::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop; use crate::workers::news_stories::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop; +use crate::workers::news_stories::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop; use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop; use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; use crate::workers::web::web_index_ingestion::main_loop::web_index_ingestion_main_loop; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; +use fakeyou_client::credentials::FakeYouCredentials; +use fakeyou_client::fakeyou_api_client::FakeYouApiClient; use log::info; use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target}; use sqlx::sqlite::SqlitePoolOptions; @@ -36,15 +42,9 @@ use std::sync::Arc; use std::thread; use std::time::Duration; use tokio::runtime::{Builder, Runtime}; -use fakeyou_client::credentials::FakeYouCredentials; -use fakeyou_client::fakeyou_api_client::FakeYouApiClient; use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper; use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; -use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create_main_loop; -use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop; -use crate::workers::news_stories::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop; -use crate::workers::news_stories::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop; //#[tokio::main] //pub async fn main() -> AnyhowResult<()> { diff --git a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs index de85520657..1e6d42f936 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/news_stories/news_story_post_production_finalization/process_single_item.rs @@ -44,6 +44,7 @@ pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &A insert_news_story(InsertArgs { news_story_token: &target.news_story_token, original_news_canonical_url: &target.original_news_canonical_url, + web_content_type: target.web_content_type, original_news_title: &target.original_news_title, audio_file_count, audio_total_duration_seconds, diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs index eb46ed98a2..83d02cfb54 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs @@ -1,5 +1,5 @@ -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; use web_scrapers::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs index c2015a2b7d..d180b0a27d 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs @@ -1,5 +1,5 @@ use crate::persistence::save_directory::SaveDirectory; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::web_content_type::WebContentType; use errors::AnyhowResult; use std::io::Write; use web_scrapers::payloads::web_scraping_result::WebScrapingResult; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs index a95ba8c47d..79490e3a02 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs @@ -69,8 +69,9 @@ pub async fn process_target_record(target: &WebScrapingTargetRecord, job_state: insert_news_story_production(ProductionArgs { news_story_token: &news_story_token, original_news_canonical_url: &target.canonical_url, - // maybe_skip_reason, // TODO: Should I do this? + web_content_type: target.web_content_type, original_news_title: target.maybe_title.as_deref().unwrap_or(""), + // maybe_skip_reason, // TODO: Should I do this? sqlite_pool: &job_state.sqlite_pool, }).await?;