diff --git a/Cargo.lock b/Cargo.lock index bc9f47f576..6063aea6c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,6 +772,7 @@ dependencies = [ "serde_yaml", "sqlite_queries", "sqlx", + "strum", "tokens", "tokio 1.25.0", "web_scrapers", @@ -8189,6 +8190,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "strum", "toml", ] diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index 1eeafc339e..26ce0081ec 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -22,6 +22,7 @@ bytes = "1.3" chrono = { version = "0.4.19", features = ["serde"] } log = "0.4.14" once_cell = "1.9.0" +strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc. # Serialization serde = "1.0.125" diff --git a/crates/lib/web_scrapers/src/lib.rs b/crates/lib/web_scrapers/src/lib.rs index 428a62e427..f2039b2dc6 100644 --- a/crates/lib/web_scrapers/src/lib.rs +++ b/crates/lib/web_scrapers/src/lib.rs @@ -19,6 +19,7 @@ #![allow(non_snake_case)] #[macro_use] extern crate serde_derive; +#[macro_use] extern crate strum; pub mod payloads; pub mod sites; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/mod.rs b/crates/lib/web_scrapers/src/sites/mod.rs index 3a3c0c60a7..3c3cf843dc 100644 --- a/crates/lib/web_scrapers/src/sites/mod.rs +++ b/crates/lib/web_scrapers/src/sites/mod.rs @@ -4,4 +4,5 @@ pub mod reddit; pub mod reuters; pub mod slashdot; pub mod techcrunch; -pub mod theguardian; \ No newline at end of file +pub mod theguardian; +pub mod twitter; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs b/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs index 79815ae637..dd95ed530f 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs @@ -1,2 +1,2 @@ -pub mod techcrunch_scraper; +pub mod techcrunch_indexer; pub mod techcrunch_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs similarity index 79% rename from crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs rename to crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs index 6d7757d351..ed8053108b 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_indexer.rs @@ -1,20 +1,37 @@ -use std::ops::Deref; use bytes::Bytes; use crate::payloads::web_scraping_target::WebScrapingTarget; use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use errors::AnyhowResult; use log::warn; use rss::Channel; +use std::ops::Deref; -// NB: Contains 20 items +// NB: Feed contains ~20 items. const MAIN_RSS_FEED : &'static str = "https://techcrunch.com/feed/"; const VENTURE_FEED : &'static str = "https://techcrunch.com/category/venture/feed/"; const STARTUPS_FEED : &'static str = "https://techcrunch.com/category/startups/feed/"; -pub async fn techcrunch_scraper_test() -> AnyhowResult> { - let content = reqwest::get(MAIN_RSS_FEED) +#[derive(Copy, Clone, Debug, EnumIter, EnumCount)] +pub enum TechcrunchFeed { + Main, + Venture, + Startups, +} + +impl TechcrunchFeed { + fn url(&self) -> &'static str { + match self { + TechcrunchFeed::Main => MAIN_RSS_FEED, + TechcrunchFeed::Venture => VENTURE_FEED, + TechcrunchFeed::Startups => STARTUPS_FEED, + } + } +} + +pub async fn techcrunch_indexer(feed: TechcrunchFeed) -> AnyhowResult> { + let content = reqwest::get(feed.url()) .await? .bytes() .await?; diff --git a/crates/programs/aichatbot-sidecar/src/jobs/fakeyou/mod.rs b/crates/lib/web_scrapers/src/sites/twitter/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/jobs/fakeyou/mod.rs rename to crates/lib/web_scrapers/src/sites/twitter/mod.rs diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index 01a29fc879..39b798ddc3 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -27,10 +27,11 @@ web_scrapers = { path = "../../lib/web_scrapers" } # External clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +dotenv = "0.15.0" easyenv = "0.0.6" log = "0.4.14" -dotenv = "0.15.0" path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet +strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc. # SQlite Database # "offline" supports typesafe builds without a live database diff --git a/crates/programs/aichatbot-sidecar/src/ingestion/mod.rs b/crates/programs/aichatbot-sidecar/src/ingestion/mod.rs deleted file mode 100644 index e241a8b76f..0000000000 --- a/crates/programs/aichatbot-sidecar/src/ingestion/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod ingest_url_scrape_and_save; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/jobs/mod.rs b/crates/programs/aichatbot-sidecar/src/jobs/mod.rs deleted file mode 100644 index 36a002775c..0000000000 --- a/crates/programs/aichatbot-sidecar/src/jobs/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod fakeyou; -pub mod openai; -pub mod reddit; -pub mod twitter; \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/jobs/twitter/mod.rs b/crates/programs/aichatbot-sidecar/src/jobs/twitter/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 95514dfb66..374ad5daae 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -3,13 +3,12 @@ #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] // hide console window on Windows in release pub mod gui; -pub mod ingestion; -pub mod jobs; pub mod main_loop; pub mod persistence; pub mod shared_state; pub mod startup_args; pub mod web_server; +pub mod workers; #[macro_use] extern crate serde_derive; @@ -34,12 +33,13 @@ use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper; use web_scrapers::sites::cnn::cnn_indexer::cnn_scraper_test; use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; -use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test; use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; -use crate::ingestion::ingest_url_scrape_and_save::ingest_url_scrape_and_save; +use workers::web_content_scraping::ingest_url_scrape_and_save::ingest_url_scrape_and_save; +use crate::shared_state::job_state::JobState; +use crate::workers::web_index_ingestion::main_loop::web_index_ingestion_main_loop; #[tokio::main] -pub async fn main() -> AnyhowResult<()> { +pub async fn main2() -> AnyhowResult<()> { let database_url = easyenv::get_env_string_required("DATABASE_URL")?; let pool = SqlitePoolOptions::new() .max_connections(5) @@ -85,7 +85,7 @@ pub async fn main() -> AnyhowResult<()> { } #[actix_web::main] -pub async fn main2() -> AnyhowResult<()> { +pub async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some("info")); // NB: Do not check this secrets-containing dotenv file into VCS. @@ -99,30 +99,57 @@ pub async fn main2() -> AnyhowResult<()> { let openai_client = Arc::new(Client::new() .with_api_key(startup_args.openai_secret_key.clone())); - let tokio_runtime = Runtime::new()?; - let save_directory = SaveDirectory::new(&startup_args.save_directory); - info!("Starting async processes..."); + let database_url = easyenv::get_env_string_required("DATABASE_URL")?; + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect(&database_url).await?; - tokio_runtime.spawn(async { - let _r = main_loop().await; + let job_state = Arc::new(JobState { + sqlite_pool: pool, + save_directory: save_directory.clone(), }); info!("Starting web server..."); let app_control_state2 = app_control_state.clone(); let openai_client2 = openai_client.clone(); + let job_state2 = job_state.clone(); + // NB: both egui and imgui (which we aren't using) complain about launching on a non-main thread. + // They even complain that this is impossible on Windows (and our program aims to be multiplatform) + // Thus, we launch everything else into its own thread. (TODO: Jobs + Server in one thread) thread::spawn(move || { let server_future = launch_web_server(LaunchWebServerArgs { app_control_state: app_control_state2, openai_client: openai_client2, save_directory, }); - actix_web::rt::System::new().block_on(server_future) + + let tokio_runtime = Runtime::new()?; + + tokio_runtime.spawn(async { + let _r = web_index_ingestion_main_loop(job_state2).await; + }); + + let runtime = actix_web::rt::System::new(); + + runtime.block_on(server_future) }); +// info!("Starting async processes..."); +// +// thread::spawn(move|| { +// +// tokio_runtime.block_on(async || { +// loop { +// thread::sleep(Duration::from_secs(60)) +// std::future:: +// } +// }.await) +// }); + info!("Starting GUI ..."); let _r = launch_gui(startup_args.clone(), app_control_state.clone()); diff --git a/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs b/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs index e69de29bb2..83913d8953 100644 --- a/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs +++ b/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs @@ -0,0 +1,8 @@ +use crate::persistence::save_directory::SaveDirectory; +use sqlx::{Pool, Sqlite}; + +pub struct JobState { + pub sqlite_pool: Pool, + pub save_directory: SaveDirectory, +} + diff --git a/crates/programs/aichatbot-sidecar/src/jobs/openai/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/fakeyou_enrichment/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/jobs/openai/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/fakeyou_enrichment/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/jobs/reddit/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/gpt_transform/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/jobs/reddit/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/gpt_transform/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/mod.rs new file mode 100644 index 0000000000..7b7e713d3b --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/mod.rs @@ -0,0 +1,4 @@ +pub mod fakeyou_enrichment; +pub mod gpt_transform; +pub mod web_content_scraping; +pub mod web_index_ingestion; diff --git a/crates/programs/aichatbot-sidecar/src/ingestion/ingest_url_scrape_and_save.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/ingest_url_scrape_and_save.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/ingestion/ingest_url_scrape_and_save.rs rename to crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/ingest_url_scrape_and_save.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs new file mode 100644 index 0000000000..218637c518 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs @@ -0,0 +1 @@ +pub mod ingest_url_scrape_and_save; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs new file mode 100644 index 0000000000..e4ee785107 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs @@ -0,0 +1,57 @@ +use crate::shared_state::job_state::JobState; +use errors::AnyhowResult; +use log::{error, info}; +use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target}; +use sqlx::sqlite::SqlitePoolOptions; +use std::future::Future; +use std::sync::Arc; +use std::thread; +use std::time::Duration; +use strum::IntoEnumIterator; +use web_scrapers::payloads::web_scraping_result::WebScrapingResult; +use web_scrapers::payloads::web_scraping_target::WebScrapingTarget; +use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed}; + +pub async fn web_index_ingestion_main_loop(job_state: Arc) { + + loop { + info!("web_index_ingestion main loop"); + + match single_iteration(&job_state).await { + Ok(_) => thread::sleep(Duration::from_secs(300)), + Err(err) => { + error!("Error with indexing: {:?}", err); + thread::sleep(Duration::from_secs(60)); + } + } + } +} + +async fn single_iteration(job_state: &Arc) -> AnyhowResult<()> { + + for variant in TechcrunchFeed::iter() { + let index_targets = techcrunch_indexer(variant).await?; + insert_targets(job_state, &index_targets).await?; + } + + + //let index_targets = techcrunch_indexer(TechcrunchFeed::Main).await?; + //insert_targets(&index_targets).await?; + + Ok(()) +} + +async fn insert_targets(job_state: &Arc, targets: &Vec) -> AnyhowResult<()> { + for target in targets.iter() { + let _r = insert_web_scraping_target(Args { + canonical_url: &target.canonical_url, + web_content_type: target.web_content_type, + maybe_title: target.maybe_title.as_deref(), + maybe_article_full_image_url: target.maybe_full_image_url.as_deref(), + maybe_article_thumbnail_image_url: target.maybe_thumbnail_image_url.as_deref(), + sqlite_pool: &job_state.sqlite_pool, + }).await; + } + + Ok(()) +} diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/mod.rs new file mode 100644 index 0000000000..ccca46ee14 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/mod.rs @@ -0,0 +1 @@ +pub mod main_loop;