diff --git a/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs b/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs new file mode 100644 index 0000000000..4f45934782 --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs @@ -0,0 +1,65 @@ +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; +use errors::AnyhowResult; +use log::warn; +use rss::Channel; + +const GIZMODO_RSS : &'static str = "kotaku_indexer.rs"; + +// TODO: Rename foo_feed() or foo_rss(), etc. +pub async fn gizmodo_indexer() -> AnyhowResult> { + let content = reqwest::get(GIZMODO_RSS) + .await? + .bytes() + .await?; + + let channel = Channel::read_from(&content[..])?; + + let mut targets = Vec::with_capacity(channel.items.len()); + + for item in channel.items { + let canonical_url = match item.link { + Some(url) => url.clone(), + None => { + warn!("Skipping item due to not having a URL"); + continue; + } + }; + + let mut maybe_skip_reason = None; + + //// Lazy heuristic to skip CNN video articles, which contain little text. + //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn + //if canonical_url.contains("/videos/") { + // maybe_skip_reason = Some(SkipReason::VideoContent); + //} + + // NB: I'm not using these as I had anticipated. + let maybe_image_url = None; + + //let maybe_image_url = item.extensions.get("media") + // .map(|media| media.get("group")) + // .flatten() + // .map(|group| group.get(0)) + // .flatten() + // .map(|extension| extension.children.get("content")) + // .flatten() + // .map(|extensions| extensions.get(0)) // NB: First image is biggest + // .flatten() + // .map(|extension| extension.attrs.get("url")) + // .flatten() + // .map(|url| url.to_string()); + + targets.push(WebScrapingTarget { + canonical_url, + web_content_type: WebContentType::GizmodoArticle, + maybe_title: item.title.clone(), + maybe_full_image_url: maybe_image_url, + maybe_thumbnail_image_url: None, + maybe_skip_reason, + }); + } + + Ok(targets) +} diff --git a/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs b/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs index ef8d6842fb..be546ecdc0 100644 --- a/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs +++ b/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs @@ -1 +1,2 @@ -pub mod gizmodo_article_scraper; \ No newline at end of file +pub mod gizmodo_article_scraper; +pub mod gizmodo_indexer; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs b/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs new file mode 100644 index 0000000000..58f1ddb168 --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs @@ -0,0 +1,65 @@ +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::common::sqlite::skip_reason::SkipReason; +use enums::common::sqlite::web_content_type::WebContentType; +use errors::AnyhowResult; +use log::warn; +use rss::Channel; + +const KOTAKU_RSS: &'static str = "https://kotaku.com/rss"; + +// TODO: Rename foo_feed() or foo_rss(), etc. +pub async fn kotaku_indexer() -> AnyhowResult> { + let content = reqwest::get(KOTAKU_RSS) + .await? + .bytes() + .await?; + + let channel = Channel::read_from(&content[..])?; + + let mut targets = Vec::with_capacity(channel.items.len()); + + for item in channel.items { + let canonical_url = match item.link { + Some(url) => url.clone(), + None => { + warn!("Skipping item due to not having a URL"); + continue; + } + }; + + let mut maybe_skip_reason = None; + + //// Lazy heuristic to skip CNN video articles, which contain little text. + //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn + //if canonical_url.contains("/videos/") { + // maybe_skip_reason = Some(SkipReason::VideoContent); + //} + + // NB: I'm not using these as I had anticipated. + let maybe_image_url = None; + + //let maybe_image_url = item.extensions.get("media") + // .map(|media| media.get("group")) + // .flatten() + // .map(|group| group.get(0)) + // .flatten() + // .map(|extension| extension.children.get("content")) + // .flatten() + // .map(|extensions| extensions.get(0)) // NB: First image is biggest + // .flatten() + // .map(|extension| extension.attrs.get("url")) + // .flatten() + // .map(|url| url.to_string()); + + targets.push(WebScrapingTarget { + canonical_url, + web_content_type: WebContentType::KotakuArticle, + maybe_title: item.title.clone(), + maybe_full_image_url: maybe_image_url, + maybe_thumbnail_image_url: None, + maybe_skip_reason, + }); + } + + Ok(targets) +} diff --git a/crates/lib/web_scrapers/src/sites/kotaku/mod.rs b/crates/lib/web_scrapers/src/sites/kotaku/mod.rs index 659b2d449c..89dad7da74 100644 --- a/crates/lib/web_scrapers/src/sites/kotaku/mod.rs +++ b/crates/lib/web_scrapers/src/sites/kotaku/mod.rs @@ -1 +1,2 @@ -pub mod kotaku_article_scraper; \ No newline at end of file +pub mod kotaku_article_scraper; +pub mod kotaku_indexer; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs index 60fd2b89b6..a6e1f9ca2b 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs @@ -11,6 +11,8 @@ use strum::IntoEnumIterator; use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle; use web_scrapers::payloads::web_scraping_target::WebScrapingTarget; use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed}; +use web_scrapers::sites::gizmodo::gizmodo_indexer::gizmodo_indexer; +use web_scrapers::sites::kotaku::kotaku_indexer::kotaku_indexer; use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed}; /// Download RSS feeds and index pages to determine which articles and content will be scraped (async/downstream of this) @@ -37,24 +39,42 @@ pub async fn web_index_ingestion_main_loop(job_state: Arc) { async fn single_iteration(job_state: &Arc) -> AnyhowResult<()> { + // Kotaku + { + let index_targets = kotaku_indexer().await?; + insert_targets(job_state, &index_targets).await?; + thread::sleep(Duration::from_secs(2)); + } + + // Gizmodo + { + let index_targets = gizmodo_indexer().await?; + insert_targets(job_state, &index_targets).await?; + thread::sleep(Duration::from_secs(2)); + } + // CNN for variant in CnnFeed::iter() { let index_targets = cnn_indexer(variant).await?; insert_targets(job_state, &index_targets).await?; - thread::sleep(Duration::from_secs(5)); + thread::sleep(Duration::from_secs(2)); } // TechCrunch for variant in TechcrunchFeed::iter() { let index_targets = techcrunch_indexer(variant).await?; insert_targets(job_state, &index_targets).await?; - thread::sleep(Duration::from_secs(5)); + thread::sleep(Duration::from_secs(2)); } Ok(()) } async fn insert_targets(job_state: &Arc, targets: &Vec) -> AnyhowResult<()> { + while job_state.app_control_state.is_scraping_paused() { + thread::sleep(Duration::from_secs(5)); + } + for target in targets.iter() { let _r = insert_web_scraping_target(Args { canonical_url: &target.canonical_url,