Kotaku + Gizmodo RSS indexers

crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs

@@ -0,0 +1,65 @@
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
+use errors::AnyhowResult;
+use log::warn;
+use rss::Channel;
+
+const GIZMODO_RSS : &'static str = "kotaku_indexer.rs";
+
+// TODO: Rename foo_feed() or foo_rss(), etc.
+pub async fn gizmodo_indexer() -> AnyhowResult<Vec<WebScrapingTarget>> {
+  let content = reqwest::get(GIZMODO_RSS)
+      .await?
+      .bytes()
+      .await?;
+
+  let channel = Channel::read_from(&content[..])?;
+
+  let mut targets = Vec::with_capacity(channel.items.len());
+
+  for item in channel.items {
+    let canonical_url = match item.link {
+      Some(url) => url.clone(),
+      None => {
+        warn!("Skipping item due to not having a URL");
+        continue;
+      }
+    };
+
+    let mut maybe_skip_reason = None;
+
+    //// Lazy heuristic to skip CNN video articles, which contain little text.
+    //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn
+    //if canonical_url.contains("/videos/") {
+    //  maybe_skip_reason = Some(SkipReason::VideoContent);
+    //}
+
+    // NB: I'm not using these as I had anticipated.
+    let maybe_image_url = None;
+
+    //let maybe_image_url = item.extensions.get("media")
+    //    .map(|media| media.get("group"))
+    //    .flatten()
+    //    .map(|group| group.get(0))
+    //    .flatten()
+    //    .map(|extension| extension.children.get("content"))
+    //    .flatten()
+    //    .map(|extensions| extensions.get(0)) // NB: First image is biggest
+    //    .flatten()
+    //    .map(|extension| extension.attrs.get("url"))
+    //    .flatten()
+    //    .map(|url| url.to_string());
+
+    targets.push(WebScrapingTarget {
+      canonical_url,
+      web_content_type: WebContentType::GizmodoArticle,
+      maybe_title: item.title.clone(),
+      maybe_full_image_url: maybe_image_url,
+      maybe_thumbnail_image_url: None,
+      maybe_skip_reason,
+    });
+  }
+
+  Ok(targets)
+}

crates/lib/web_scrapers/src/sites/gizmodo/mod.rs

@@ -1 +1,2 @@
-pub mod gizmodo_article_scraper;
+pub mod gizmodo_article_scraper;
+pub mod gizmodo_indexer;

crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs

@@ -0,0 +1,65 @@
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::common::sqlite::skip_reason::SkipReason;
+use enums::common::sqlite::web_content_type::WebContentType;
+use errors::AnyhowResult;
+use log::warn;
+use rss::Channel;
+
+const KOTAKU_RSS: &'static str = "https://kotaku.com/rss";
+
+// TODO: Rename foo_feed() or foo_rss(), etc.
+pub async fn kotaku_indexer() -> AnyhowResult<Vec<WebScrapingTarget>> {
+  let content = reqwest::get(KOTAKU_RSS)
+      .await?
+      .bytes()
+      .await?;
+
+  let channel = Channel::read_from(&content[..])?;
+
+  let mut targets = Vec::with_capacity(channel.items.len());
+
+  for item in channel.items {
+    let canonical_url = match item.link {
+      Some(url) => url.clone(),
+      None => {
+        warn!("Skipping item due to not having a URL");
+        continue;
+      }
+    };
+
+    let mut maybe_skip_reason = None;
+
+    //// Lazy heuristic to skip CNN video articles, which contain little text.
+    //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn
+    //if canonical_url.contains("/videos/") {
+    //  maybe_skip_reason = Some(SkipReason::VideoContent);
+    //}
+
+    // NB: I'm not using these as I had anticipated.
+    let maybe_image_url = None;
+
+    //let maybe_image_url = item.extensions.get("media")
+    //    .map(|media| media.get("group"))
+    //    .flatten()
+    //    .map(|group| group.get(0))
+    //    .flatten()
+    //    .map(|extension| extension.children.get("content"))
+    //    .flatten()
+    //    .map(|extensions| extensions.get(0)) // NB: First image is biggest
+    //    .flatten()
+    //    .map(|extension| extension.attrs.get("url"))
+    //    .flatten()
+    //    .map(|url| url.to_string());
+
+    targets.push(WebScrapingTarget {
+      canonical_url,
+      web_content_type: WebContentType::KotakuArticle,
+      maybe_title: item.title.clone(),
+      maybe_full_image_url: maybe_image_url,
+      maybe_thumbnail_image_url: None,
+      maybe_skip_reason,
+    });
+  }
+
+  Ok(targets)
+}

crates/lib/web_scrapers/src/sites/kotaku/mod.rs

@@ -1 +1,2 @@
-pub mod kotaku_article_scraper;
+pub mod kotaku_article_scraper;
+pub mod kotaku_indexer;

crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs

@@ -11,6 +11,8 @@ use strum::IntoEnumIterator;
 use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
 use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
 use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
+use web_scrapers::sites::gizmodo::gizmodo_indexer::gizmodo_indexer;
+use web_scrapers::sites::kotaku::kotaku_indexer::kotaku_indexer;
 use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};
 
 /// Download RSS feeds and index pages to determine which articles and content will be scraped (async/downstream of this)
@@ -37,24 +39,42 @@ pub async fn web_index_ingestion_main_loop(job_state: Arc<JobState>) {
 
 async fn single_iteration(job_state: &Arc<JobState>) -> AnyhowResult<()> {
 
+  // Kotaku
+  {
+    let index_targets = kotaku_indexer().await?;
+    insert_targets(job_state, &index_targets).await?;
+    thread::sleep(Duration::from_secs(2));
+  }
+
+  // Gizmodo
+  {
+    let index_targets = gizmodo_indexer().await?;
+    insert_targets(job_state, &index_targets).await?;
+    thread::sleep(Duration::from_secs(2));
+  }
+
   // CNN
   for variant in CnnFeed::iter() {
     let index_targets = cnn_indexer(variant).await?;
     insert_targets(job_state, &index_targets).await?;
-    thread::sleep(Duration::from_secs(5));
+    thread::sleep(Duration::from_secs(2));
   }
 
   // TechCrunch
   for variant in TechcrunchFeed::iter() {
     let index_targets = techcrunch_indexer(variant).await?;
     insert_targets(job_state, &index_targets).await?;
-    thread::sleep(Duration::from_secs(5));
+    thread::sleep(Duration::from_secs(2));
   }
 
   Ok(())
 }
 
 async fn insert_targets(job_state: &Arc<JobState>, targets: &Vec<WebScrapingTarget>) -> AnyhowResult<()> {
+  while job_state.app_control_state.is_scraping_paused() {
+    thread::sleep(Duration::from_secs(5));
+  }
+
   for target in targets.iter() {
     let _r = insert_web_scraping_target(Args {
       canonical_url: &target.canonical_url,