CBS News scraper

crates/_types/enums/src/common/sqlite/web_content_type.rs

@@ -12,6 +12,9 @@ use strum::EnumIter;
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
 pub enum WebContentType {
+  #[serde(rename = "cbs_news_article")]
+  CbsNewsArticle,
+
   #[serde(rename = "cnn_article")]
   CnnArticle,
 
@@ -42,6 +45,7 @@ impl_sqlite_enum_coders!(WebContentType);
 impl WebContentType {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::CbsNewsArticle => "cbs_news_article",
       Self::CnnArticle => "cnn_article",
       Self::HackerNewsThread => "hacker_news_thread",
       Self::RedditThread => "reddit_thread",
@@ -54,6 +58,7 @@ impl WebContentType {
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "cbs_news_article" => Ok(Self::CbsNewsArticle),
       "cnn_article" => Ok(Self::CnnArticle),
       "hacker_news_thread" => Ok(Self::HackerNewsThread),
       "reddit_thread" => Ok(Self::RedditThread),
@@ -76,6 +81,7 @@ mod tests {
 
     #[test]
     fn test_serialization() {
+      assert_serialization(WebContentType::CbsNewsArticle, "cbs_news_article");
       assert_serialization(WebContentType::CnnArticle, "cnn_article");
       assert_serialization(WebContentType::HackerNewsThread, "hacker_news_thread");
       assert_serialization(WebContentType::RedditThread, "reddit_thread");
@@ -91,6 +97,7 @@ mod tests {
 
     #[test]
     fn test_to_str() {
+      assert_eq!(WebContentType::CbsNewsArticle.to_str(), "cbs_news_article");
       assert_eq!(WebContentType::CnnArticle.to_str(), "cnn_article");
       assert_eq!(WebContentType::HackerNewsThread.to_str(), "hacker_news_thread");
       assert_eq!(WebContentType::RedditThread.to_str(), "reddit_thread");
@@ -102,6 +109,7 @@ mod tests {
 
     #[test]
     fn test_from_str() {
+      assert_eq!(WebContentType::from_str("cbs_news_article").unwrap(), WebContentType::CbsNewsArticle);
       assert_eq!(WebContentType::from_str("cnn_article").unwrap(), WebContentType::CnnArticle);
       assert_eq!(WebContentType::from_str("hacker_news_thread").unwrap(), WebContentType::HackerNewsThread);
       assert_eq!(WebContentType::from_str("reddit_thread").unwrap(), WebContentType::RedditThread);

crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs

@@ -0,0 +1,41 @@
+use scraper::{Html, Selector};
+
+pub fn extract_attribute(document: &Html, element_selector: &Selector, attribute_name: &str) -> Option<String> {
+  let element = match document.select(element_selector).next() {
+    None => return None,
+    Some(element) => element,
+  };
+  element.value()
+      .attr(attribute_name)
+      .map(|value| value.to_string())
+}
+
+//#[cfg(test)]
+//mod tests {
+//  use crate::common_extractors::extract_text::extract_text;
+//  use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR;
+//  use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR;
+//  use scraper::{Html, Selector};
+//
+//  #[test]
+//  fn test_extract_text_cnn() {
+//    let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
+//    let document = Html::parse_document(&html);
+//
+//    let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR);
+//
+//    assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"),
+//               maybe_text.as_deref());
+//  }
+//
+//  #[test]
+//  fn test_extract_text_techcrunch() {
+//    let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
+//    let document = Html::parse_document(&html);
+//
+//    let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR);
+//
+//    assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"),
+//               maybe_text.as_deref());
+//  }
+//}

crates/lib/web_scrapers/src/common_extractors/extract_text.rs

@@ -20,6 +20,7 @@ pub fn extract_text(document: &Html, text_selector: &Selector) -> Option<String>
 
 #[cfg(test)]
 mod tests {
+  use crate::common_extractors::extract_text::extract_text;
   use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR;
   use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR;
   use scraper::{Html, Selector};

crates/lib/web_scrapers/src/common_extractors/mod.rs

@@ -1,3 +1,4 @@
+pub mod extract_attribute;
 pub mod extract_featured_image;
 pub mod extract_text;
 pub mod extract_title;

crates/lib/web_scrapers/src/scrape_supported_webpage.rs

@@ -4,6 +4,7 @@ use reqwest::Url;
 use errors::AnyhowResult;
 use crate::payloads::web_scraping_result::WebScrapingResult;
 use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError;
+use crate::sites::cbsnews::cbsnews_article_scraper::cbsnews_article_scraper;
 use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper;
 use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper;
 
@@ -33,6 +34,7 @@ pub async fn scrape_supported_webpage(url: &str) -> Result<WebScrapingResult, Sc
       .to_string();
 
   let maybe_result = match hostname.as_ref() {
+    "cbsnews.com" | "www.cbsnews.com" => cbsnews_article_scraper(url).await,
     "cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await,
     "theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await,
     _ => return Err(ScrapeUtilityError::UnknownUrl),

crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs

@@ -0,0 +1,126 @@
+use crate::common_extractors::extract_featured_image::extract_featured_image;
+use crate::common_extractors::extract_title::extract_title;
+use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::common::sqlite::web_content_type::WebContentType;
+use errors::{anyhow, AnyhowResult};
+use log::{error, warn};
+use once_cell::sync::Lazy;
+use rss::Channel;
+use scraper::{Html, Selector};
+use std::ops::{Add, Deref};
+use chrono::{DateTime, Duration, Local, LocalResult, NaiveDate, NaiveDateTime, TimeZone, Utc};
+use crate::common_extractors::extract_attribute::extract_attribute;
+use crate::common_extractors::extract_text::extract_text;
+use crate::utils::remove_timestamp_abbreviated_weekday_name::remove_timestamp_abbreviated_weekday_name;
+
+/// The main article content container
+static ARTICLE_CONTENT_SELECTOR : Lazy<Selector> = Lazy::new(|| {
+  Selector::parse("section.content__body").expect("this selector should parse")
+});
+
+/// Paragraphs within the article
+static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
+  Selector::parse("p").expect("this selector should parse")
+});
+
+/// The title of the article
+pub static TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
+  Selector::parse("h1.content__title").expect("this selector should parse")
+});
+
+///// The title of the article
+//pub static SUBTITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
+//  Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse")
+//});
+
+/// The article featured image
+pub static FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
+  Selector::parse("figure .embed__content img").expect("this selector should parse")
+});
+
+pub static DATETIME_SELECTOR : Lazy<Selector> = Lazy::new(|| {
+  Selector::parse(".content__meta--timestamp time").expect("this selector should parse")
+});
+
+pub async fn cbsnews_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
+  let downloaded_document= reqwest::get(url)
+      .await?
+      .bytes()
+      .await?;
+
+  let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
+  let document = Html::parse_document(&downloaded_document);
+
+  let mut paragraphs = Vec::new();
+
+  if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {
+    for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() {
+
+      let mut paragraph_assembly = Vec::new();
+
+      for text in paragraph.text() {
+        let stripped = text.trim();
+        if !stripped.is_empty() {
+          paragraph_assembly.push(stripped.to_string());
+        }
+      }
+
+      let paragraph_full_text = paragraph_assembly.join(" ")
+          .trim()
+          .to_string();
+
+      if !paragraph_full_text.is_empty() {
+        paragraphs.push(paragraph_full_text);
+      }
+    }
+  }
+
+  let maybe_title = extract_title(&document, &TITLE_SELECTOR);
+  let maybe_featured_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR);
+
+  let maybe_timestamp_raw = extract_attribute(&document, &DATETIME_SELECTOR, "datetime");
+
+  println!("Timestamp: {:?}", maybe_timestamp_raw.as_deref().unwrap());
+
+  // Timestamp is present as *almost* RFC3339: 2023-02-23T19:30:30-0500
+  // Frustratingly, there should be a colon in the timezone offset.
+  let maybe_timestamp = maybe_timestamp_raw
+      .as_deref()
+      // This is a lazy af fix for the RFC3339 non-compliance.
+      .map(|invalid| invalid.replace("-0400", "-04:00"))
+      .map(|invalid| invalid.replace("-0500", "-05:00"))
+      .map(|invalid| invalid.replace("-0600", "-06:00"))
+      .map(|invalid| invalid.replace("-0700", "-07:00"))
+      .map(|invalid| invalid.replace("-0800", "-08:00"))
+      .map(|invalid| invalid.replace("-0900", "-09:00"))
+      .map(|ts| DateTime::parse_from_rfc3339(ts.trim()))
+      .transpose()?
+      // We need to juggle from DateTime<NaiveOffset> to DateTime<Utc>.
+      .map(|dt| dt.naive_utc())
+      .map(|naive| match Utc.from_local_datetime(&naive) {
+        LocalResult::None => Err(anyhow!("invalid datetime conversion")),
+        LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")),
+        LocalResult::Single(time) => Ok(time),
+      })
+      .transpose()?;
+
+  let body_text = paragraphs.join("\n\n");
+
+  Ok(WebScrapingResult {
+    original_html: downloaded_document,
+    result: ScrapedWebArticle {
+      url: url.to_string(),
+      web_content_type: WebContentType::CbsNewsArticle,
+      maybe_title,
+      maybe_subtitle: None,
+      maybe_author: None, // TODO
+      paragraphs,
+      body_text,
+      maybe_heading_image_url: None, // TODO
+      maybe_featured_image_url,
+      maybe_publish_timestamp_raw: maybe_timestamp_raw,
+      maybe_publish_datetime_utc: maybe_timestamp,
+    }
+  })
+}

crates/lib/web_scrapers/src/sites/cbsnews/mod.rs

@@ -0,0 +1 @@
+pub mod cbsnews_article_scraper;

crates/lib/web_scrapers/src/sites/mod.rs

@@ -5,4 +5,5 @@ pub mod reuters;
 pub mod slashdot;
 pub mod techcrunch;
 pub mod theguardian;
-pub mod twitter;
+pub mod twitter;
+pub mod cbsnews;

crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs

@@ -8,6 +8,8 @@ const RSS_WORLD_NEWS : &'static str = "https://www.theguardian.com/world/rss";
 
 const RSS_TECHNOLOGY : &'static str = "https://www.theguardian.com/technology/rss";
 
+// TODO: This hasn't been tested.
+
 pub async fn theguardian_indexer() -> AnyhowResult<Vec<WebScrapingTarget>> {
   let content = reqwest::get(RSS_WORLD_NEWS)
       .await?