Add timestamp to theguardian scraper

crates/lib/web_scrapers/src/common_extractors/extract_text.rs

@@ -0,0 +1,48 @@
+use scraper::{Html, Selector};
+
+pub fn extract_text(document: &Html, text_selector: &Selector) -> Option<String> {
+  let text_element = match document.select(text_selector).next() {
+    None => return None,
+    Some(text_element) => text_element,
+  };
+
+  let mut pieces = Vec::new();
+
+  for mut text in text_element.text() {
+    text = text.trim();
+    if !text.is_empty() {
+      pieces.push(text.to_string());
+    }
+  }
+
+  Some(pieces.join(" ").trim().to_string())
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR;
+  use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR;
+  use scraper::{Html, Selector};
+
+  #[test]
+  fn test_extract_text_cnn() {
+    let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
+    let document = Html::parse_document(&html);
+
+    let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR);
+
+    assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"),
+               maybe_text.as_deref());
+  }
+
+  #[test]
+  fn test_extract_text_techcrunch() {
+    let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
+    let document = Html::parse_document(&html);
+
+    let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR);
+
+    assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"),
+               maybe_text.as_deref());
+  }
+}

crates/lib/web_scrapers/src/common_extractors/mod.rs

@@ -1,3 +1,4 @@
 pub mod extract_featured_image;
+pub mod extract_text;
 pub mod extract_title;
 

crates/lib/web_scrapers/src/lib.rs

@@ -25,3 +25,4 @@ pub mod common_extractors;
 pub mod payloads;
 pub mod scrape_supported_webpage;
 pub mod sites;
+pub (crate) mod utils;

crates/lib/web_scrapers/src/payloads/web_scraping_result.rs

@@ -1,3 +1,4 @@
+use chrono::{DateTime, Utc};
 use enums::common::sqlite::web_content_type::WebContentType;
 
 // TODO: Rename this to not confuse with `Result<T, E>`.
@@ -39,8 +40,11 @@ pub struct ScrapedWebArticle {
   /// A featured image (somewhere in the body), if present
   pub maybe_featured_image_url: Option<String>,
 
-  // TODO: Add publish date
-  // pub maybe_publish_date_utc: Option<DateTime<Utc>>
+  /// A timestamp that has not been parsed to a proper time
+  pub maybe_publish_timestamp_raw: Option<String>,
+
+  /// A timestamp that has not been parsed to a proper time
+  pub maybe_publish_datetime_utc: Option<DateTime<Utc>>,
 }
 
 // TODO: Other types - threaded discussions (HN, Reddit), video essays (CNN, YouTube), etc.

crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs

@@ -80,6 +80,8 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
       body_text,
       maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
+      maybe_publish_timestamp_raw: None,
+      maybe_publish_datetime_utc: None,
     }
   })
 }

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs

@@ -83,6 +83,8 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingRe
       body_text,
       maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
+      maybe_publish_timestamp_raw: None,
+      maybe_publish_datetime_utc: None,
     }
   })
 }

crates/lib/web_scrapers/src/sites/theguardian/theguardian_article_scraper.rs

@@ -8,7 +8,10 @@ use log::{error, warn};
 use once_cell::sync::Lazy;
 use rss::Channel;
 use scraper::{Html, Selector};
-use std::ops::Deref;
+use std::ops::{Add, Deref};
+use chrono::{DateTime, Duration, Local, LocalResult, NaiveDate, NaiveDateTime, TimeZone, Utc};
+use crate::common_extractors::extract_text::extract_text;
+use crate::utils::remove_timestamp_abbreviated_weekday_name::remove_timestamp_abbreviated_weekday_name;
 
 /// The main article content container
 static ARTICLE_CONTENT_SELECTOR : Lazy<Selector> = Lazy::new(|| {
@@ -32,7 +35,11 @@ pub static SUBTITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
 
 /// The article featured image
 pub static FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("figure > picture > img").expect("this selector should parse")
+  Selector::parse("figure picture img").expect("this selector should parse")
+});
+
+pub static DATETIME_SELECTOR : Lazy<Selector> = Lazy::new(|| {
+  Selector::parse("aside[data-gu-name=\"meta\"] summary span").expect("this selector should parse")
 });
 
 pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
@@ -72,6 +79,27 @@ pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult<WebScrapingR
   let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR);
   let maybe_heading_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR);
 
+  let maybe_timestamp_raw = extract_text(&document, &DATETIME_SELECTOR);
+
+  // Format on webpage: "Thu 23 Feb 2023 12.39 EST",
+  // Parsed with: "Thu (%a) 23 (%e) Feb (%b) 2023 (%Y) 12 (%H).39 (%M) EST (%Z)",
+  // However, the "%a" (abbreviated weekday) fails to parse, so we remove it.
+  // Furthermore, it's unable to use the timezone data and the library hates it, so we
+  // have to type juggle.
+  let maybe_timestamp = maybe_timestamp_raw
+      .as_deref()
+      .map(|ts| remove_timestamp_abbreviated_weekday_name(&ts))
+      .map(|ts| NaiveDateTime::parse_from_str(&ts, "%e %b %Y %H.%M %Z"))
+      .transpose()?
+      .map(|naive| match Utc.from_local_datetime(&naive) {
+        LocalResult::None => Err(anyhow!("invalid datetime conversion")),
+        LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")),
+        LocalResult::Single(time) => Ok(time),
+      })
+      .transpose()?
+      // NB: The "Utc" time above is actually EST/EDT.
+      .map(|utc| utc.add(Duration::hours(5)));
+
   let body_text = paragraphs.join("\n\n");
 
   Ok(WebScrapingResult {
@@ -86,6 +114,8 @@ pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult<WebScrapingR
       body_text,
       maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
+      maybe_publish_timestamp_raw: maybe_timestamp_raw,
+      maybe_publish_datetime_utc: maybe_timestamp,
     }
   })
 }

crates/lib/web_scrapers/src/utils/mod.rs

@@ -0,0 +1 @@
+pub (crate) mod remove_timestamp_abbreviated_weekday_name;

crates/lib/web_scrapers/src/utils/remove_timestamp_abbreviated_weekday_name.rs

@@ -0,0 +1,12 @@
+// NB: Having a hard time parsing dates with "%a", and it also appears to be an issue in other
+// languages, eg. PHP online tools for strptime, etc. Let's just get rid of it.
+pub fn remove_timestamp_abbreviated_weekday_name(timestamp: &str) -> String {
+  // TODO: More efficient regex-based solution
+  timestamp.replace("Sun", "")
+      .replace("Mon", "")
+      .replace("Tue", "")
+      .replace("Wed", "")
+      .replace("Thu", "")
+      .replace("Fri", "")
+      .replace("Sat", "")
+}