diff --git a/crates/_types/enums/src/common/sqlite/web_content_type.rs b/crates/_types/enums/src/common/sqlite/web_content_type.rs index c1dc66d55a..2344eb4b3b 100644 --- a/crates/_types/enums/src/common/sqlite/web_content_type.rs +++ b/crates/_types/enums/src/common/sqlite/web_content_type.rs @@ -12,6 +12,9 @@ use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] pub enum WebContentType { + #[serde(rename = "cbs_news_article")] + CbsNewsArticle, + #[serde(rename = "cnn_article")] CnnArticle, @@ -42,6 +45,7 @@ impl_sqlite_enum_coders!(WebContentType); impl WebContentType { pub fn to_str(&self) -> &'static str { match self { + Self::CbsNewsArticle => "cbs_news_article", Self::CnnArticle => "cnn_article", Self::HackerNewsThread => "hacker_news_thread", Self::RedditThread => "reddit_thread", @@ -54,6 +58,7 @@ impl WebContentType { pub fn from_str(value: &str) -> Result { match value { + "cbs_news_article" => Ok(Self::CbsNewsArticle), "cnn_article" => Ok(Self::CnnArticle), "hacker_news_thread" => Ok(Self::HackerNewsThread), "reddit_thread" => Ok(Self::RedditThread), @@ -76,6 +81,7 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(WebContentType::CbsNewsArticle, "cbs_news_article"); assert_serialization(WebContentType::CnnArticle, "cnn_article"); assert_serialization(WebContentType::HackerNewsThread, "hacker_news_thread"); assert_serialization(WebContentType::RedditThread, "reddit_thread"); @@ -91,6 +97,7 @@ mod tests { #[test] fn test_to_str() { + assert_eq!(WebContentType::CbsNewsArticle.to_str(), "cbs_news_article"); assert_eq!(WebContentType::CnnArticle.to_str(), "cnn_article"); assert_eq!(WebContentType::HackerNewsThread.to_str(), "hacker_news_thread"); assert_eq!(WebContentType::RedditThread.to_str(), "reddit_thread"); @@ -102,6 +109,7 @@ mod tests { #[test] fn test_from_str() { + assert_eq!(WebContentType::from_str("cbs_news_article").unwrap(), WebContentType::CbsNewsArticle); assert_eq!(WebContentType::from_str("cnn_article").unwrap(), WebContentType::CnnArticle); assert_eq!(WebContentType::from_str("hacker_news_thread").unwrap(), WebContentType::HackerNewsThread); assert_eq!(WebContentType::from_str("reddit_thread").unwrap(), WebContentType::RedditThread); diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs b/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs new file mode 100644 index 0000000000..7719ecc9e3 --- /dev/null +++ b/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs @@ -0,0 +1,41 @@ +use scraper::{Html, Selector}; + +pub fn extract_attribute(document: &Html, element_selector: &Selector, attribute_name: &str) -> Option { + let element = match document.select(element_selector).next() { + None => return None, + Some(element) => element, + }; + element.value() + .attr(attribute_name) + .map(|value| value.to_string()) +} + +//#[cfg(test)] +//mod tests { +// use crate::common_extractors::extract_text::extract_text; +// use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; +// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; +// use scraper::{Html, Selector}; +// +// #[test] +// fn test_extract_text_cnn() { +// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); +// let document = Html::parse_document(&html); +// +// let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); +// +// assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), +// maybe_text.as_deref()); +// } +// +// #[test] +// fn test_extract_text_techcrunch() { +// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); +// let document = Html::parse_document(&html); +// +// let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); +// +// assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), +// maybe_text.as_deref()); +// } +//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs index b0732605ea..546833e195 100644 --- a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs +++ b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs @@ -20,6 +20,7 @@ pub fn extract_text(document: &Html, text_selector: &Selector) -> Option #[cfg(test)] mod tests { + use crate::common_extractors::extract_text::extract_text; use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; use scraper::{Html, Selector}; diff --git a/crates/lib/web_scrapers/src/common_extractors/mod.rs b/crates/lib/web_scrapers/src/common_extractors/mod.rs index 6c29d2692f..8d7c564017 100644 --- a/crates/lib/web_scrapers/src/common_extractors/mod.rs +++ b/crates/lib/web_scrapers/src/common_extractors/mod.rs @@ -1,3 +1,4 @@ +pub mod extract_attribute; pub mod extract_featured_image; pub mod extract_text; pub mod extract_title; diff --git a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs index 3b873e352f..2a626b9702 100644 --- a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs +++ b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs @@ -4,6 +4,7 @@ use reqwest::Url; use errors::AnyhowResult; use crate::payloads::web_scraping_result::WebScrapingResult; use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError; +use crate::sites::cbsnews::cbsnews_article_scraper::cbsnews_article_scraper; use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper; use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper; @@ -33,6 +34,7 @@ pub async fn scrape_supported_webpage(url: &str) -> Result cbsnews_article_scraper(url).await, "cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await, "theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await, _ => return Err(ScrapeUtilityError::UnknownUrl), diff --git a/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs new file mode 100644 index 0000000000..713d9c7351 --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs @@ -0,0 +1,126 @@ +use crate::common_extractors::extract_featured_image::extract_featured_image; +use crate::common_extractors::extract_title::extract_title; +use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle}; +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::common::sqlite::web_content_type::WebContentType; +use errors::{anyhow, AnyhowResult}; +use log::{error, warn}; +use once_cell::sync::Lazy; +use rss::Channel; +use scraper::{Html, Selector}; +use std::ops::{Add, Deref}; +use chrono::{DateTime, Duration, Local, LocalResult, NaiveDate, NaiveDateTime, TimeZone, Utc}; +use crate::common_extractors::extract_attribute::extract_attribute; +use crate::common_extractors::extract_text::extract_text; +use crate::utils::remove_timestamp_abbreviated_weekday_name::remove_timestamp_abbreviated_weekday_name; + +/// The main article content container +static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { + Selector::parse("section.content__body").expect("this selector should parse") +}); + +/// Paragraphs within the article +static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { + Selector::parse("p").expect("this selector should parse") +}); + +/// The title of the article +pub static TITLE_SELECTOR: Lazy = Lazy::new(|| { + Selector::parse("h1.content__title").expect("this selector should parse") +}); + +///// The title of the article +//pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { +// Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse") +//}); + +/// The article featured image +pub static FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { + Selector::parse("figure .embed__content img").expect("this selector should parse") +}); + +pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { + Selector::parse(".content__meta--timestamp time").expect("this selector should parse") +}); + +pub async fn cbsnews_article_scraper(url: &str) -> AnyhowResult { + let downloaded_document= reqwest::get(url) + .await? + .bytes() + .await?; + + let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); + let document = Html::parse_document(&downloaded_document); + + let mut paragraphs = Vec::new(); + + if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { + for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { + + let mut paragraph_assembly = Vec::new(); + + for text in paragraph.text() { + let stripped = text.trim(); + if !stripped.is_empty() { + paragraph_assembly.push(stripped.to_string()); + } + } + + let paragraph_full_text = paragraph_assembly.join(" ") + .trim() + .to_string(); + + if !paragraph_full_text.is_empty() { + paragraphs.push(paragraph_full_text); + } + } + } + + let maybe_title = extract_title(&document, &TITLE_SELECTOR); + let maybe_featured_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR); + + let maybe_timestamp_raw = extract_attribute(&document, &DATETIME_SELECTOR, "datetime"); + + println!("Timestamp: {:?}", maybe_timestamp_raw.as_deref().unwrap()); + + // Timestamp is present as *almost* RFC3339: 2023-02-23T19:30:30-0500 + // Frustratingly, there should be a colon in the timezone offset. + let maybe_timestamp = maybe_timestamp_raw + .as_deref() + // This is a lazy af fix for the RFC3339 non-compliance. + .map(|invalid| invalid.replace("-0400", "-04:00")) + .map(|invalid| invalid.replace("-0500", "-05:00")) + .map(|invalid| invalid.replace("-0600", "-06:00")) + .map(|invalid| invalid.replace("-0700", "-07:00")) + .map(|invalid| invalid.replace("-0800", "-08:00")) + .map(|invalid| invalid.replace("-0900", "-09:00")) + .map(|ts| DateTime::parse_from_rfc3339(ts.trim())) + .transpose()? + // We need to juggle from DateTime to DateTime. + .map(|dt| dt.naive_utc()) + .map(|naive| match Utc.from_local_datetime(&naive) { + LocalResult::None => Err(anyhow!("invalid datetime conversion")), + LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), + LocalResult::Single(time) => Ok(time), + }) + .transpose()?; + + let body_text = paragraphs.join("\n\n"); + + Ok(WebScrapingResult { + original_html: downloaded_document, + result: ScrapedWebArticle { + url: url.to_string(), + web_content_type: WebContentType::CbsNewsArticle, + maybe_title, + maybe_subtitle: None, + maybe_author: None, // TODO + paragraphs, + body_text, + maybe_heading_image_url: None, // TODO + maybe_featured_image_url, + maybe_publish_timestamp_raw: maybe_timestamp_raw, + maybe_publish_datetime_utc: maybe_timestamp, + } + }) +} diff --git a/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs b/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs new file mode 100644 index 0000000000..d1e25b853d --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs @@ -0,0 +1 @@ +pub mod cbsnews_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/mod.rs b/crates/lib/web_scrapers/src/sites/mod.rs index 3c3cf843dc..1c96e8c4f7 100644 --- a/crates/lib/web_scrapers/src/sites/mod.rs +++ b/crates/lib/web_scrapers/src/sites/mod.rs @@ -5,4 +5,5 @@ pub mod reuters; pub mod slashdot; pub mod techcrunch; pub mod theguardian; -pub mod twitter; \ No newline at end of file +pub mod twitter; +pub mod cbsnews; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs index 7561dd5d39..019185e79f 100644 --- a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs +++ b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs @@ -8,6 +8,8 @@ const RSS_WORLD_NEWS : &'static str = "https://www.theguardian.com/world/rss"; const RSS_TECHNOLOGY : &'static str = "https://www.theguardian.com/technology/rss"; +// TODO: This hasn't been tested. + pub async fn theguardian_indexer() -> AnyhowResult> { let content = reqwest::get(RSS_WORLD_NEWS) .await?