diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs new file mode 100644 index 0000000000..b0732605ea --- /dev/null +++ b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs @@ -0,0 +1,48 @@ +use scraper::{Html, Selector}; + +pub fn extract_text(document: &Html, text_selector: &Selector) -> Option { + let text_element = match document.select(text_selector).next() { + None => return None, + Some(text_element) => text_element, + }; + + let mut pieces = Vec::new(); + + for mut text in text_element.text() { + text = text.trim(); + if !text.is_empty() { + pieces.push(text.to_string()); + } + } + + Some(pieces.join(" ").trim().to_string()) +} + +#[cfg(test)] +mod tests { + use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; + use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; + use scraper::{Html, Selector}; + + #[test] + fn test_extract_text_cnn() { + let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); + let document = Html::parse_document(&html); + + let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); + + assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), + maybe_text.as_deref()); + } + + #[test] + fn test_extract_text_techcrunch() { + let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); + let document = Html::parse_document(&html); + + let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); + + assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), + maybe_text.as_deref()); + } +} diff --git a/crates/lib/web_scrapers/src/common_extractors/mod.rs b/crates/lib/web_scrapers/src/common_extractors/mod.rs index 06d313d7bd..6c29d2692f 100644 --- a/crates/lib/web_scrapers/src/common_extractors/mod.rs +++ b/crates/lib/web_scrapers/src/common_extractors/mod.rs @@ -1,3 +1,4 @@ pub mod extract_featured_image; +pub mod extract_text; pub mod extract_title; diff --git a/crates/lib/web_scrapers/src/lib.rs b/crates/lib/web_scrapers/src/lib.rs index 4f0b384074..d531794638 100644 --- a/crates/lib/web_scrapers/src/lib.rs +++ b/crates/lib/web_scrapers/src/lib.rs @@ -25,3 +25,4 @@ pub mod common_extractors; pub mod payloads; pub mod scrape_supported_webpage; pub mod sites; +pub (crate) mod utils; diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs index dcaa48cb04..32065feb32 100644 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use enums::common::sqlite::web_content_type::WebContentType; // TODO: Rename this to not confuse with `Result`. @@ -39,8 +40,11 @@ pub struct ScrapedWebArticle { /// A featured image (somewhere in the body), if present pub maybe_featured_image_url: Option, - // TODO: Add publish date - // pub maybe_publish_date_utc: Option> + /// A timestamp that has not been parsed to a proper time + pub maybe_publish_timestamp_raw: Option, + + /// A timestamp that has not been parsed to a proper time + pub maybe_publish_datetime_utc: Option>, } // TODO: Other types - threaded discussions (HN, Reddit), video essays (CNN, YouTube), etc. diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs index e5c67a8cb5..d3aa7eb628 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs @@ -80,6 +80,8 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult { body_text, maybe_heading_image_url, maybe_featured_image_url: None, // TODO + maybe_publish_timestamp_raw: None, + maybe_publish_datetime_utc: None, } }) } diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs index 3e0303e9bc..260005fc4c 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs @@ -83,6 +83,8 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult = Lazy::new(|| { @@ -32,7 +35,11 @@ pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { /// The article featured image pub static FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("figure > picture > img").expect("this selector should parse") + Selector::parse("figure picture img").expect("this selector should parse") +}); + +pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { + Selector::parse("aside[data-gu-name=\"meta\"] summary span").expect("this selector should parse") }); pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult { @@ -72,6 +79,27 @@ pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult Err(anyhow!("invalid datetime conversion")), + LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), + LocalResult::Single(time) => Ok(time), + }) + .transpose()? + // NB: The "Utc" time above is actually EST/EDT. + .map(|utc| utc.add(Duration::hours(5))); + let body_text = paragraphs.join("\n\n"); Ok(WebScrapingResult { @@ -86,6 +114,8 @@ pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult String { + // TODO: More efficient regex-based solution + timestamp.replace("Sun", "") + .replace("Mon", "") + .replace("Tue", "") + .replace("Wed", "") + .replace("Thu", "") + .replace("Fri", "") + .replace("Sat", "") +} \ No newline at end of file