diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs index b147cba1d0..dcaa48cb04 100644 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs @@ -21,6 +21,9 @@ pub struct ScrapedWebArticle { /// From page title or heading pub maybe_title: Option, + /// Some pages have subtitles under the main title. + pub maybe_subtitle: Option, + /// The author of the article pub maybe_author: Option, diff --git a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs index 18676993ca..3b873e352f 100644 --- a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs +++ b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs @@ -5,6 +5,7 @@ use errors::AnyhowResult; use crate::payloads::web_scraping_result::WebScrapingResult; use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError; use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper; +use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper; #[derive(Debug, Clone)] pub enum ScrapeUtilityError { @@ -33,6 +34,7 @@ pub async fn scrape_supported_webpage(url: &str) -> Result cnn_article_scraper(url).await, + "theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await, _ => return Err(ScrapeUtilityError::UnknownUrl), }; diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs index 1902c491f0..e5c67a8cb5 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs @@ -74,6 +74,7 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult { url: url.to_string(), web_content_type: WebContentType::CnnArticle, maybe_title, + maybe_subtitle: None, maybe_author: None, // TODO paragraphs, body_text, diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs index cc1acf0a1c..3e0303e9bc 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs @@ -77,6 +77,7 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult = Lazy::new(|| { + Selector::parse("#maincontent").expect("this selector should parse") +}); + +/// Paragraphs within the article +static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { + Selector::parse("p").expect("this selector should parse") +}); + +/// The title of the article +pub static TITLE_SELECTOR: Lazy = Lazy::new(|| { + Selector::parse("h1").expect("this selector should parse") +}); + +/// The title of the article +pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { + Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse") +}); + +/// The article featured image +pub static FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { + Selector::parse("figure > picture > img").expect("this selector should parse") +}); + +pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult { + let downloaded_document= reqwest::get(url) + .await? + .bytes() + .await?; + + let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); + let document = Html::parse_document(&downloaded_document); + + let mut paragraphs = Vec::new(); + + if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { + for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { + + let mut paragraph_assembly = Vec::new(); + + for text in paragraph.text() { + let stripped = text.trim(); + if !stripped.is_empty() { + paragraph_assembly.push(stripped.to_string()); + } + } + + let paragraph_full_text = paragraph_assembly.join(" ") + .trim() + .to_string(); + + if !paragraph_full_text.is_empty() { + paragraphs.push(paragraph_full_text); + } + } + } + + let maybe_title = extract_title(&document, &TITLE_SELECTOR); + let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR); + let maybe_heading_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR); + + let body_text = paragraphs.join("\n\n"); + + Ok(WebScrapingResult { + original_html: downloaded_document, + result: ScrapedWebArticle { + url: url.to_string(), + web_content_type: WebContentType::TheGuardianArticle, + maybe_title, + maybe_subtitle, + maybe_author: None, // TODO + paragraphs, + body_text, + maybe_heading_image_url, + maybe_featured_image_url: None, // TODO + } + }) +} diff --git a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs similarity index 95% rename from crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs rename to crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs index 09d8d99b8f..7561dd5d39 100644 --- a/crates/lib/web_scrapers/src/sites/theguardian/theguardian_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/theguardian/theguardian_indexer.rs @@ -8,7 +8,7 @@ const RSS_WORLD_NEWS : &'static str = "https://www.theguardian.com/world/rss"; const RSS_TECHNOLOGY : &'static str = "https://www.theguardian.com/technology/rss"; -pub async fn theguardian_scraper_test() -> AnyhowResult> { +pub async fn theguardian_indexer() -> AnyhowResult> { let content = reqwest::get(RSS_WORLD_NEWS) .await? .bytes() diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 2fcfb10ec7..3f58ae5dff 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -48,7 +48,7 @@ use std::time::Duration; use tokio::runtime::{Builder, Runtime}; use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper; use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; -use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; +use web_scrapers::sites::theguardian::theguardian_indexer::theguardian_scraper_test; use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop; use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;