TheGuardian scraper

This commit is contained in:
Brandon Thomas committed 2023-02-23 19:21:19 -05:00
1 parent ccb36f7f6e
commit 1cfb3bb715
8 files changed
+102 -3

No files matched your search

@@ -21,6 +21,9 @@ pub struct ScrapedWebArticle {
/// From page title or heading
pub maybe_title: Option<String>,
/// Some pages have subtitles under the main title.
pub maybe_subtitle: Option<String>,
/// The author of the article
pub maybe_author: Option<String>,
@@ -5,6 +5,7 @@ use errors::AnyhowResult;
use crate::payloads::web_scraping_result::WebScrapingResult;
use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError;
use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper;
use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper;
#[derive(Debug, Clone)]
pub enum ScrapeUtilityError {
@@ -33,6 +34,7 @@ pub async fn scrape_supported_webpage(url: &str) -> Result<WebScrapingResult, Sc
let maybe_result = match hostname.as_ref() {
"cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await,
"theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await,
_ => return Err(ScrapeUtilityError::UnknownUrl),
};
@@ -74,6 +74,7 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
url: url.to_string(),
web_content_type: WebContentType::CnnArticle,
maybe_title,
maybe_subtitle: None,
maybe_author: None, // TODO
paragraphs,
body_text,
@@ -77,6 +77,7 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingRe
url: url.to_string(),
web_content_type: WebContentType::TechCrunchArticle,
maybe_title,
maybe_subtitle: None,
maybe_author: None, // TODO
paragraphs,
body_text,
@@ -1 +1,2 @@
pub mod theguardian_scraper;
pub mod theguardian_indexer;
pub mod theguardian_article_scraper;
@@ -0,0 +1,91 @@
use crate::common_extractors::extract_featured_image::extract_featured_image;
use crate::common_extractors::extract_title::extract_title;
use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
use crate::payloads::web_scraping_target::WebScrapingTarget;
use enums::common::sqlite::web_content_type::WebContentType;
use errors::{anyhow, AnyhowResult};
use log::{error, warn};
use once_cell::sync::Lazy;
use rss::Channel;
use scraper::{Html, Selector};
use std::ops::Deref;
/// The main article content container
static ARTICLE_CONTENT_SELECTOR : Lazy<Selector> = Lazy::new(|| {
Selector::parse("#maincontent").expect("this selector should parse")
});
/// Paragraphs within the article
static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
Selector::parse("p").expect("this selector should parse")
});
/// The title of the article
pub static TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse("h1").expect("this selector should parse")
});
/// The title of the article
pub static SUBTITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse")
});
/// The article featured image
pub static FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse("figure > picture > img").expect("this selector should parse")
});
pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
let downloaded_document= reqwest::get(url)
.await?
.bytes()
.await?;
let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
let document = Html::parse_document(&downloaded_document);
let mut paragraphs = Vec::new();
if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {
for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() {
let mut paragraph_assembly = Vec::new();
for text in paragraph.text() {
let stripped = text.trim();
if !stripped.is_empty() {
paragraph_assembly.push(stripped.to_string());
}
}
let paragraph_full_text = paragraph_assembly.join(" ")
.trim()
.to_string();
if !paragraph_full_text.is_empty() {
paragraphs.push(paragraph_full_text);
}
}
}
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR);
let maybe_heading_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR);
let body_text = paragraphs.join("\n\n");
Ok(WebScrapingResult {
original_html: downloaded_document,
result: ScrapedWebArticle {
url: url.to_string(),
web_content_type: WebContentType::TheGuardianArticle,
maybe_title,
maybe_subtitle,
maybe_author: None, // TODO
paragraphs,
body_text,
maybe_heading_image_url,
maybe_featured_image_url: None, // TODO
}
})
}
@@ -8,7 +8,7 @@ const RSS_WORLD_NEWS : &'static str = "https://www.theguardian.com/world/rss";
const RSS_TECHNOLOGY : &'static str = "https://www.theguardian.com/technology/rss";
pub async fn theguardian_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
pub async fn theguardian_indexer() -> AnyhowResult<Vec<WebScrapingTarget>> {
let content = reqwest::get(RSS_WORLD_NEWS)
.await?
.bytes()
@@ -48,7 +48,7 @@ use std::time::Duration;
use tokio::runtime::{Builder, Runtime};
use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper;
use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper;
use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test;
use web_scrapers::sites::theguardian::theguardian_indexer::theguardian_scraper_test;
use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;