mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-17 07:04:36 +00:00
TheGuardian scraper
This commit is contained in:
1 parent
ccb36f7f6e
commit
1cfb3bb715
8 files changed
+102
-3
No files matched your search
@@ -21,6 +21,9 @@ pub struct ScrapedWebArticle {
|
||||
/// From page title or heading
|
||||
pub maybe_title: Option<String>,
|
||||
|
||||
/// Some pages have subtitles under the main title.
|
||||
pub maybe_subtitle: Option<String>,
|
||||
|
||||
/// The author of the article
|
||||
pub maybe_author: Option<String>,
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ use errors::AnyhowResult;
|
||||
use crate::payloads::web_scraping_result::WebScrapingResult;
|
||||
use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError;
|
||||
use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper;
|
||||
use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ScrapeUtilityError {
|
||||
@@ -33,6 +34,7 @@ pub async fn scrape_supported_webpage(url: &str) -> Result<WebScrapingResult, Sc
|
||||
|
||||
let maybe_result = match hostname.as_ref() {
|
||||
"cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await,
|
||||
"theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await,
|
||||
_ => return Err(ScrapeUtilityError::UnknownUrl),
|
||||
};
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
|
||||
url: url.to_string(),
|
||||
web_content_type: WebContentType::CnnArticle,
|
||||
maybe_title,
|
||||
maybe_subtitle: None,
|
||||
maybe_author: None, // TODO
|
||||
paragraphs,
|
||||
body_text,
|
||||
|
||||
@@ -77,6 +77,7 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingRe
|
||||
url: url.to_string(),
|
||||
web_content_type: WebContentType::TechCrunchArticle,
|
||||
maybe_title,
|
||||
maybe_subtitle: None,
|
||||
maybe_author: None, // TODO
|
||||
paragraphs,
|
||||
body_text,
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
pub mod theguardian_scraper;
|
||||
pub mod theguardian_indexer;
|
||||
pub mod theguardian_article_scraper;
|
||||
@@ -0,0 +1,91 @@
|
||||
use crate::common_extractors::extract_featured_image::extract_featured_image;
|
||||
use crate::common_extractors::extract_title::extract_title;
|
||||
use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
|
||||
use crate::payloads::web_scraping_target::WebScrapingTarget;
|
||||
use enums::common::sqlite::web_content_type::WebContentType;
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use log::{error, warn};
|
||||
use once_cell::sync::Lazy;
|
||||
use rss::Channel;
|
||||
use scraper::{Html, Selector};
|
||||
use std::ops::Deref;
|
||||
|
||||
/// The main article content container
|
||||
static ARTICLE_CONTENT_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse("#maincontent").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// Paragraphs within the article
|
||||
static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse("p").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// The title of the article
|
||||
pub static TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse("h1").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// The title of the article
|
||||
pub static SUBTITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// The article featured image
|
||||
pub static FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse("figure > picture > img").expect("this selector should parse")
|
||||
});
|
||||
|
||||
pub async fn theguardian_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
|
||||
let downloaded_document= reqwest::get(url)
|
||||
.await?
|
||||
.bytes()
|
||||
.await?;
|
||||
|
||||
let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
|
||||
let document = Html::parse_document(&downloaded_document);
|
||||
|
||||
let mut paragraphs = Vec::new();
|
||||
|
||||
if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {
|
||||
for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() {
|
||||
|
||||
let mut paragraph_assembly = Vec::new();
|
||||
|
||||
for text in paragraph.text() {
|
||||
let stripped = text.trim();
|
||||
if !stripped.is_empty() {
|
||||
paragraph_assembly.push(stripped.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let paragraph_full_text = paragraph_assembly.join(" ")
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
if !paragraph_full_text.is_empty() {
|
||||
paragraphs.push(paragraph_full_text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
|
||||
let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR);
|
||||
let maybe_heading_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR);
|
||||
|
||||
let body_text = paragraphs.join("\n\n");
|
||||
|
||||
Ok(WebScrapingResult {
|
||||
original_html: downloaded_document,
|
||||
result: ScrapedWebArticle {
|
||||
url: url.to_string(),
|
||||
web_content_type: WebContentType::TheGuardianArticle,
|
||||
maybe_title,
|
||||
maybe_subtitle,
|
||||
maybe_author: None, // TODO
|
||||
paragraphs,
|
||||
body_text,
|
||||
maybe_heading_image_url,
|
||||
maybe_featured_image_url: None, // TODO
|
||||
}
|
||||
})
|
||||
}
|
||||
+1
-1
@@ -8,7 +8,7 @@ const RSS_WORLD_NEWS : &'static str = "https://www.theguardian.com/world/rss";
|
||||
|
||||
const RSS_TECHNOLOGY : &'static str = "https://www.theguardian.com/technology/rss";
|
||||
|
||||
pub async fn theguardian_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
|
||||
pub async fn theguardian_indexer() -> AnyhowResult<Vec<WebScrapingTarget>> {
|
||||
let content = reqwest::get(RSS_WORLD_NEWS)
|
||||
.await?
|
||||
.bytes()
|
||||
@@ -48,7 +48,7 @@ use std::time::Duration;
|
||||
use tokio::runtime::{Builder, Runtime};
|
||||
use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper;
|
||||
use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper;
|
||||
use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test;
|
||||
use web_scrapers::sites::theguardian::theguardian_indexer::theguardian_scraper_test;
|
||||
use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
|
||||
use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;
|
||||
|
||||
|
||||
Reference in new issue
Block a user