shared title/image extractors

This commit is contained in:
Brandon Thomas committed 2023-02-07 17:50:58 -05:00
1 parent ce17a3e1bf
commit bc58b50408
6 files changed
+9709 -17

No files matched your search

@@ -0,0 +1,47 @@
use scraper::{Html, Selector};
pub fn extract_featured_image(document: &Html, image_selector: &Selector) -> Option<String> {
let image = match document.select(image_selector).next() {
None => return None,
Some(title_element) => title_element,
};
if let Some(img_source) = image.value().attr("src") {
return Some(img_source.to_string());
}
// TODO: We might want more heuristics here, though there's a danger this method gets complicated.
None
}
#[cfg(test)]
mod tests {
use crate::common_extractors::extract_featured_image::extract_featured_image;
use crate::sites::cnn::cnn_article_scraper::CNN_FEATURED_IMAGE_SELECTOR;
use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_FEATURED_IMAGE_SELECTOR;
use scraper::{Html, Selector};
#[test]
fn test_extract_cnn_with_video() {
let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
let document = Html::parse_document(&html);
let maybe_image = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR);
assert_eq!(Some("https://media.cnn.com/api/v1/images/stellar/prod/220426143055-bill-ford-executive-chairman-ford-motor-company.jpg?c=16x9&q=w_850,c_fill"),
maybe_image.as_deref());
}
#[test]
fn test_extract_techcrunch() {
let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
let document = Html::parse_document(&html);
let selector = Selector::parse(".article__title").expect("selector should parse");
let maybe_image = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR);
assert_eq!(Some("https://techcrunch.com/wp-content/uploads/2023/02/Screen-Shot-2023-02-02-at-1.46.55-PM.png?w=650"),
maybe_image.as_deref());
}
}
@@ -20,28 +20,28 @@ pub fn extract_title(document: &Html, title_selector: &Selector) -> Option<Strin
#[cfg(test)]
mod tests {
use crate::common_extractors::extract_title::extract_title;
use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR;
use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR;
use scraper::{Html, Selector};
use crate::common_extractors::title_extractor::extract_title;
#[test]
fn text_extract_title_cnn() {
let html = include_str!("../../../../../test_data/html_scraping/cnn_article.html");
fn test_extract_title_cnn() {
let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
let document = Html::parse_document(&html);
let selector = Selector::parse(".headline__text").expect("selector should parse");
let maybe_title = extract_title(&document, &selector);
let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"),
maybe_title.as_deref());
}
#[test]
fn text_extract_title_techcrunch() {
fn test_extract_title_techcrunch() {
let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
let document = Html::parse_document(&html);
let selector = Selector::parse(".article__title").expect("selector should parse");
let maybe_title = extract_title(&document, &selector);
let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"),
maybe_title.as_deref());
@@ -1 +1,3 @@
pub mod title_extractor;
pub mod extract_featured_image;
pub mod extract_title;
@@ -1,4 +1,4 @@
use crate::common_extractors::title_extractor::extract_title;
use crate::common_extractors::extract_title::extract_title;
use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
use crate::payloads::web_scraping_target::WebScrapingTarget;
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
@@ -20,10 +20,15 @@ static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
});
/// The title of the article
static TITLE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
pub static CNN_TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse(".headline__text").expect("this selector should parse")
});
/// The article featured image
pub static CNN_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse(".image__lede .image__picture > img").expect("this selector should parse")
});
pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
let downloaded_document= reqwest::get(url)
.await?
@@ -62,7 +67,7 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebS
}
}
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
let body_text = paragraphs.join("\n\n");
@@ -1,4 +1,4 @@
use crate::common_extractors::title_extractor::extract_title;
use crate::common_extractors::extract_title::extract_title;
use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
use crate::payloads::web_scraping_target::WebScrapingTarget;
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
@@ -23,10 +23,15 @@ static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
});
/// The title of the article
static TITLE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
pub static TECHCRUNCH_TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse(".article__title").expect("this selector should parse")
});
/// The article featured image
pub static TECHCRUNCH_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
Selector::parse(".article__featured-image").expect("this selector should parse")
});
pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
let downloaded_document= reqwest::get(url)
.await?
@@ -36,8 +41,6 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
let document = Html::parse_document(&downloaded_document);
let mut article_title = None;
let mut paragraphs = Vec::new();
if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {
@@ -62,7 +65,7 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
}
}
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
let body_text = paragraphs.join("\n\n");
File diff suppressed because it is too large. Load diff