mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-16 14:52:31 +00:00
shared title/image extractors
This commit is contained in:
1 parent
ce17a3e1bf
commit
bc58b50408
6 files changed
+9709
-17
No files matched your search
@@ -0,0 +1,47 @@
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
pub fn extract_featured_image(document: &Html, image_selector: &Selector) -> Option<String> {
|
||||
let image = match document.select(image_selector).next() {
|
||||
None => return None,
|
||||
Some(title_element) => title_element,
|
||||
};
|
||||
|
||||
if let Some(img_source) = image.value().attr("src") {
|
||||
return Some(img_source.to_string());
|
||||
}
|
||||
|
||||
// TODO: We might want more heuristics here, though there's a danger this method gets complicated.
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common_extractors::extract_featured_image::extract_featured_image;
|
||||
use crate::sites::cnn::cnn_article_scraper::CNN_FEATURED_IMAGE_SELECTOR;
|
||||
use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_FEATURED_IMAGE_SELECTOR;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
#[test]
|
||||
fn test_extract_cnn_with_video() {
|
||||
let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
|
||||
let document = Html::parse_document(&html);
|
||||
|
||||
let maybe_image = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR);
|
||||
|
||||
assert_eq!(Some("https://media.cnn.com/api/v1/images/stellar/prod/220426143055-bill-ford-executive-chairman-ford-motor-company.jpg?c=16x9&q=w_850,c_fill"),
|
||||
maybe_image.as_deref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_techcrunch() {
|
||||
let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
|
||||
let document = Html::parse_document(&html);
|
||||
|
||||
let selector = Selector::parse(".article__title").expect("selector should parse");
|
||||
let maybe_image = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR);
|
||||
|
||||
assert_eq!(Some("https://techcrunch.com/wp-content/uploads/2023/02/Screen-Shot-2023-02-02-at-1.46.55-PM.png?w=650"),
|
||||
maybe_image.as_deref());
|
||||
}
|
||||
}
|
||||
+8
-8
@@ -20,28 +20,28 @@ pub fn extract_title(document: &Html, title_selector: &Selector) -> Option<Strin
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common_extractors::extract_title::extract_title;
|
||||
use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR;
|
||||
use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR;
|
||||
use scraper::{Html, Selector};
|
||||
use crate::common_extractors::title_extractor::extract_title;
|
||||
|
||||
#[test]
|
||||
fn text_extract_title_cnn() {
|
||||
let html = include_str!("../../../../../test_data/html_scraping/cnn_article.html");
|
||||
fn test_extract_title_cnn() {
|
||||
let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html");
|
||||
let document = Html::parse_document(&html);
|
||||
|
||||
let selector = Selector::parse(".headline__text").expect("selector should parse");
|
||||
let maybe_title = extract_title(&document, &selector);
|
||||
let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
|
||||
|
||||
assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"),
|
||||
maybe_title.as_deref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_extract_title_techcrunch() {
|
||||
fn test_extract_title_techcrunch() {
|
||||
let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html");
|
||||
let document = Html::parse_document(&html);
|
||||
|
||||
let selector = Selector::parse(".article__title").expect("selector should parse");
|
||||
let maybe_title = extract_title(&document, &selector);
|
||||
let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
|
||||
|
||||
assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"),
|
||||
maybe_title.as_deref());
|
||||
@@ -1 +1,3 @@
|
||||
pub mod title_extractor;
|
||||
pub mod extract_featured_image;
|
||||
pub mod extract_title;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::common_extractors::title_extractor::extract_title;
|
||||
use crate::common_extractors::extract_title::extract_title;
|
||||
use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
|
||||
use crate::payloads::web_scraping_target::WebScrapingTarget;
|
||||
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
|
||||
@@ -20,10 +20,15 @@ static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
});
|
||||
|
||||
/// The title of the article
|
||||
static TITLE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
pub static CNN_TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse(".headline__text").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// The article featured image
|
||||
pub static CNN_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse(".image__lede .image__picture > img").expect("this selector should parse")
|
||||
});
|
||||
|
||||
pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
|
||||
let downloaded_document= reqwest::get(url)
|
||||
.await?
|
||||
@@ -62,7 +67,7 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebS
|
||||
}
|
||||
}
|
||||
|
||||
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
|
||||
let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
|
||||
|
||||
let body_text = paragraphs.join("\n\n");
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::common_extractors::title_extractor::extract_title;
|
||||
use crate::common_extractors::extract_title::extract_title;
|
||||
use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
|
||||
use crate::payloads::web_scraping_target::WebScrapingTarget;
|
||||
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
|
||||
@@ -23,10 +23,15 @@ static PARAGRAPH_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
});
|
||||
|
||||
/// The title of the article
|
||||
static TITLE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
|
||||
pub static TECHCRUNCH_TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse(".article__title").expect("this selector should parse")
|
||||
});
|
||||
|
||||
/// The article featured image
|
||||
pub static TECHCRUNCH_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
|
||||
Selector::parse(".article__featured-image").expect("this selector should parse")
|
||||
});
|
||||
|
||||
pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
|
||||
let downloaded_document= reqwest::get(url)
|
||||
.await?
|
||||
@@ -36,8 +41,6 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
|
||||
let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
|
||||
let document = Html::parse_document(&downloaded_document);
|
||||
|
||||
let mut article_title = None;
|
||||
|
||||
let mut paragraphs = Vec::new();
|
||||
|
||||
if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {
|
||||
@@ -62,7 +65,7 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
|
||||
}
|
||||
}
|
||||
|
||||
let maybe_title = extract_title(&document, &TITLE_SELECTOR);
|
||||
let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
|
||||
|
||||
let body_text = paragraphs.join("\n\n");
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user