diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs b/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs index 43a979853d..a9fe506e78 100644 --- a/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs +++ b/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs @@ -15,34 +15,34 @@ pub fn extract_featured_image(document: &Html, image_selector: &Selector) -> Opt None } -#[cfg(test)] -mod tests { - use scraper::{Html, Selector}; - - use crate::common_extractors::extract_featured_image::extract_featured_image; - use crate::sites::cnn::cnn_article_scraper::CNN_FEATURED_IMAGE_SELECTOR; - use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_FEATURED_IMAGE_SELECTOR; - - #[test] - fn test_extract_cnn_with_video() { - let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); - let document = Html::parse_document(&html); - - let maybe_image = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR); - - assert_eq!(Some("https://media.cnn.com/api/v1/images/stellar/prod/220426143055-bill-ford-executive-chairman-ford-motor-company.jpg?c=16x9&q=w_850,c_fill"), - maybe_image.as_deref()); - } - - #[test] - fn test_extract_techcrunch() { - let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); - let document = Html::parse_document(&html); - - let selector = Selector::parse(".article__title").expect("selector should parse"); - let maybe_image = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR); - - assert_eq!(Some("https://techcrunch.com/wp-content/uploads/2023/02/Screen-Shot-2023-02-02-at-1.46.55-PM.png?w=650"), - maybe_image.as_deref()); - } -} +//#[cfg(test)] +//mod tests { +// use scraper::{Html, Selector}; +// +// use crate::common_extractors::extract_featured_image::extract_featured_image; +// use crate::sites::cnn::cnn_article_scraper::CNN_FEATURED_IMAGE_SELECTOR; +// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_FEATURED_IMAGE_SELECTOR; +// +// #[test] +// fn test_extract_cnn_with_video() { +// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); +// let document = Html::parse_document(&html); +// +// let maybe_image = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR); +// +// assert_eq!(Some("https://media.cnn.com/api/v1/images/stellar/prod/220426143055-bill-ford-executive-chairman-ford-motor-company.jpg?c=16x9&q=w_850,c_fill"), +// maybe_image.as_deref()); +// } +// +// #[test] +// fn test_extract_techcrunch() { +// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); +// let document = Html::parse_document(&html); +// +// let selector = Selector::parse(".article__title").expect("selector should parse"); +// let maybe_image = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR); +// +// assert_eq!(Some("https://techcrunch.com/wp-content/uploads/2023/02/Screen-Shot-2023-02-02-at-1.46.55-PM.png?w=650"), +// maybe_image.as_deref()); +// } +//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs index d9d29f2598..8ad2ce4520 100644 --- a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs +++ b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs @@ -18,33 +18,33 @@ pub fn extract_text(document: &Html, text_selector: &Selector) -> Option Some(pieces.join(" ").trim().to_string()) } -#[cfg(test)] -mod tests { - use scraper::Html; - - use crate::common_extractors::extract_text::extract_text; - use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; - use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; - - #[test] - fn test_extract_text_cnn() { - let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); - let document = Html::parse_document(&html); - - let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); - - assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), - maybe_text.as_deref()); - } - - #[test] - fn test_extract_text_techcrunch() { - let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); - let document = Html::parse_document(&html); - - let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); - - assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), - maybe_text.as_deref()); - } -} +//#[cfg(test)] +//mod tests { +// use scraper::Html; +// +// use crate::common_extractors::extract_text::extract_text; +// use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; +// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; +// +// #[test] +// fn test_extract_text_cnn() { +// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); +// let document = Html::parse_document(&html); +// +// let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); +// +// assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), +// maybe_text.as_deref()); +// } +// +// #[test] +// fn test_extract_text_techcrunch() { +// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); +// let document = Html::parse_document(&html); +// +// let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); +// +// assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), +// maybe_text.as_deref()); +// } +//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_title.rs b/crates/lib/web_scrapers/src/common_extractors/extract_title.rs index 1eceaafae4..a1ef30c50a 100644 --- a/crates/lib/web_scrapers/src/common_extractors/extract_title.rs +++ b/crates/lib/web_scrapers/src/common_extractors/extract_title.rs @@ -18,33 +18,33 @@ pub fn extract_title(document: &Html, title_selector: &Selector) -> Option