diff --git a/Cargo.lock b/Cargo.lock index 0b867fa883..40ebf1c36f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5695,6 +5695,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scrape_test" +version = "0.0.1" +dependencies = [ + "clap", + "errors", + "tokio 1.25.0", + "web_scrapers", +] + [[package]] name = "scraper" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index 59fce83710..d840a1390f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ # CLI tools for operations, etc. "crates/cli/database_backup", "crates/cli/database_cleanup", + "crates/cli/scrape_test", # Shared libraries "crates/lib/actix_helpers", diff --git a/crates/cli/scrape_test/Cargo.toml b/crates/cli/scrape_test/Cargo.toml new file mode 100644 index 0000000000..81439ef747 --- /dev/null +++ b/crates/cli/scrape_test/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "scrape_test" +publish = false +edition = "2021" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[[bin]] +name = "scrape_test" +path = "src/main.rs" + +[dependencies] + +# Internal +errors = { path = "../../lib/errors" } +web_scrapers = { path = "../../lib/web_scrapers" } + +# External +clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +tokio = { version = "1.6.0", features = ["macros"] } diff --git a/crates/cli/scrape_test/src/main.rs b/crates/cli/scrape_test/src/main.rs new file mode 100644 index 0000000000..5169643e87 --- /dev/null +++ b/crates/cli/scrape_test/src/main.rs @@ -0,0 +1,37 @@ +use clap::{App, Arg}; +use errors::{anyhow, AnyhowResult}; +use web_scrapers::scrape_supported_webpage::scrape_supported_webpage; + +#[tokio::main] +pub async fn main() -> AnyhowResult<()> { + let matches = App::new("scrape_test") + .arg(Arg::with_name("url") + .short("u") + .long("url") + .value_name("URL") + .help("URL of the page to scrape") + .takes_value(true) + .required(true)) + .arg(Arg::with_name("print_html") + .long("html") + .help("Print the HTML scraped") + .takes_value(false) + .required(false)) + .get_matches(); + + let url = matches.value_of("url") + .map(|val| val.trim().to_string()) + .ok_or(anyhow!("no url supplied"))?; + + let print_html = matches.is_present("print_html"); + + let scrape_result = scrape_supported_webpage(&url).await?; + + if print_html { + println!("{}", scrape_result.original_html) + } else { + println!("{:#?}", scrape_result.result) + } + + Ok(()) +} diff --git a/crates/lib/web_scrapers/src/lib.rs b/crates/lib/web_scrapers/src/lib.rs index baff45f2b2..4f0b384074 100644 --- a/crates/lib/web_scrapers/src/lib.rs +++ b/crates/lib/web_scrapers/src/lib.rs @@ -23,4 +23,5 @@ pub mod common_extractors; pub mod payloads; -pub mod sites; \ No newline at end of file +pub mod scrape_supported_webpage; +pub mod sites; diff --git a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs new file mode 100644 index 0000000000..18676993ca --- /dev/null +++ b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs @@ -0,0 +1,40 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; +use reqwest::Url; +use errors::AnyhowResult; +use crate::payloads::web_scraping_result::WebScrapingResult; +use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError; +use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper; + +#[derive(Debug, Clone)] +pub enum ScrapeUtilityError { + UrlParseError, + NoHostnameError, + UnknownUrl, + // TODO: Make inner reason more strongly typed. + ScrapingError { reason: String }, +} + +impl Display for ScrapeUtilityError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl Error for ScrapeUtilityError {} + +pub async fn scrape_supported_webpage(url: &str) -> Result { + let parsed_url = Url::parse(url) + .map_err(|_err| UrlParseError)?; + + let hostname = parsed_url.host() + .ok_or(ScrapeUtilityError::NoHostnameError)? + .to_string(); + + let maybe_result = match hostname.as_ref() { + "cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await, + _ => return Err(ScrapeUtilityError::UnknownUrl), + }; + + maybe_result.map_err(|err| ScrapeUtilityError::ScrapingError { reason: err.to_string() }) +} diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 2e25366d46..2fcfb10ec7 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -27,7 +27,9 @@ use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop; use crate::workers::news_stories::phase_1::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop; use crate::workers::news_stories::phase_2::news_story_image_generation::main_loop::news_story_image_generation_main_loop; +use crate::workers::news_stories::phase_2::news_story_llm_category_summary::main_loop::news_story_llm_category_summary_main_loop; use crate::workers::news_stories::phase_2::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop; +use crate::workers::news_stories::phase_2::news_story_llm_title_summary::main_loop::news_story_llm_title_summary_main_loop; use crate::workers::news_stories::phase_3::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop; use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop; use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; @@ -49,8 +51,6 @@ use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_arti use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop; use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop; -use crate::workers::news_stories::phase_2::news_story_llm_category_summary::main_loop::news_story_llm_category_summary_main_loop; -use crate::workers::news_stories::phase_2::news_story_llm_title_summary::main_loop::news_story_llm_title_summary_main_loop; //#[tokio::main] //pub async fn main() -> AnyhowResult<()> {