CLI utility to test parsing

Cargo.lock

@@ -5695,6 +5695,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scrape_test"
+version = "0.0.1"
+dependencies = [
+ "clap",
+ "errors",
+ "tokio 1.25.0",
+ "web_scrapers",
+]
+
 [[package]]
 name = "scraper"
 version = "0.14.0"

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   # CLI tools for operations, etc.
   "crates/cli/database_backup",
   "crates/cli/database_cleanup",
+  "crates/cli/scrape_test",
 
   # Shared libraries
   "crates/lib/actix_helpers",

crates/cli/scrape_test/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "scrape_test"
+publish = false
+edition = "2021"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+
+[[bin]]
+name = "scrape_test"
+path = "src/main.rs"
+
+[dependencies]
+
+# Internal
+errors = { path = "../../lib/errors" }
+web_scrapers = { path = "../../lib/web_scrapers" }
+
+# External
+clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+tokio = { version = "1.6.0", features = ["macros"] }

crates/cli/scrape_test/src/main.rs

@@ -0,0 +1,37 @@
+use clap::{App, Arg};
+use errors::{anyhow, AnyhowResult};
+use web_scrapers::scrape_supported_webpage::scrape_supported_webpage;
+
+#[tokio::main]
+pub async fn main() -> AnyhowResult<()> {
+  let matches = App::new("scrape_test")
+      .arg(Arg::with_name("url")
+          .short("u")
+          .long("url")
+          .value_name("URL")
+          .help("URL of the page to scrape")
+          .takes_value(true)
+          .required(true))
+      .arg(Arg::with_name("print_html")
+          .long("html")
+          .help("Print the HTML scraped")
+          .takes_value(false)
+          .required(false))
+      .get_matches();
+
+  let url = matches.value_of("url")
+      .map(|val| val.trim().to_string())
+      .ok_or(anyhow!("no url supplied"))?;
+
+  let print_html = matches.is_present("print_html");
+
+  let scrape_result = scrape_supported_webpage(&url).await?;
+
+  if print_html {
+    println!("{}", scrape_result.original_html)
+  } else {
+    println!("{:#?}", scrape_result.result)
+  }
+
+  Ok(())
+}

crates/lib/web_scrapers/src/lib.rs

@@ -23,4 +23,5 @@
 
 pub mod common_extractors;
 pub mod payloads;
-pub mod sites;
+pub mod scrape_supported_webpage;
+pub mod sites;

crates/lib/web_scrapers/src/scrape_supported_webpage.rs

@@ -0,0 +1,40 @@
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+use reqwest::Url;
+use errors::AnyhowResult;
+use crate::payloads::web_scraping_result::WebScrapingResult;
+use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError;
+use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper;
+
+#[derive(Debug, Clone)]
+pub enum ScrapeUtilityError {
+  UrlParseError,
+  NoHostnameError,
+  UnknownUrl,
+  // TODO: Make inner reason more strongly typed.
+  ScrapingError { reason: String },
+}
+
+impl Display for ScrapeUtilityError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+    write!(f, "{:?}", self)
+  }
+}
+
+impl Error for ScrapeUtilityError {}
+
+pub async fn scrape_supported_webpage(url: &str) -> Result<WebScrapingResult, ScrapeUtilityError> {
+  let parsed_url = Url::parse(url)
+      .map_err(|_err| UrlParseError)?;
+
+  let hostname = parsed_url.host()
+      .ok_or(ScrapeUtilityError::NoHostnameError)?
+      .to_string();
+
+  let maybe_result = match hostname.as_ref() {
+    "cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await,
+    _ => return Err(ScrapeUtilityError::UnknownUrl),
+  };
+
+  maybe_result.map_err(|err| ScrapeUtilityError::ScrapingError { reason: err.to_string() })
+}

crates/programs/aichatbot-sidecar/src/main.rs

@@ -27,7 +27,9 @@ use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create
 use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop;
 use crate::workers::news_stories::phase_1::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop;
 use crate::workers::news_stories::phase_2::news_story_image_generation::main_loop::news_story_image_generation_main_loop;
+use crate::workers::news_stories::phase_2::news_story_llm_category_summary::main_loop::news_story_llm_category_summary_main_loop;
 use crate::workers::news_stories::phase_2::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop;
+use crate::workers::news_stories::phase_2::news_story_llm_title_summary::main_loop::news_story_llm_title_summary_main_loop;
 use crate::workers::news_stories::phase_3::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop;
 use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop;
 use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save;
@@ -49,8 +51,6 @@ use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_arti
 use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test;
 use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
 use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;
-use crate::workers::news_stories::phase_2::news_story_llm_category_summary::main_loop::news_story_llm_category_summary_main_loop;
-use crate::workers::news_stories::phase_2::news_story_llm_title_summary::main_loop::news_story_llm_title_summary_main_loop;
 
 //#[tokio::main]
 //pub async fn main() -> AnyhowResult<()> {