techcrunch parsing sort of works

Cargo.lock

@@ -8142,6 +8142,7 @@ name = "web_scrapers"
 version = "0.0.1"
 dependencies = [
  "bytes 1.3.0",
+ "chrono",
  "enums",
  "errors",
  "log",

crates/lib/web_scrapers/Cargo.toml

@@ -19,6 +19,7 @@ errors = { path = "../../lib/errors" }
 
 # External
 bytes = "1.3"
+chrono = { version = "0.4.19", features = ["serde"] }
 log = "0.4.14"
 reqwest = { version = "0.11", features = ["json"] } # HTTP Client
 rss = "2.0.2"

crates/lib/web_scrapers/src/payloads/mod.rs

@@ -1 +1,2 @@
-pub mod web_scraping_target;
+pub mod web_scraping_target;
+pub mod web_scraping_result;

crates/lib/web_scrapers/src/payloads/web_scraping_result.rs

@@ -0,0 +1,28 @@
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+
+// TODO: Add publish date
+
+#[derive(Clone, Debug)]
+pub struct WebScrapingResult {
+
+  /// What this is and where it came from.
+  pub web_content_type: WebContentType,
+
+  /// From page title or heading
+  pub maybe_title: Option<String>,
+
+  /// The author of the  article
+  pub maybe_author: Option<String>,
+
+  /// The paragraphs
+  pub body_text: String,
+
+  /// A heading image (at top), if present
+  pub maybe_heading_image_url: Option<String>,
+
+  /// A featured image (somewhere in the body), if present
+  pub maybe_featured_image_url: Option<String>,
+
+  // TODO
+  // pub maybe_publish_date_utc: Option<DateTime<Utc>>
+}

crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs

@@ -0,0 +1,58 @@
+use std::ops::Deref;
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use errors::{anyhow, AnyhowResult};
+use log::{error, warn};
+use rss::Channel;
+use scraper::{Html, Selector};
+use crate::payloads::web_scraping_result::WebScrapingResult;
+
+pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
+  let downloaded_document= reqwest::get(url)
+      .await?
+      .bytes()
+      .await?;
+
+  // TODO: Save raw html download
+  let content = String::from_utf8_lossy(downloaded_document.deref()).to_string();
+  let document = Html::parse_document(&content);
+
+  //println!("Document: {:?}", document);
+
+  //let selector = Selector::parse(".article__content")
+  //let selector = Selector::parse("p.paragraph")
+  let selector = Selector::parse(".article__content")
+      .map_err(|e| {
+        error!("Could not parse selector: {:?}", e);
+        anyhow!("Could not parse selector: {:?}", e)
+      })?;
+
+  let selector2 = Selector::parse("p.paragraph")
+      .map_err(|e| {
+        error!("Could not parse selector: {:?}", e);
+        anyhow!("Could not parse selector: {:?}", e)
+      })?;
+
+
+  let matches = document.select(&selector);
+
+  for mat in matches.into_iter() {
+
+    let matches2 = mat.select(&selector2);
+
+    for mat2 in matches2.into_iter() {
+      println!("\n\n{:?}\n\n", mat2);
+    }
+  }
+
+
+
+  Ok(WebScrapingResult {
+    web_content_type: WebContentType::CnnArticle,
+    maybe_title: None,
+    maybe_author: None,
+    body_text: "".to_string(),
+    maybe_heading_image_url: None,
+    maybe_featured_image_url: None,
+  })
+}

crates/lib/web_scrapers/src/sites/cnn/mod.rs

@@ -1 +1,2 @@
-pub mod cnn_scraper;
+pub mod cnn_indexer;
+pub mod cnn_article_scraper;

crates/lib/web_scrapers/src/sites/techcrunch/mod.rs

@@ -1 +1,2 @@
-pub mod techcrunch_scraper;
+pub mod techcrunch_scraper;
+pub mod techcrunch_article_scraper;

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs

@@ -0,0 +1,82 @@
+use std::ops::Deref;
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use errors::{anyhow, AnyhowResult};
+use log::{error, warn};
+use rss::Channel;
+use scraper::{Html, Selector};
+use crate::payloads::web_scraping_result::WebScrapingResult;
+
+pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
+  let downloaded_document= reqwest::get(url)
+      .await?
+      .bytes()
+      .await?;
+
+  // TODO: Save raw html download
+  let content = String::from_utf8_lossy(downloaded_document.deref()).to_string();
+  let document = Html::parse_document(&content);
+
+  //println!("Document: {:?}", document);
+
+  //let selector = Selector::parse(".article__content")
+  //let selector = Selector::parse("p.paragraph")
+  let selector = Selector::parse(".article-content")
+      .map_err(|e| {
+        error!("Could not parse selector: {:?}", e);
+        anyhow!("Could not parse selector: {:?}", e)
+      })?;
+
+  let p_selector = Selector::parse("p")
+      .map_err(|e| {
+        error!("Could not parse selector: {:?}", e);
+        anyhow!("Could not parse selector: {:?}", e)
+      })?;
+
+  let mut paragraphs = Vec::new();
+
+  if let Some(article_content_div) = document.select(&selector).next() {
+
+    for paragraph in article_content_div.select(&p_selector).into_iter() {
+
+      //println!("\n\n=================\n\n");
+      //println!("\n\n{:?}\n\n", paragraph.inner_html());
+
+      let mut paragraph_assembly = Vec::new();
+
+      for text in paragraph.text() {
+        let stripped = text.trim();
+        if !stripped.is_empty() {
+          paragraph_assembly.push(stripped.to_string());
+        }
+      }
+
+      let paragraph_full_text = paragraph_assembly.join(" ")
+          .trim()
+          .to_string();
+
+      if !paragraph_full_text.is_empty() {
+        paragraphs.push(paragraph_full_text);
+      }
+
+      //println!("\n\n=================\n\n");
+    }
+  }
+
+  for paragraph in paragraphs {
+    println!("\n\nParagraph: {:?}", &paragraph);
+
+  }
+
+
+
+
+  Ok(WebScrapingResult {
+    web_content_type: WebContentType::CnnArticle,
+    maybe_title: None,
+    maybe_author: None,
+    body_text: "".to_string(),
+    maybe_heading_image_url: None,
+    maybe_featured_image_url: None,
+  })
+}

crates/programs/aichatbot-sidecar/src/main.rs

@@ -30,7 +30,9 @@ use std::time::Duration;
 use sqlx::sqlite::SqlitePoolOptions;
 use tokio::runtime::Runtime;
 use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
-use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test;
+use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper;
+use web_scrapers::sites::cnn::cnn_indexer::cnn_scraper_test;
+use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper;
 use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test;
 use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test;
 
@@ -41,9 +43,15 @@ pub async fn main() -> AnyhowResult<()> {
       .max_connections(5)
       .connect(&database_url).await?;
 
-  //let targets = cnn_scraper_test().await?;
+  /*
+  let mut targets = Vec::new();
+
   //let targets = techcrunch_scraper_test().await?;
-  let targets = theguardian_scraper_test().await?;
+  //let targets = theguardian_scraper_test().await?;
+
+  targets.extend(cnn_scraper_test().await?);
+  targets.extend(techcrunch_scraper_test().await?);
+  targets.extend(theguardian_scraper_test().await?);
 
   for target in targets.iter() {
     //println!("\n\nTarget: {:?}", target);
@@ -57,6 +65,11 @@ pub async fn main() -> AnyhowResult<()> {
       sqlite_pool: &pool,
     }).await;
   }
+  */
+
+  //let _r = cnn_article_scraper("https://www.cnn.com/2023/02/02/tech/first-generation-iphone-auction/index.html").await?;
+  let _r = techcrunch_article_scraper("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/").await?;
+
 
   Ok(())
 }