From 5497b7c0aada39cc13bee49c80efe7680ea0b6bb Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Tue, 7 Feb 2023 18:54:23 -0500
Subject: [PATCH] small refactor

---
 .../src/payloads/web_scraping_result.rs       | 27 ++++++++++++-------
 .../src/sites/cnn/cnn_article_scraper.rs      | 12 +++++----
 .../techcrunch/techcrunch_article_scraper.rs  | 12 +++++----
 .../workers/web_content_scraping/main_loop.rs |  2 +-
 .../workers/web_index_ingestion/main_loop.rs  |  2 +-
 5 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs
index 0290151ab5..b8ff57e35e 100644
--- a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs
+++ b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs
@@ -1,9 +1,17 @@
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 
-// TODO: Add publish date
-
+// TODO: Rename this to not confuse with `Result<T, E>`.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct WebScrapingResult {
+  /// Surface the original HTML to callers that care about it.
+  pub original_html: String,
+
+  /// Output of *successful* scraping.
+  pub result: ScrapedWebArticle,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ScrapedWebArticle {
   /// Location the article came from
   pub url: String,
 
@@ -28,15 +36,14 @@ pub struct WebScrapingResult {
   /// A featured image (somewhere in the body), if present
   pub maybe_featured_image_url: Option<String>,
 
-  // TODO
+  // TODO: Add publish date
   // pub maybe_publish_date_utc: Option<DateTime<Utc>>
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct OriginalHtmlWithWebScrapingResult {
-  /// Surface the original HTML to callers that care about it.
-  pub original_html: String,
+// TODO: Other types - threaded discussions (HN, Reddit), video essays (CNN, YouTube), etc.
+//  then return these in an enum container.
+pub struct ScrapedThreadedDiscussion {
+}
 
-  /// Output of *successful* scraping.
-  pub result: WebScrapingResult,
-}
\ No newline at end of file
+pub struct ScrapedVideo {
+}
diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs
index d0e85c660c..e48307e2d5 100644
--- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs
+++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs
@@ -1,5 +1,6 @@
+use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
-use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
+use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
@@ -29,7 +30,7 @@ pub static CNN_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
   Selector::parse(".image__lede .image__picture > img").expect("this selector should parse")
 });
 
-pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
+pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
   let downloaded_document= reqwest::get(url)
       .await?
       .bytes()
@@ -68,19 +69,20 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebS
   }
 
   let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
+  let maybe_heading_image_url = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR);
 
   let body_text = paragraphs.join("\n\n");
 
-  Ok(OriginalHtmlWithWebScrapingResult {
+  Ok(WebScrapingResult {
     original_html: downloaded_document,
-    result: WebScrapingResult {
+    result: ScrapedWebArticle {
       url: url.to_string(),
       web_content_type: WebContentType::CnnArticle,
       maybe_title,
       maybe_author: None, // TODO
       paragraphs,
       body_text,
-      maybe_heading_image_url: None, // TODO
+      maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
     }
   })
diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs
index a3201c5e6c..a3198dcbc1 100644
--- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs
+++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs
@@ -1,5 +1,6 @@
+use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
-use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
+use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
@@ -32,7 +33,7 @@ pub static TECHCRUNCH_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
   Selector::parse(".article__featured-image").expect("this selector should parse")
 });
 
-pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
+pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
   let downloaded_document= reqwest::get(url)
       .await?
       .bytes()
@@ -66,19 +67,20 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
   }
 
   let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
+  let maybe_heading_image_url = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR);
 
   let body_text = paragraphs.join("\n\n");
 
-  Ok(OriginalHtmlWithWebScrapingResult {
+  Ok(WebScrapingResult {
     original_html: downloaded_document,
-    result: WebScrapingResult {
+    result: ScrapedWebArticle {
       url: url.to_string(),
       web_content_type: WebContentType::TechCrunchArticle,
       maybe_title,
       maybe_author: None, // TODO
       paragraphs,
       body_text,
-      maybe_heading_image_url: None, // TODO
+      maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
     }
   })
diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs
index 76a18396c6..029e26a22d 100644
--- a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs
+++ b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs
@@ -11,7 +11,7 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use strum::IntoEnumIterator;
-use web_scrapers::payloads::web_scraping_result::WebScrapingResult;
+use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
 use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
 use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
 use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};
diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs
index bc034fd471..1e8d5785b3 100644
--- a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs
+++ b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use strum::IntoEnumIterator;
-use web_scrapers::payloads::web_scraping_result::WebScrapingResult;
+use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
 use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
 use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
 use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};