small refactor

crates/lib/web_scrapers/src/payloads/web_scraping_result.rs

@@ -1,9 +1,17 @@
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 
-// TODO: Add publish date
-
+// TODO: Rename this to not confuse with `Result<T, E>`.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct WebScrapingResult {
+  /// Surface the original HTML to callers that care about it.
+  pub original_html: String,
+
+  /// Output of *successful* scraping.
+  pub result: ScrapedWebArticle,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ScrapedWebArticle {
   /// Location the article came from
   pub url: String,
 
@@ -28,15 +36,14 @@ pub struct WebScrapingResult {
   /// A featured image (somewhere in the body), if present
   pub maybe_featured_image_url: Option<String>,
 
-  // TODO
+  // TODO: Add publish date
   // pub maybe_publish_date_utc: Option<DateTime<Utc>>
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct OriginalHtmlWithWebScrapingResult {
-  /// Surface the original HTML to callers that care about it.
-  pub original_html: String,
+// TODO: Other types - threaded discussions (HN, Reddit), video essays (CNN, YouTube), etc.
+//  then return these in an enum container.
+pub struct ScrapedThreadedDiscussion {
+}
 
-  /// Output of *successful* scraping.
-  pub result: WebScrapingResult,
-}
+pub struct ScrapedVideo {
+}

crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs

@@ -1,5 +1,6 @@
+use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
-use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
+use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
@@ -29,7 +30,7 @@ pub static CNN_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
   Selector::parse(".image__lede .image__picture > img").expect("this selector should parse")
 });
 
-pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
+pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
   let downloaded_document= reqwest::get(url)
       .await?
       .bytes()
@@ -68,19 +69,20 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebS
   }
 
   let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR);
+  let maybe_heading_image_url = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR);
 
   let body_text = paragraphs.join("\n\n");
 
-  Ok(OriginalHtmlWithWebScrapingResult {
+  Ok(WebScrapingResult {
     original_html: downloaded_document,
-    result: WebScrapingResult {
+    result: ScrapedWebArticle {
       url: url.to_string(),
       web_content_type: WebContentType::CnnArticle,
       maybe_title,
       maybe_author: None, // TODO
       paragraphs,
       body_text,
-      maybe_heading_image_url: None, // TODO
+      maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
     }
   })

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs

@@ -1,5 +1,6 @@
+use crate::common_extractors::extract_featured_image::extract_featured_image;
 use crate::common_extractors::extract_title::extract_title;
-use crate::payloads::web_scraping_result::{OriginalHtmlWithWebScrapingResult, WebScrapingResult};
+use crate::payloads::web_scraping_result::{WebScrapingResult, ScrapedWebArticle};
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
@@ -32,7 +33,7 @@ pub static TECHCRUNCH_FEATURED_IMAGE_SELECTOR: Lazy<Selector> = Lazy::new(|| {
   Selector::parse(".article__featured-image").expect("this selector should parse")
 });
 
-pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlWithWebScrapingResult> {
+pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
   let downloaded_document= reqwest::get(url)
       .await?
       .bytes()
@@ -66,19 +67,20 @@ pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult<OriginalHtmlW
   }
 
   let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR);
+  let maybe_heading_image_url = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR);
 
   let body_text = paragraphs.join("\n\n");
 
-  Ok(OriginalHtmlWithWebScrapingResult {
+  Ok(WebScrapingResult {
     original_html: downloaded_document,
-    result: WebScrapingResult {
+    result: ScrapedWebArticle {
       url: url.to_string(),
       web_content_type: WebContentType::TechCrunchArticle,
       maybe_title,
       maybe_author: None, // TODO
       paragraphs,
       body_text,
-      maybe_heading_image_url: None, // TODO
+      maybe_heading_image_url,
       maybe_featured_image_url: None, // TODO
     }
   })

crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use strum::IntoEnumIterator;
-use web_scrapers::payloads::web_scraping_result::WebScrapingResult;
+use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
 use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
 use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
 use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};

crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs

@@ -8,7 +8,7 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use strum::IntoEnumIterator;
-use web_scrapers::payloads::web_scraping_result::WebScrapingResult;
+use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
 use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
 use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
 use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};