From 201481f969bd01adf73df8f6678c3aa2f6e294f2 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Sun, 5 Feb 2023 05:24:54 -0500
Subject: [PATCH] handle TechCrunch venture feed

---
 Cargo.lock                                           |  1 +
 crates/lib/web_scrapers/Cargo.toml                   |  1 +
 crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs |  1 +
 .../src/sites/techcrunch/techcrunch_scraper.rs       | 12 +++++++++++-
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 42c90a4fd3..d08c18b5ad 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8141,6 +8141,7 @@ dependencies = [
 name = "web_scrapers"
 version = "0.0.1"
 dependencies = [
+ "bytes 1.3.0",
  "enums",
  "errors",
  "log",
diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml
index a68599b4e3..36faddc5ba 100644
--- a/crates/lib/web_scrapers/Cargo.toml
+++ b/crates/lib/web_scrapers/Cargo.toml
@@ -18,6 +18,7 @@ enums = { path = "../../_types/enums" }
 errors = { path = "../../lib/errors" }
 
 # External
+bytes = "1.3"
 log = "0.4.14"
 reqwest = { version = "0.11", features = ["json"] } # HTTP Client
 rss = "2.0.2"
diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs
index 5e49b31a59..b779115d86 100644
--- a/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs
+++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs
@@ -4,6 +4,7 @@ use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;
 
+// NB: Contains nearly 70 items
 const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss";
 
 const RSS_WORLD : &'static str = "http://rss.cnn.com/rss/cnn_world.rss";
diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
index ad28cb27e8..268adfa450 100644
--- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
+++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
@@ -1,17 +1,27 @@
+use std::ops::Deref;
+use bytes::Bytes;
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;
 
+// NB: Contains 20 items
 const RSS_FEED : &'static str = "https://techcrunch.com/feed/";
 
+const VENTURE_FEED : &'static str = "https://techcrunch.com/category/venture/feed/";
+
 pub async fn techcrunch_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
-  let content = reqwest::get(RSS_FEED)
+  let content = reqwest::get(VENTURE_FEED)
       .await?
       .bytes()
       .await?;
 
+  // NB: TechCrunch's venture feed contains an "&bull;" HTML entity that makes the RSS client choke
+  let content = String::from_utf8_lossy(content.deref());
+  let content = content.replace("&bull;", "");
+  let content = Bytes::from(content);
+
   let channel = Channel::read_from(&content[..])?;
 
   let mut targets = Vec::with_capacity(channel.items.len());