handle TechCrunch venture feed

Cargo.lock

@@ -8141,6 +8141,7 @@ dependencies = [
 name = "web_scrapers"
 version = "0.0.1"
 dependencies = [
+ "bytes 1.3.0",
  "enums",
  "errors",
  "log",

crates/lib/web_scrapers/Cargo.toml

@@ -18,6 +18,7 @@ enums = { path = "../../_types/enums" }
 errors = { path = "../../lib/errors" }
 
 # External
+bytes = "1.3"
 log = "0.4.14"
 reqwest = { version = "0.11", features = ["json"] } # HTTP Client
 rss = "2.0.2"

crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs

@@ -4,6 +4,7 @@ use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;
 
+// NB: Contains nearly 70 items
 const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss";
 
 const RSS_WORLD : &'static str = "http://rss.cnn.com/rss/cnn_world.rss";

crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs

@@ -1,17 +1,27 @@
+use std::ops::Deref;
+use bytes::Bytes;
 use crate::payloads::web_scraping_target::WebScrapingTarget;
 use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::AnyhowResult;
 use log::warn;
 use rss::Channel;
 
+// NB: Contains 20 items
 const RSS_FEED : &'static str = "https://techcrunch.com/feed/";
 
+const VENTURE_FEED : &'static str = "https://techcrunch.com/category/venture/feed/";
+
 pub async fn techcrunch_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
-  let content = reqwest::get(RSS_FEED)
+  let content = reqwest::get(VENTURE_FEED)
       .await?
       .bytes()
       .await?;
 
+  // NB: TechCrunch's venture feed contains an "&bull;" HTML entity that makes the RSS client choke
+  let content = String::from_utf8_lossy(content.deref());
+  let content = content.replace("&bull;", "");
+  let content = Bytes::from(content);
+
   let channel = Channel::read_from(&content[..])?;
 
   let mut targets = Vec::with_capacity(channel.items.len());