diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml
index a0230c4261..bfa695f576 100644
--- a/.idea/dictionaries/bt.xml
+++ b/.idea/dictionaries/bt.xml
@@ -15,6 +15,7 @@
softvc
symphonia
tacotron
+ techcrunch
transcoded
transcodings
upserting
diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs
index 67c7185e25..22565e9423 100644
--- a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs
+++ b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs
@@ -1,5 +1,7 @@
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+// TODO: Include the article publish date(!)
+
/// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets.
/// These will be inserted into the database for later downstream processing.
#[derive(Clone, Debug)]
diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
index e69de29bb2..ad28cb27e8 100644
--- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
+++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs
@@ -0,0 +1,52 @@
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use errors::AnyhowResult;
+use log::warn;
+use rss::Channel;
+
+const RSS_FEED : &'static str = "https://techcrunch.com/feed/";
+
+pub async fn techcrunch_scraper_test() -> AnyhowResult> {
+ let content = reqwest::get(RSS_FEED)
+ .await?
+ .bytes()
+ .await?;
+
+ let channel = Channel::read_from(&content[..])?;
+
+ let mut targets = Vec::with_capacity(channel.items.len());
+
+ for item in channel.items {
+ let canonical_url = match item.link {
+ Some(url) => url.clone(),
+ None => {
+ warn!("Skipping item due to not having a URL");
+ continue;
+ }
+ };
+
+ // NB: TechCrunch doesn't have images, but there are sometimes some embedded in article payloads
+ //let maybe_image_url = item.extensions.get("media")
+ // .map(|media| media.get("group"))
+ // .flatten()
+ // .map(|group| group.get(0))
+ // .flatten()
+ // .map(|extension| extension.children.get("content"))
+ // .flatten()
+ // .map(|extensions| extensions.get(0)) // NB: First image is biggest
+ // .flatten()
+ // .map(|extension| extension.attrs.get("url"))
+ // .flatten()
+ // .map(|url| url.to_string());
+
+ targets.push(WebScrapingTarget {
+ canonical_url,
+ web_content_type: WebContentType::TechCrunchArticle,
+ maybe_title: item.title.clone(),
+ maybe_full_image_url: None,
+ maybe_thumbnail_image_url: None,
+ });
+ }
+
+ Ok(targets)
+}
diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs
index 9e50352658..fbf10298de 100644
--- a/crates/programs/aichatbot-sidecar/src/main.rs
+++ b/crates/programs/aichatbot-sidecar/src/main.rs
@@ -31,6 +31,7 @@ use sqlx::sqlite::SqlitePoolOptions;
use tokio::runtime::Runtime;
use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test;
+use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test;
#[tokio::main]
pub async fn main() -> AnyhowResult<()> {
@@ -39,13 +40,12 @@ pub async fn main() -> AnyhowResult<()> {
.max_connections(5)
.connect(&database_url).await?;
- let targets = cnn_scraper_test().await?;
+ //let targets = cnn_scraper_test().await?;
+ let targets = techcrunch_scraper_test().await?;
for target in targets.iter() {
- //println!("Target: {:?}", target)
- }
+ println!("\n\nTarget: {:?}", target);
- for target in targets.iter() {
let _r = insert_web_scraping_target(Args {
canonical_url: &target.canonical_url,
web_content_type: target.web_content_type,