diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index a0230c4261..bfa695f576 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -15,6 +15,7 @@ softvc symphonia tacotron + techcrunch transcoded transcodings upserting diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs index 67c7185e25..22565e9423 100644 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs @@ -1,5 +1,7 @@ use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +// TODO: Include the article publish date(!) + /// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets. /// These will be inserted into the database for later downstream processing. #[derive(Clone, Debug)] diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs index e69de29bb2..ad28cb27e8 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_scraper.rs @@ -0,0 +1,52 @@ +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use errors::AnyhowResult; +use log::warn; +use rss::Channel; + +const RSS_FEED : &'static str = "https://techcrunch.com/feed/"; + +pub async fn techcrunch_scraper_test() -> AnyhowResult> { + let content = reqwest::get(RSS_FEED) + .await? + .bytes() + .await?; + + let channel = Channel::read_from(&content[..])?; + + let mut targets = Vec::with_capacity(channel.items.len()); + + for item in channel.items { + let canonical_url = match item.link { + Some(url) => url.clone(), + None => { + warn!("Skipping item due to not having a URL"); + continue; + } + }; + + // NB: TechCrunch doesn't have images, but there are sometimes some embedded in article payloads + //let maybe_image_url = item.extensions.get("media") + // .map(|media| media.get("group")) + // .flatten() + // .map(|group| group.get(0)) + // .flatten() + // .map(|extension| extension.children.get("content")) + // .flatten() + // .map(|extensions| extensions.get(0)) // NB: First image is biggest + // .flatten() + // .map(|extension| extension.attrs.get("url")) + // .flatten() + // .map(|url| url.to_string()); + + targets.push(WebScrapingTarget { + canonical_url, + web_content_type: WebContentType::TechCrunchArticle, + maybe_title: item.title.clone(), + maybe_full_image_url: None, + maybe_thumbnail_image_url: None, + }); + } + + Ok(targets) +} diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 9e50352658..fbf10298de 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -31,6 +31,7 @@ use sqlx::sqlite::SqlitePoolOptions; use tokio::runtime::Runtime; use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target}; use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test; +use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test; #[tokio::main] pub async fn main() -> AnyhowResult<()> { @@ -39,13 +40,12 @@ pub async fn main() -> AnyhowResult<()> { .max_connections(5) .connect(&database_url).await?; - let targets = cnn_scraper_test().await?; + //let targets = cnn_scraper_test().await?; + let targets = techcrunch_scraper_test().await?; for target in targets.iter() { - //println!("Target: {:?}", target) - } + println!("\n\nTarget: {:?}", target); - for target in targets.iter() { let _r = insert_web_scraping_target(Args { canonical_url: &target.canonical_url, web_content_type: target.web_content_type,