techcrunch scraping of RSS feed

This commit is contained in:
Brandon Thomas committed 2023-02-05 05:12:42 -05:00
1 parent eefe48f8ca
commit 665cd83739
4 files changed
+59 -4

No files matched your search

+1
View File
@@ -15,6 +15,7 @@
<w>softvc</w>
<w>symphonia</w>
<w>tacotron</w>
<w>techcrunch</w>
<w>transcoded</w>
<w>transcodings</w>
<w>upserting</w>
@@ -1,5 +1,7 @@
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
// TODO: Include the article publish date(!)
/// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets.
/// These will be inserted into the database for later downstream processing.
#[derive(Clone, Debug)]
@@ -0,0 +1,52 @@
use crate::payloads::web_scraping_target::WebScrapingTarget;
use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
use errors::AnyhowResult;
use log::warn;
use rss::Channel;
const RSS_FEED : &'static str = "https://techcrunch.com/feed/";
pub async fn techcrunch_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
let content = reqwest::get(RSS_FEED)
.await?
.bytes()
.await?;
let channel = Channel::read_from(&content[..])?;
let mut targets = Vec::with_capacity(channel.items.len());
for item in channel.items {
let canonical_url = match item.link {
Some(url) => url.clone(),
None => {
warn!("Skipping item due to not having a URL");
continue;
}
};
// NB: TechCrunch doesn't have images, but there are sometimes some embedded in article payloads
//let maybe_image_url = item.extensions.get("media")
// .map(|media| media.get("group"))
// .flatten()
// .map(|group| group.get(0))
// .flatten()
// .map(|extension| extension.children.get("content"))
// .flatten()
// .map(|extensions| extensions.get(0)) // NB: First image is biggest
// .flatten()
// .map(|extension| extension.attrs.get("url"))
// .flatten()
// .map(|url| url.to_string());
targets.push(WebScrapingTarget {
canonical_url,
web_content_type: WebContentType::TechCrunchArticle,
maybe_title: item.title.clone(),
maybe_full_image_url: None,
maybe_thumbnail_image_url: None,
});
}
Ok(targets)
}
@@ -31,6 +31,7 @@ use sqlx::sqlite::SqlitePoolOptions;
use tokio::runtime::Runtime;
use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test;
use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test;
#[tokio::main]
pub async fn main() -> AnyhowResult<()> {
@@ -39,13 +40,12 @@ pub async fn main() -> AnyhowResult<()> {
.max_connections(5)
.connect(&database_url).await?;
let targets = cnn_scraper_test().await?;
//let targets = cnn_scraper_test().await?;
let targets = techcrunch_scraper_test().await?;
for target in targets.iter() {
//println!("Target: {:?}", target)
}
println!("\n\nTarget: {:?}", target);
for target in targets.iter() {
let _r = insert_web_scraping_target(Args {
canonical_url: &target.canonical_url,
web_content_type: target.web_content_type,