From f7b84236238aef43f2cf1554d760b0fa3ff89f0b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 5 Feb 2023 16:29:17 -0500 Subject: [PATCH] techcrunch parsing sort of works --- Cargo.lock | 1 + crates/lib/web_scrapers/Cargo.toml | 1 + crates/lib/web_scrapers/src/payloads/mod.rs | 3 +- .../src/payloads/web_scraping_result.rs | 28 +++++++ .../src/sites/cnn/cnn_article_scraper.rs | 58 +++++++++++++ .../cnn/{cnn_scraper.rs => cnn_indexer.rs} | 0 crates/lib/web_scrapers/src/sites/cnn/mod.rs | 3 +- .../web_scrapers/src/sites/techcrunch/mod.rs | 3 +- .../techcrunch/techcrunch_article_scraper.rs | 82 +++++++++++++++++++ crates/programs/aichatbot-sidecar/src/main.rs | 19 ++++- 10 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 crates/lib/web_scrapers/src/payloads/web_scraping_result.rs create mode 100644 crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs rename crates/lib/web_scrapers/src/sites/cnn/{cnn_scraper.rs => cnn_indexer.rs} (100%) create mode 100644 crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs diff --git a/Cargo.lock b/Cargo.lock index d08c18b5ad..33cb89364e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8142,6 +8142,7 @@ name = "web_scrapers" version = "0.0.1" dependencies = [ "bytes 1.3.0", + "chrono", "enums", "errors", "log", diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index 36faddc5ba..902ec99ce7 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -19,6 +19,7 @@ errors = { path = "../../lib/errors" } # External bytes = "1.3" +chrono = { version = "0.4.19", features = ["serde"] } log = "0.4.14" reqwest = { version = "0.11", features = ["json"] } # HTTP Client rss = "2.0.2" diff --git a/crates/lib/web_scrapers/src/payloads/mod.rs b/crates/lib/web_scrapers/src/payloads/mod.rs index 2c881cbe5f..4c5bdc3ffc 100644 --- a/crates/lib/web_scrapers/src/payloads/mod.rs +++ b/crates/lib/web_scrapers/src/payloads/mod.rs @@ -1 +1,2 @@ -pub mod web_scraping_target; \ No newline at end of file +pub mod web_scraping_target; +pub mod web_scraping_result; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs new file mode 100644 index 0000000000..63ddae5239 --- /dev/null +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs @@ -0,0 +1,28 @@ +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; + +// TODO: Add publish date + +#[derive(Clone, Debug)] +pub struct WebScrapingResult { + + /// What this is and where it came from. + pub web_content_type: WebContentType, + + /// From page title or heading + pub maybe_title: Option, + + /// The author of the article + pub maybe_author: Option, + + /// The paragraphs + pub body_text: String, + + /// A heading image (at top), if present + pub maybe_heading_image_url: Option, + + /// A featured image (somewhere in the body), if present + pub maybe_featured_image_url: Option, + + // TODO + // pub maybe_publish_date_utc: Option> +} diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs new file mode 100644 index 0000000000..e393dbb55f --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs @@ -0,0 +1,58 @@ +use std::ops::Deref; +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use errors::{anyhow, AnyhowResult}; +use log::{error, warn}; +use rss::Channel; +use scraper::{Html, Selector}; +use crate::payloads::web_scraping_result::WebScrapingResult; + +pub async fn cnn_article_scraper(url: &str) -> AnyhowResult { + let downloaded_document= reqwest::get(url) + .await? + .bytes() + .await?; + + // TODO: Save raw html download + let content = String::from_utf8_lossy(downloaded_document.deref()).to_string(); + let document = Html::parse_document(&content); + + //println!("Document: {:?}", document); + + //let selector = Selector::parse(".article__content") + //let selector = Selector::parse("p.paragraph") + let selector = Selector::parse(".article__content") + .map_err(|e| { + error!("Could not parse selector: {:?}", e); + anyhow!("Could not parse selector: {:?}", e) + })?; + + let selector2 = Selector::parse("p.paragraph") + .map_err(|e| { + error!("Could not parse selector: {:?}", e); + anyhow!("Could not parse selector: {:?}", e) + })?; + + + let matches = document.select(&selector); + + for mat in matches.into_iter() { + + let matches2 = mat.select(&selector2); + + for mat2 in matches2.into_iter() { + println!("\n\n{:?}\n\n", mat2); + } + } + + + + Ok(WebScrapingResult { + web_content_type: WebContentType::CnnArticle, + maybe_title: None, + maybe_author: None, + body_text: "".to_string(), + maybe_heading_image_url: None, + maybe_featured_image_url: None, + }) +} diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs similarity index 100% rename from crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs rename to crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs diff --git a/crates/lib/web_scrapers/src/sites/cnn/mod.rs b/crates/lib/web_scrapers/src/sites/cnn/mod.rs index 60ca97661e..5709b91d9e 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/mod.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/mod.rs @@ -1 +1,2 @@ -pub mod cnn_scraper; \ No newline at end of file +pub mod cnn_indexer; +pub mod cnn_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs b/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs index 7ff58348ef..79815ae637 100644 --- a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs +++ b/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs @@ -1 +1,2 @@ -pub mod techcrunch_scraper; \ No newline at end of file +pub mod techcrunch_scraper; +pub mod techcrunch_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs new file mode 100644 index 0000000000..e372cda79c --- /dev/null +++ b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs @@ -0,0 +1,82 @@ +use std::ops::Deref; +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use errors::{anyhow, AnyhowResult}; +use log::{error, warn}; +use rss::Channel; +use scraper::{Html, Selector}; +use crate::payloads::web_scraping_result::WebScrapingResult; + +pub async fn techcrunch_article_scraper(url: &str) -> AnyhowResult { + let downloaded_document= reqwest::get(url) + .await? + .bytes() + .await?; + + // TODO: Save raw html download + let content = String::from_utf8_lossy(downloaded_document.deref()).to_string(); + let document = Html::parse_document(&content); + + //println!("Document: {:?}", document); + + //let selector = Selector::parse(".article__content") + //let selector = Selector::parse("p.paragraph") + let selector = Selector::parse(".article-content") + .map_err(|e| { + error!("Could not parse selector: {:?}", e); + anyhow!("Could not parse selector: {:?}", e) + })?; + + let p_selector = Selector::parse("p") + .map_err(|e| { + error!("Could not parse selector: {:?}", e); + anyhow!("Could not parse selector: {:?}", e) + })?; + + let mut paragraphs = Vec::new(); + + if let Some(article_content_div) = document.select(&selector).next() { + + for paragraph in article_content_div.select(&p_selector).into_iter() { + + //println!("\n\n=================\n\n"); + //println!("\n\n{:?}\n\n", paragraph.inner_html()); + + let mut paragraph_assembly = Vec::new(); + + for text in paragraph.text() { + let stripped = text.trim(); + if !stripped.is_empty() { + paragraph_assembly.push(stripped.to_string()); + } + } + + let paragraph_full_text = paragraph_assembly.join(" ") + .trim() + .to_string(); + + if !paragraph_full_text.is_empty() { + paragraphs.push(paragraph_full_text); + } + + //println!("\n\n=================\n\n"); + } + } + + for paragraph in paragraphs { + println!("\n\nParagraph: {:?}", ¶graph); + + } + + + + + Ok(WebScrapingResult { + web_content_type: WebContentType::CnnArticle, + maybe_title: None, + maybe_author: None, + body_text: "".to_string(), + maybe_heading_image_url: None, + maybe_featured_image_url: None, + }) +} diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index a87f68cd13..0139aae1fd 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -30,7 +30,9 @@ use std::time::Duration; use sqlx::sqlite::SqlitePoolOptions; use tokio::runtime::Runtime; use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target}; -use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test; +use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper; +use web_scrapers::sites::cnn::cnn_indexer::cnn_scraper_test; +use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; use web_scrapers::sites::techcrunch::techcrunch_scraper::techcrunch_scraper_test; use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; @@ -41,9 +43,15 @@ pub async fn main() -> AnyhowResult<()> { .max_connections(5) .connect(&database_url).await?; - //let targets = cnn_scraper_test().await?; + /* + let mut targets = Vec::new(); + //let targets = techcrunch_scraper_test().await?; - let targets = theguardian_scraper_test().await?; + //let targets = theguardian_scraper_test().await?; + + targets.extend(cnn_scraper_test().await?); + targets.extend(techcrunch_scraper_test().await?); + targets.extend(theguardian_scraper_test().await?); for target in targets.iter() { //println!("\n\nTarget: {:?}", target); @@ -57,6 +65,11 @@ pub async fn main() -> AnyhowResult<()> { sqlite_pool: &pool, }).await; } + */ + + //let _r = cnn_article_scraper("https://www.cnn.com/2023/02/02/tech/first-generation-iphone-auction/index.html").await?; + let _r = techcrunch_article_scraper("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/").await?; + Ok(()) }