CNN candidate articles via RSS + Sqlite inserts

Cargo.lock

@@ -767,6 +767,8 @@ dependencies = [
  "serde",
  "serde_derive",
  "serde_json",
+ "sqlite_queries",
+ "sqlx",
  "tokio 1.25.0",
  "web_scrapers",
 ]
@@ -2653,6 +2655,18 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "flume"
+version = "0.10.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "pin-project 1.0.7",
+ "spin 0.9.4",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -3833,7 +3847,7 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 dependencies = [
- "spin",
+ "spin 0.5.2",
 ]
 
 [[package]]
@@ -3876,6 +3890,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "limitation"
 version = "0.1.1"
@@ -5534,7 +5559,7 @@ dependencies = [
  "cc",
  "libc",
  "once_cell",
- "spin",
+ "spin 0.5.2",
  "untrusted",
  "web-sys",
  "winapi 0.3.9",
@@ -5549,7 +5574,7 @@ dependencies = [
  "cc",
  "libc",
  "once_cell",
- "spin",
+ "spin 0.5.2",
  "untrusted",
  "web-sys",
  "winapi 0.3.9",
@@ -6191,6 +6216,15 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "spin"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"
+dependencies = [
+ "lock_api 0.4.6",
+]
+
 [[package]]
 name = "spki"
 version = "0.5.4"
@@ -6212,6 +6246,21 @@ dependencies = [
  "unicode_categories",
 ]
 
+[[package]]
+name = "sqlite_queries"
+version = "0.0.1"
+dependencies = [
+ "chrono",
+ "dotenv",
+ "enums",
+ "errors",
+ "log",
+ "once_cell",
+ "regex",
+ "sqlx",
+ "tokens",
+]
+
 [[package]]
 name = "sqlx"
 version = "0.6.2"
@@ -6240,8 +6289,10 @@ dependencies = [
  "dotenvy",
  "either",
  "event-listener",
+ "flume",
  "futures-channel",
  "futures-core",
+ "futures-executor",
  "futures-intrusive",
  "futures-util",
  "generic-array",
@@ -6250,6 +6301,7 @@ dependencies = [
  "indexmap",
  "itoa 1.0.3",
  "libc",
+ "libsqlite3-sys",
  "log",
  "memchr",
  "num-bigint 0.4.3",
@@ -8089,7 +8141,9 @@ dependencies = [
 name = "web_scrapers"
 version = "0.0.1"
 dependencies = [
+ "enums",
  "errors",
+ "log",
  "reqwest",
  "rss",
  "scraper",

Cargo.toml

@@ -28,6 +28,7 @@ members = [
   "crates/lib/media",
   "crates/lib/memory_caching",
   "crates/lib/redis_common",
+  "crates/lib/sqlite_queries",
   "crates/lib/subprocess_common",
   "crates/lib/tts_common",
   "crates/lib/twitch_common",

crates/_types/enums/Cargo.toml

@@ -23,7 +23,7 @@ toml = "0.5.8"
 # "offline" supports typesafe builds without a live database
 # "rustls" doesn't appear to work with IP addresses, per:
 # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
-sqlx = { version = "0.6.2", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
+sqlx = { version = "0.6.2", features = [ "mysql", "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] }
 sqlx-core = { version = "0.6.2" }
 
 [dev-dependencies]

crates/_types/enums/src/by_table/mod.rs

@@ -2,3 +2,4 @@ pub mod media_uploads;
 pub mod model_categories;
 pub mod trending_model_analytics;
 pub mod user_ratings;
+pub mod web_scraping_targets; // NB: SqLite

crates/_types/enums/src/by_table/web_scraping_targets/mod.rs

@@ -0,0 +1 @@
+pub mod web_content_type;

crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs

@@ -0,0 +1,105 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `web_content_type`.
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub enum WebContentType {
+  #[serde(rename = "cnn_article")]
+  CnnArticle,
+
+  #[serde(rename = "hacker_news_thread")]
+  HackerNewsThread,
+
+  #[serde(rename = "reddit_thread")]
+  RedditThread,
+
+  #[serde(rename = "slashdot_article")]
+  SlashdotArticle,
+  
+  #[serde(rename = "substack_post")]
+  SubstackPost,
+
+  #[serde(rename = "techcrunch_article")]
+  TechCrunchArticle,
+}
+
+// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests!
+impl_enum_display_and_debug_using_to_str!(WebContentType);
+//impl_enum_sqlx_coders!(WebContentType); // TODO(bt, 2023-02-05): This doesn't exist for sqlite
+
+/// NB: Legacy API for older code.
+impl WebContentType {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::CnnArticle => "cnn_article",
+      Self::HackerNewsThread => "hacker_news_thread",
+      Self::RedditThread => "reddit_thread",
+      Self::SlashdotArticle => "slashdot_article",
+      Self::SubstackPost => "substack_post",
+      Self::TechCrunchArticle => "techcrunch_article",
+    }
+  }
+
+  pub fn from_str(value: &str) -> Result<Self, String> {
+    match value {
+      "cnn_article" => Ok(Self::CnnArticle),
+      "hacker_news_thread" => Ok(Self::HackerNewsThread),
+      "reddit_thread" => Ok(Self::RedditThread),
+      "slashdot_article" => Ok(Self::SlashdotArticle),
+      "substack_post" => Ok(Self::SubstackPost),
+      "techcrunch_article" => Ok(Self::TechCrunchArticle),
+      _ => Err(format!("invalid value: {:?}", value)),
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::by_table::web_scraping_targets::web_content_type::WebContentType;
+  use crate::test_helpers::assert_serialization;
+
+  mod serde {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(WebContentType::CnnArticle, "cnn_article");
+      assert_serialization(WebContentType::HackerNewsThread, "hacker_news_thread");
+      assert_serialization(WebContentType::RedditThread, "reddit_thread");
+      assert_serialization(WebContentType::SlashdotArticle, "slashdot_article");
+      assert_serialization(WebContentType::SubstackPost, "substack_post");
+      assert_serialization(WebContentType::TechCrunchArticle, "techcrunch_article");
+    }
+  }
+
+  mod impl_methods {
+    use super::*;
+
+    #[test]
+    fn test_to_str() {
+      assert_eq!(WebContentType::CnnArticle.to_str(), "cnn_article");
+      assert_eq!(WebContentType::HackerNewsThread.to_str(), "hacker_news_thread");
+      assert_eq!(WebContentType::RedditThread.to_str(), "reddit_thread");
+      assert_eq!(WebContentType::SlashdotArticle.to_str(), "slashdot_article");
+      assert_eq!(WebContentType::SubstackPost.to_str(), "substack_post");
+      assert_eq!(WebContentType::TechCrunchArticle.to_str(), "techcrunch_article");
+    }
+
+    #[test]
+    fn test_from_str() {
+      assert_eq!(WebContentType::from_str("cnn_article").unwrap(), WebContentType::CnnArticle);
+      assert_eq!(WebContentType::from_str("hacker_news_thread").unwrap(), WebContentType::HackerNewsThread);
+      assert_eq!(WebContentType::from_str("reddit_thread").unwrap(), WebContentType::RedditThread);
+      assert_eq!(WebContentType::from_str("slashdot_article").unwrap(), WebContentType::SlashdotArticle);
+      assert_eq!(WebContentType::from_str("substack_post").unwrap(), WebContentType::SubstackPost);
+      assert_eq!(WebContentType::from_str("techcrunch_article").unwrap(), WebContentType::TechCrunchArticle);
+      assert!(WebContentType::from_str("foo").is_err());
+    }
+  }
+}

crates/lib/sqlite_queries/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "sqlite_queries"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "sqlite_queries"
+path = "src/lib.rs"
+
+[dependencies]
+
+# Internal
+enums = { path = "../../_types/enums" }
+errors = { path = "../../lib/errors" }
+tokens = { path = "../../_types/tokens" }
+
+# External
+chrono = { version = "0.4.19", features = ["serde"] }
+dotenv = "0.15.0"
+log = "0.4.14"
+
+# Needed to safety check some queries we can't use escaping for
+once_cell = "1.9.0"
+regex = "1.5.4"
+
+# SQlite Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+#  ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^
+sqlx = { version = "0.6.2", features = [ "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] }
+
+[dev-dependencies]
+# None yet

crates/lib/sqlite_queries/src/lib.rs

@@ -0,0 +1,22 @@
+//! sqlite_queries
+//!
+//! The purpose of this library is to contain all of our SQlite-related queries.
+//! Keeping them in a single place abstracts away the data layer and makes it easy
+//! to statically generate bindings and write tests.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+//// Okay to toggle
+//#![forbid(unreachable_patterns)]
+//#![forbid(unused_imports)]
+//#![forbid(unused_mut)]
+//#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod queries;

crates/lib/sqlite_queries/src/queries/by_table/mod.rs

@@ -0,0 +1 @@
+pub mod web_scraping_targets;

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs

@@ -0,0 +1,56 @@
+use errors::{anyhow, AnyhowResult};
+use sqlx::SqlitePool;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+
+pub struct Args <'a> {
+  pub canonical_url: &'a str,
+
+  pub web_content_type: WebContentType,
+
+  pub maybe_title: Option<&'a str>,
+  pub maybe_article_full_image_url: Option<&'a str>,
+  pub maybe_article_thumbnail_image_url: Option<&'a str>,
+
+  pub sqlite_pool: &'a SqlitePool,
+}
+
+pub async fn insert_web_scraping_target(args: Args<'_>) -> AnyhowResult<()> {
+
+  let web_content_type = args.web_content_type.to_str().to_string();
+
+  let query = sqlx::query!(
+        r#"
+INSERT INTO web_scraping_targets(
+  canonical_url,
+  web_content_type,
+  maybe_title,
+  maybe_article_full_image_url,
+  maybe_article_thumbnail_image_url
+)
+VALUES (
+  ?,
+  ?,
+  ?,
+  ?,
+  ?
+)
+        "#,
+        args.canonical_url,
+        web_content_type,
+        args.maybe_title,
+        args.maybe_article_full_image_url,
+        args.maybe_article_thumbnail_image_url,
+    );
+
+  let query_result = query.execute(args.sqlite_pool)
+      .await;
+
+  let _record_id = match query_result {
+    Ok(res) => res.last_insert_rowid(),
+    Err(err) => {
+      return Err(anyhow!("error inserting: {:?}", err));
+    }
+  };
+
+  Ok(())
+}

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs

@@ -0,0 +1 @@
+pub mod insert_web_scraping_target;

crates/lib/sqlite_queries/src/queries/mod.rs

@@ -0,0 +1 @@
+pub mod by_table;

crates/lib/web_scrapers/Cargo.toml

@@ -9,15 +9,16 @@ authors = [
 publish = false
 
 [lib]
-# NB: Public "errors" crate is name-squatted, so we're fine to use this internally.
 name = "web_scrapers"
 path = "src/lib.rs"
 
 [dependencies]
 # Internal
+enums = { path = "../../_types/enums" }
 errors = { path = "../../lib/errors" }
 
 # External
+log = "0.4.14"
 reqwest = { version = "0.11", features = ["json"] } # HTTP Client
 rss = "2.0.2"
 scraper = "0.14.0"

crates/lib/web_scrapers/src/lib.rs

@@ -1,7 +1,7 @@
-//! errors
+//! web_scrapers
 //!
-//! The purpose of this library is to pin to a single 'anyhow' and also develop
-//! common error utilities.
+//! The purpose of this library is to scrape content from websites.
+//! This should power our AI Chatbot and other functionalities.
 //!
 
 // Never allow these
@@ -18,4 +18,5 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]
 
+pub mod payloads;
 pub mod sites;

crates/lib/web_scrapers/src/payloads/mod.rs

@@ -0,0 +1 @@
+pub mod web_scraping_target;

crates/lib/web_scrapers/src/payloads/web_scraping_target.rs

@@ -0,0 +1,12 @@
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+
+/// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets.
+/// These will be inserted into the database for later downstream processing.
+#[derive(Clone, Debug)]
+pub struct WebScrapingTarget {
+  pub canonical_url: String,
+  pub web_content_type: WebContentType,
+  pub maybe_title: Option<String>,
+  pub maybe_full_image_url: Option<String>,
+  pub maybe_thumbnail_image_url: Option<String>,
+}

crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs

@@ -1,5 +1,8 @@
-use rss::Channel;
+use crate::payloads::web_scraping_target::WebScrapingTarget;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::AnyhowResult;
+use log::warn;
+use rss::Channel;
 
 const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss";
 
@@ -9,7 +12,7 @@ const RSS_US : &'static str = "http://rss.cnn.com/rss/cnn_us.rss";
 
 const RSS_TECH : &'static str = "http://rss.cnn.com/rss/cnn_tech.rss";
 
-pub async fn cnn_scraper_test() -> AnyhowResult<()> {
+pub async fn cnn_scraper_test() -> AnyhowResult<Vec<WebScrapingTarget>> {
   let content = reqwest::get(RSS_TOP_STORIES)
       .await?
       .bytes()
@@ -17,8 +20,16 @@ pub async fn cnn_scraper_test() -> AnyhowResult<()> {
 
   let channel = Channel::read_from(&content[..])?;
 
+  let mut targets = Vec::with_capacity(channel.items.len());
+
   for item in channel.items {
-    println!("\n\nItem: {:?}", item);
+    let canonical_url = match item.link {
+      Some(url) => url.clone(),
+      None => {
+        warn!("Skipping item due to not having a URL");
+        continue;
+      }
+    };
 
     let maybe_image_url = item.extensions.get("media")
         .map(|media| media.get("group"))
@@ -30,12 +41,17 @@ pub async fn cnn_scraper_test() -> AnyhowResult<()> {
         .map(|extensions| extensions.get(0)) // NB: First image is biggest
         .flatten()
         .map(|extension| extension.attrs.get("url"))
-        .flatten();
+        .flatten()
+        .map(|url| url.to_string());
 
-    if let Some(image_url) = maybe_image_url {
-      println!("\n-{:?} ", image_url);
-    }
+    targets.push(WebScrapingTarget {
+      canonical_url,
+      web_content_type: WebContentType::CnnArticle,
+      maybe_title: item.title.clone(),
+      maybe_full_image_url: maybe_image_url,
+      maybe_thumbnail_image_url: None,
+    });
   }
 
-  Ok(())
+  Ok(targets)
 }

crates/programs/aichatbot-sidecar/Cargo.toml

@@ -19,6 +19,7 @@ actix_helpers = { path = "../../lib/actix_helpers" }
 errors = { path = "../../lib/errors" }
 files = { path = "../../lib/files" }
 http_server_common = { path = "../../lib/http_server_common" }
+sqlite_queries = { path = "../../lib/sqlite_queries" }
 web_scrapers = { path = "../../lib/web_scrapers" }
 
 # External
@@ -28,6 +29,13 @@ log = "0.4.14"
 dotenv = "0.15.0"
 path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet
 
+# SQlite Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+#  ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^
+sqlx = { version = "0.6.2", features = [ "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] }
+
 # Async Runtime
 tokio = { version = "1.6.0", features = ["macros"] }
 

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064307_web_scraping_targets/up.sql

@@ -7,13 +7,16 @@
 CREATE TABLE web_scraping_targets (
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
 
-  -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
-  web_content_type TEXT NOT NULL,
-
   -- Canonical URLs help us avoid aliases of the same content.
   -- We'll need to figure this out for each site.
   canonical_url TEXT NOT NULL UNIQUE,
 
+  -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
+  web_content_type TEXT NOT NULL,
+
+  -- Optional title of the target, which might help downstream.
+  maybe_title TEXT,
+
   -- A full image will only be set at this stage if the RSS feed or HTML specifies one.
   -- This can be useful downstream during the actual scraping.
   maybe_article_full_image_url TEXT,
@@ -24,11 +27,13 @@ CREATE TABLE web_scraping_targets (
 
   -- Scraping status: "new", "failed", "permanently_failed", "success"
   scraping_status TEXT DEFAULT "new" NOT NULL,
-  scrape_attempts INT NOT NULL,
+  scrape_attempts INT DEFAULT 0 NOT NULL,
 
   created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
   updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
-  fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
+  fetched_at DATETIME DEFAULT NULL,
+
+  version INT DEFAULT 0 NOT NULL
 );
 
 CREATE INDEX index_scraping_status ON web_scraping_targets(scraping_status);

crates/programs/aichatbot-sidecar/src/main.rs

@@ -27,14 +27,34 @@ use log::info;
 use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
+use sqlx::sqlite::SqlitePoolOptions;
 use tokio::runtime::Runtime;
+use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
 use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test;
 
 #[tokio::main]
 pub async fn main() -> AnyhowResult<()> {
+  let database_url = easyenv::get_env_string_required("DATABASE_URL")?;
+  let pool = SqlitePoolOptions::new()
+      .max_connections(5)
+      .connect(&database_url).await?;
 
-  cnn_scraper_test().await;
+  let targets = cnn_scraper_test().await?;
 
+  for target in targets.iter() {
+    //println!("Target: {:?}", target)
+  }
+
+  for target in targets.iter() {
+    let _r = insert_web_scraping_target(Args {
+      canonical_url: &target.canonical_url,
+      web_content_type: target.web_content_type,
+      maybe_title: target.maybe_title.as_deref(),
+      maybe_article_full_image_url: target.maybe_full_image_url.as_deref(),
+      maybe_article_thumbnail_image_url: target.maybe_thumbnail_image_url.as_deref(),
+      sqlite_pool: &pool,
+    }).await;
+  }
 
   Ok(())
 }