diff --git a/Cargo.lock b/Cargo.lock index 30cb7ceaee..42c90a4fd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,6 +767,8 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sqlite_queries", + "sqlx", "tokio 1.25.0", "web_scrapers", ] @@ -2653,6 +2655,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project 1.0.7", + "spin 0.9.4", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3833,7 +3847,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -3876,6 +3890,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" +[[package]] +name = "libsqlite3-sys" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "limitation" version = "0.1.1" @@ -5534,7 +5559,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi 0.3.9", @@ -5549,7 +5574,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi 0.3.9", @@ -6191,6 +6216,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api 0.4.6", +] + [[package]] name = "spki" version = "0.5.4" @@ -6212,6 +6246,21 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "sqlite_queries" +version = "0.0.1" +dependencies = [ + "chrono", + "dotenv", + "enums", + "errors", + "log", + "once_cell", + "regex", + "sqlx", + "tokens", +] + [[package]] name = "sqlx" version = "0.6.2" @@ -6240,8 +6289,10 @@ dependencies = [ "dotenvy", "either", "event-listener", + "flume", "futures-channel", "futures-core", + "futures-executor", "futures-intrusive", "futures-util", "generic-array", @@ -6250,6 +6301,7 @@ dependencies = [ "indexmap", "itoa 1.0.3", "libc", + "libsqlite3-sys", "log", "memchr", "num-bigint 0.4.3", @@ -8089,7 +8141,9 @@ dependencies = [ name = "web_scrapers" version = "0.0.1" dependencies = [ + "enums", "errors", + "log", "reqwest", "rss", "scraper", diff --git a/Cargo.toml b/Cargo.toml index 5825880394..901a241a2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "crates/lib/media", "crates/lib/memory_caching", "crates/lib/redis_common", + "crates/lib/sqlite_queries", "crates/lib/subprocess_common", "crates/lib/tts_common", "crates/lib/twitch_common", diff --git a/crates/_types/enums/Cargo.toml b/crates/_types/enums/Cargo.toml index 69c3ff2335..9efd7c3148 100644 --- a/crates/_types/enums/Cargo.toml +++ b/crates/_types/enums/Cargo.toml @@ -23,7 +23,7 @@ toml = "0.5.8" # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -sqlx = { version = "0.6.2", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.6.2", features = [ "mysql", "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] } sqlx-core = { version = "0.6.2" } [dev-dependencies] diff --git a/crates/_types/enums/src/by_table/mod.rs b/crates/_types/enums/src/by_table/mod.rs index c099576241..a071d1bf0c 100644 --- a/crates/_types/enums/src/by_table/mod.rs +++ b/crates/_types/enums/src/by_table/mod.rs @@ -2,3 +2,4 @@ pub mod media_uploads; pub mod model_categories; pub mod trending_model_analytics; pub mod user_ratings; +pub mod web_scraping_targets; // NB: SqLite diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs new file mode 100644 index 0000000000..033c4741b9 --- /dev/null +++ b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs @@ -0,0 +1 @@ +pub mod web_content_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs b/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs new file mode 100644 index 0000000000..b8ecb71bd1 --- /dev/null +++ b/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs @@ -0,0 +1,105 @@ +use serde::Deserialize; +use serde::Serialize; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `web_content_type`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum WebContentType { + #[serde(rename = "cnn_article")] + CnnArticle, + + #[serde(rename = "hacker_news_thread")] + HackerNewsThread, + + #[serde(rename = "reddit_thread")] + RedditThread, + + #[serde(rename = "slashdot_article")] + SlashdotArticle, + + #[serde(rename = "substack_post")] + SubstackPost, + + #[serde(rename = "techcrunch_article")] + TechCrunchArticle, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(WebContentType); +//impl_enum_sqlx_coders!(WebContentType); // TODO(bt, 2023-02-05): This doesn't exist for sqlite + +/// NB: Legacy API for older code. +impl WebContentType { + pub fn to_str(&self) -> &'static str { + match self { + Self::CnnArticle => "cnn_article", + Self::HackerNewsThread => "hacker_news_thread", + Self::RedditThread => "reddit_thread", + Self::SlashdotArticle => "slashdot_article", + Self::SubstackPost => "substack_post", + Self::TechCrunchArticle => "techcrunch_article", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "cnn_article" => Ok(Self::CnnArticle), + "hacker_news_thread" => Ok(Self::HackerNewsThread), + "reddit_thread" => Ok(Self::RedditThread), + "slashdot_article" => Ok(Self::SlashdotArticle), + "substack_post" => Ok(Self::SubstackPost), + "techcrunch_article" => Ok(Self::TechCrunchArticle), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::web_scraping_targets::web_content_type::WebContentType; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(WebContentType::CnnArticle, "cnn_article"); + assert_serialization(WebContentType::HackerNewsThread, "hacker_news_thread"); + assert_serialization(WebContentType::RedditThread, "reddit_thread"); + assert_serialization(WebContentType::SlashdotArticle, "slashdot_article"); + assert_serialization(WebContentType::SubstackPost, "substack_post"); + assert_serialization(WebContentType::TechCrunchArticle, "techcrunch_article"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(WebContentType::CnnArticle.to_str(), "cnn_article"); + assert_eq!(WebContentType::HackerNewsThread.to_str(), "hacker_news_thread"); + assert_eq!(WebContentType::RedditThread.to_str(), "reddit_thread"); + assert_eq!(WebContentType::SlashdotArticle.to_str(), "slashdot_article"); + assert_eq!(WebContentType::SubstackPost.to_str(), "substack_post"); + assert_eq!(WebContentType::TechCrunchArticle.to_str(), "techcrunch_article"); + } + + #[test] + fn test_from_str() { + assert_eq!(WebContentType::from_str("cnn_article").unwrap(), WebContentType::CnnArticle); + assert_eq!(WebContentType::from_str("hacker_news_thread").unwrap(), WebContentType::HackerNewsThread); + assert_eq!(WebContentType::from_str("reddit_thread").unwrap(), WebContentType::RedditThread); + assert_eq!(WebContentType::from_str("slashdot_article").unwrap(), WebContentType::SlashdotArticle); + assert_eq!(WebContentType::from_str("substack_post").unwrap(), WebContentType::SubstackPost); + assert_eq!(WebContentType::from_str("techcrunch_article").unwrap(), WebContentType::TechCrunchArticle); + assert!(WebContentType::from_str("foo").is_err()); + } + } +} diff --git a/crates/lib/sqlite_queries/Cargo.toml b/crates/lib/sqlite_queries/Cargo.toml new file mode 100644 index 0000000000..9a0328c574 --- /dev/null +++ b/crates/lib/sqlite_queries/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "sqlite_queries" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "sqlite_queries" +path = "src/lib.rs" + +[dependencies] + +# Internal +enums = { path = "../../_types/enums" } +errors = { path = "../../lib/errors" } +tokens = { path = "../../_types/tokens" } + +# External +chrono = { version = "0.4.19", features = ["serde"] } +dotenv = "0.15.0" +log = "0.4.14" + +# Needed to safety check some queries we can't use escaping for +once_cell = "1.9.0" +regex = "1.5.4" + +# SQlite Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ +sqlx = { version = "0.6.2", features = [ "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] } + +[dev-dependencies] +# None yet diff --git a/crates/lib/sqlite_queries/src/lib.rs b/crates/lib/sqlite_queries/src/lib.rs new file mode 100644 index 0000000000..207d3ba287 --- /dev/null +++ b/crates/lib/sqlite_queries/src/lib.rs @@ -0,0 +1,22 @@ +//! sqlite_queries +//! +//! The purpose of this library is to contain all of our SQlite-related queries. +//! Keeping them in a single place abstracts away the data layer and makes it easy +//! to statically generate bindings and write tests. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +//// Okay to toggle +//#![forbid(unreachable_patterns)] +//#![forbid(unused_imports)] +//#![forbid(unused_mut)] +//#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod queries; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/mod.rs new file mode 100644 index 0000000000..30f4526b09 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/mod.rs @@ -0,0 +1 @@ +pub mod web_scraping_targets; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs new file mode 100644 index 0000000000..fbe1fe7ad9 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs @@ -0,0 +1,56 @@ +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; + +pub struct Args <'a> { + pub canonical_url: &'a str, + + pub web_content_type: WebContentType, + + pub maybe_title: Option<&'a str>, + pub maybe_article_full_image_url: Option<&'a str>, + pub maybe_article_thumbnail_image_url: Option<&'a str>, + + pub sqlite_pool: &'a SqlitePool, +} + +pub async fn insert_web_scraping_target(args: Args<'_>) -> AnyhowResult<()> { + + let web_content_type = args.web_content_type.to_str().to_string(); + + let query = sqlx::query!( + r#" +INSERT INTO web_scraping_targets( + canonical_url, + web_content_type, + maybe_title, + maybe_article_full_image_url, + maybe_article_thumbnail_image_url +) +VALUES ( + ?, + ?, + ?, + ?, + ? +) + "#, + args.canonical_url, + web_content_type, + args.maybe_title, + args.maybe_article_full_image_url, + args.maybe_article_thumbnail_image_url, + ); + + let query_result = query.execute(args.sqlite_pool) + .await; + + let _record_id = match query_result { + Ok(res) => res.last_insert_rowid(), + Err(err) => { + return Err(anyhow!("error inserting: {:?}", err)); + } + }; + + Ok(()) +} \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs new file mode 100644 index 0000000000..41156e6123 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs @@ -0,0 +1 @@ +pub mod insert_web_scraping_target; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/mod.rs b/crates/lib/sqlite_queries/src/queries/mod.rs new file mode 100644 index 0000000000..dc279ee573 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/mod.rs @@ -0,0 +1 @@ +pub mod by_table; \ No newline at end of file diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml index 75fc85e7ce..a68599b4e3 100644 --- a/crates/lib/web_scrapers/Cargo.toml +++ b/crates/lib/web_scrapers/Cargo.toml @@ -9,15 +9,16 @@ authors = [ publish = false [lib] -# NB: Public "errors" crate is name-squatted, so we're fine to use this internally. name = "web_scrapers" path = "src/lib.rs" [dependencies] # Internal +enums = { path = "../../_types/enums" } errors = { path = "../../lib/errors" } # External +log = "0.4.14" reqwest = { version = "0.11", features = ["json"] } # HTTP Client rss = "2.0.2" scraper = "0.14.0" diff --git a/crates/lib/web_scrapers/src/lib.rs b/crates/lib/web_scrapers/src/lib.rs index 227aa28080..b1a837422e 100644 --- a/crates/lib/web_scrapers/src/lib.rs +++ b/crates/lib/web_scrapers/src/lib.rs @@ -1,7 +1,7 @@ -//! errors +//! web_scrapers //! -//! The purpose of this library is to pin to a single 'anyhow' and also develop -//! common error utilities. +//! The purpose of this library is to scrape content from websites. +//! This should power our AI Chatbot and other functionalities. //! // Never allow these @@ -18,4 +18,5 @@ #![allow(dead_code)] #![allow(non_snake_case)] +pub mod payloads; pub mod sites; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/payloads/mod.rs b/crates/lib/web_scrapers/src/payloads/mod.rs new file mode 100644 index 0000000000..2c881cbe5f --- /dev/null +++ b/crates/lib/web_scrapers/src/payloads/mod.rs @@ -0,0 +1 @@ +pub mod web_scraping_target; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs new file mode 100644 index 0000000000..67c7185e25 --- /dev/null +++ b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs @@ -0,0 +1,12 @@ +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; + +/// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets. +/// These will be inserted into the database for later downstream processing. +#[derive(Clone, Debug)] +pub struct WebScrapingTarget { + pub canonical_url: String, + pub web_content_type: WebContentType, + pub maybe_title: Option, + pub maybe_full_image_url: Option, + pub maybe_thumbnail_image_url: Option, +} diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs index 03986e2a68..5e49b31a59 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_scraper.rs @@ -1,5 +1,8 @@ -use rss::Channel; +use crate::payloads::web_scraping_target::WebScrapingTarget; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use errors::AnyhowResult; +use log::warn; +use rss::Channel; const RSS_TOP_STORIES : &'static str = "http://rss.cnn.com/rss/cnn_topstories.rss"; @@ -9,7 +12,7 @@ const RSS_US : &'static str = "http://rss.cnn.com/rss/cnn_us.rss"; const RSS_TECH : &'static str = "http://rss.cnn.com/rss/cnn_tech.rss"; -pub async fn cnn_scraper_test() -> AnyhowResult<()> { +pub async fn cnn_scraper_test() -> AnyhowResult> { let content = reqwest::get(RSS_TOP_STORIES) .await? .bytes() @@ -17,8 +20,16 @@ pub async fn cnn_scraper_test() -> AnyhowResult<()> { let channel = Channel::read_from(&content[..])?; + let mut targets = Vec::with_capacity(channel.items.len()); + for item in channel.items { - println!("\n\nItem: {:?}", item); + let canonical_url = match item.link { + Some(url) => url.clone(), + None => { + warn!("Skipping item due to not having a URL"); + continue; + } + }; let maybe_image_url = item.extensions.get("media") .map(|media| media.get("group")) @@ -30,12 +41,17 @@ pub async fn cnn_scraper_test() -> AnyhowResult<()> { .map(|extensions| extensions.get(0)) // NB: First image is biggest .flatten() .map(|extension| extension.attrs.get("url")) - .flatten(); + .flatten() + .map(|url| url.to_string()); - if let Some(image_url) = maybe_image_url { - println!("\n-{:?} ", image_url); - } + targets.push(WebScrapingTarget { + canonical_url, + web_content_type: WebContentType::CnnArticle, + maybe_title: item.title.clone(), + maybe_full_image_url: maybe_image_url, + maybe_thumbnail_image_url: None, + }); } - Ok(()) + Ok(targets) } diff --git a/crates/programs/aichatbot-sidecar/Cargo.toml b/crates/programs/aichatbot-sidecar/Cargo.toml index cdc16b7788..0fabc30393 100644 --- a/crates/programs/aichatbot-sidecar/Cargo.toml +++ b/crates/programs/aichatbot-sidecar/Cargo.toml @@ -19,6 +19,7 @@ actix_helpers = { path = "../../lib/actix_helpers" } errors = { path = "../../lib/errors" } files = { path = "../../lib/files" } http_server_common = { path = "../../lib/http_server_common" } +sqlite_queries = { path = "../../lib/sqlite_queries" } web_scrapers = { path = "../../lib/web_scrapers" } # External @@ -28,6 +29,13 @@ log = "0.4.14" dotenv = "0.15.0" path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet +# SQlite Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +# ^^^ Granted, this applies to *MySQL* and probably not SQLite for our use case here ^^^ +sqlx = { version = "0.6.2", features = [ "sqlite", "runtime-actix-native-tls", "offline", "chrono" ] } + # Async Runtime tokio = { version = "1.6.0", features = ["macros"] } diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064307_web_scraping_targets/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064307_web_scraping_targets/up.sql index b49b720246..af8f15ff2c 100644 --- a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064307_web_scraping_targets/up.sql +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064307_web_scraping_targets/up.sql @@ -7,13 +7,16 @@ CREATE TABLE web_scraping_targets ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc. - web_content_type TEXT NOT NULL, - -- Canonical URLs help us avoid aliases of the same content. -- We'll need to figure this out for each site. canonical_url TEXT NOT NULL UNIQUE, + -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc. + web_content_type TEXT NOT NULL, + + -- Optional title of the target, which might help downstream. + maybe_title TEXT, + -- A full image will only be set at this stage if the RSS feed or HTML specifies one. -- This can be useful downstream during the actual scraping. maybe_article_full_image_url TEXT, @@ -24,11 +27,13 @@ CREATE TABLE web_scraping_targets ( -- Scraping status: "new", "failed", "permanently_failed", "success" scraping_status TEXT DEFAULT "new" NOT NULL, - scrape_attempts INT NOT NULL, + scrape_attempts INT DEFAULT 0 NOT NULL, created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + fetched_at DATETIME DEFAULT NULL, + + version INT DEFAULT 0 NOT NULL ); CREATE INDEX index_scraping_status ON web_scraping_targets(scraping_status); diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 1305dd3f54..9e50352658 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -27,14 +27,34 @@ use log::info; use std::sync::Arc; use std::thread; use std::time::Duration; +use sqlx::sqlite::SqlitePoolOptions; use tokio::runtime::Runtime; +use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target}; use web_scrapers::sites::cnn::cnn_scraper::cnn_scraper_test; #[tokio::main] pub async fn main() -> AnyhowResult<()> { + let database_url = easyenv::get_env_string_required("DATABASE_URL")?; + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect(&database_url).await?; - cnn_scraper_test().await; + let targets = cnn_scraper_test().await?; + for target in targets.iter() { + //println!("Target: {:?}", target) + } + + for target in targets.iter() { + let _r = insert_web_scraping_target(Args { + canonical_url: &target.canonical_url, + web_content_type: target.web_content_type, + maybe_title: target.maybe_title.as_deref(), + maybe_article_full_image_url: target.maybe_full_image_url.as_deref(), + maybe_article_thumbnail_image_url: target.maybe_thumbnail_image_url.as_deref(), + sqlite_pool: &pool, + }).await; + } Ok(()) }