From 6abbbaae012e62f778beeae2ec5d5cb4161bc141 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 6 Feb 2023 23:46:22 -0500 Subject: [PATCH] list query, enum macros for web_scraping_targets (sqlite) --- .../src/by_table/web_scraping_targets/mod.rs | 1 + .../web_scraping_targets/scraping_status.rs | 89 +++++++++++++++++++ .../web_scraping_targets/web_content_type.rs | 2 +- crates/_types/enums/src/lib.rs | 3 + .../src/macros2/impl_sqlite_enum_coders.rs | 48 ++++++++++ crates/_types/enums/src/macros2/mod.rs | 2 + .../insert_web_scraping_target.rs | 4 +- .../list_web_scraping_targets.rs | 74 +++++++++++++++ .../by_table/web_scraping_targets/mod.rs | 3 +- 9 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 crates/_types/enums/src/by_table/web_scraping_targets/scraping_status.rs create mode 100644 crates/_types/enums/src/macros2/impl_sqlite_enum_coders.rs create mode 100644 crates/_types/enums/src/macros2/mod.rs create mode 100644 crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs index 033c4741b..d66a3a21b 100644 --- a/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs +++ b/crates/_types/enums/src/by_table/web_scraping_targets/mod.rs @@ -1 +1,2 @@ +pub mod scraping_status; pub mod web_content_type; \ No newline at end of file diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/scraping_status.rs b/crates/_types/enums/src/by_table/web_scraping_targets/scraping_status.rs new file mode 100644 index 000000000..5f47bb9e3 --- /dev/null +++ b/crates/_types/enums/src/by_table/web_scraping_targets/scraping_status.rs @@ -0,0 +1,89 @@ +use serde::Deserialize; +use serde::Serialize; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum ScrapingStatus { + #[serde(rename = "new")] + New, + + #[serde(rename = "failed")] + Failed, + + #[serde(rename = "permanently_failed")] + PermanentlyFailed, + + #[serde(rename = "success")] + Success, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(ScrapingStatus); +impl_enum_sqlx_coders!(ScrapingStatus); + +/// NB: Legacy API for older code. +impl ScrapingStatus { + pub fn to_str(&self) -> &'static str { + match self { + Self::New => "new", + Self::Failed => "failed", + Self::PermanentlyFailed => "permanently_failed", + Self::Success => "success", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "new" => Ok(Self::New), + "failed" => Ok(Self::Failed), + "permanently_failed" => Ok(Self::PermanentlyFailed), + "success" => Ok(Self::Success), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::web_scraping_targets::scraping_status::ScrapingStatus; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(ScrapingStatus::New, "new"); + assert_serialization(ScrapingStatus::Failed, "failed"); + assert_serialization(ScrapingStatus::PermanentlyFailed, "permanently_failed"); + assert_serialization(ScrapingStatus::Success, "success"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(ScrapingStatus::New.to_str(), "new"); + assert_eq!(ScrapingStatus::Failed.to_str(), "failed"); + assert_eq!(ScrapingStatus::PermanentlyFailed.to_str(), "permanently_failed"); + assert_eq!(ScrapingStatus::Success.to_str(), "success"); + } + + #[test] + fn test_from_str() { + assert_eq!(ScrapingStatus::from_str("new").unwrap(), ScrapingStatus::New); + assert_eq!(ScrapingStatus::from_str("failed").unwrap(), ScrapingStatus::Failed); + assert_eq!(ScrapingStatus::from_str("permanently_failed").unwrap(), ScrapingStatus::PermanentlyFailed); + assert_eq!(ScrapingStatus::from_str("success").unwrap(), ScrapingStatus::Success); + assert!(ScrapingStatus::from_str("foo").is_err()); + } + } +} diff --git a/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs b/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs index 35bb085a8..8c1958782 100644 --- a/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs +++ b/crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs @@ -34,7 +34,7 @@ pub enum WebContentType { // TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(WebContentType); -//impl_enum_sqlx_coders!(WebContentType); // TODO(bt, 2023-02-05): This doesn't exist for sqlite +impl_enum_sqlx_coders!(WebContentType); /// NB: Legacy API for older code. impl WebContentType { diff --git a/crates/_types/enums/src/lib.rs b/crates/_types/enums/src/lib.rs index d08c8d20c..137c27610 100644 --- a/crates/_types/enums/src/lib.rs +++ b/crates/_types/enums/src/lib.rs @@ -28,6 +28,9 @@ #[macro_use] mod macros; +#[macro_use] +mod macros2; + #[cfg(test)] pub mod test_helpers; pub mod by_table; diff --git a/crates/_types/enums/src/macros2/impl_sqlite_enum_coders.rs b/crates/_types/enums/src/macros2/impl_sqlite_enum_coders.rs new file mode 100644 index 000000000..980768c4e --- /dev/null +++ b/crates/_types/enums/src/macros2/impl_sqlite_enum_coders.rs @@ -0,0 +1,48 @@ +/// This overt approach is being taken because of the following error: +/// +/// `MySqlDatabaseError { code: Some("HY000"), number: 1210, message: "Incorrect arguments to mysqld_stmt_execute" }` +/// +/// Basically, sqlx can't turn our enum into a VARCHAR when using #[derive(sqlx::Type)]. +/// Unfortunately, by not using this, we also lose the ability to `#[sqlx(rename_all="lowercase")]`, etc., +/// so our encoder/decoder need to set the rules. +/// +/// Solution adapted from https://github.com/launchbadge/sqlx/discussions/1502 +/// The 0.6.2 series solution adapted from https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 +/// +macro_rules! impl_enum_sqlx_coders { + ($t:ident) => { + + impl sqlx::Type for $t { + fn type_info() -> sqlx_core::sqlite::SqliteTypeInfo { + // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 serves as an example + >::type_info() + } + } + + impl<'q> sqlx::Encode<'q, sqlx_core::sqlite::Sqlite> for $t { + fn encode_by_ref( + &self, + buf: &mut >::ArgumentBuffer + ) -> sqlx_core::encode::IsNull { + // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 and + // https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/str.rs.html#75-78 serves as examples + let value = self.to_str(); + <&str as sqlx::Encode>::encode(&*value, buf) + } + } + + impl<'r> sqlx::Decode<'r, sqlx_core::sqlite::Sqlite> for $t { + fn decode( + value: sqlx_core::sqlite::SqliteValueRef<'r>, + ) -> Result { + // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 serves as an example + // delegate to the &str type to decode from MySQL + let text = <&str as sqlx::Decode>::decode(value)?; + let value = $t::from_str(&text)?; + Ok(value) + } + } + + } +} + diff --git a/crates/_types/enums/src/macros2/mod.rs b/crates/_types/enums/src/macros2/mod.rs new file mode 100644 index 000000000..df0e3fa08 --- /dev/null +++ b/crates/_types/enums/src/macros2/mod.rs @@ -0,0 +1,2 @@ +#[macro_use] +pub mod impl_sqlite_enum_coders; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs index fbe1fe7ad..aebf91749 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs @@ -1,6 +1,6 @@ +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use errors::{anyhow, AnyhowResult}; use sqlx::SqlitePool; -use enums::by_table::web_scraping_targets::web_content_type::WebContentType; pub struct Args <'a> { pub canonical_url: &'a str, @@ -15,9 +15,7 @@ pub struct Args <'a> { } pub async fn insert_web_scraping_target(args: Args<'_>) -> AnyhowResult<()> { - let web_content_type = args.web_content_type.to_str().to_string(); - let query = sqlx::query!( r#" INSERT INTO web_scraping_targets( diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs new file mode 100644 index 000000000..b6294856e --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs @@ -0,0 +1,74 @@ +use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; + +pub struct WebScrapingTarget { + pub canonical_url: String, + pub web_content_type: WebContentType, + pub maybe_title: Option, + pub maybe_article_full_image_url: Option, + pub maybe_article_thumbnail_image_url: Option, + pub scraping_status: ScrapingStatus, + pub scrape_attempts: i64, +} + +pub async fn insert_web_scraping_target( + scraping_status: ScrapingStatus, + limit: i64, + sqlite_pool: &SqlitePool, +) -> AnyhowResult> { + + let scraping_status = scraping_status.to_str().to_string(); + + let query = sqlx::query_as!( + RawInternalWebScrapingTarget, + r#" +SELECT + canonical_url, + web_content_type as `web_content_type: enums::by_table::web_scraping_targets::web_content_type::WebContentType`, + maybe_title, + maybe_article_full_image_url, + maybe_article_thumbnail_image_url, + scraping_status as `scraping_status: enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus`, + scrape_attempts +FROM web_scraping_targets +WHERE + scraping_status = ? +ORDER BY id DESC +LIMIT ? + "#, + scraping_status, + limit, + ); + + let records = query.fetch_all(sqlite_pool) + .await?; + + + let records = records.into_iter() + .map(|record : RawInternalWebScrapingTarget| { + WebScrapingTarget { + canonical_url: record.canonical_url, + web_content_type: record.web_content_type, + maybe_title: record.maybe_title, + maybe_article_full_image_url: record.maybe_article_full_image_url, + maybe_article_thumbnail_image_url: record.maybe_article_thumbnail_image_url, + scraping_status: record.scraping_status, + scrape_attempts: record.scrape_attempts, + } + }) + .collect::>(); + + Ok(records) +} + +struct RawInternalWebScrapingTarget { + canonical_url: String, + web_content_type: WebContentType, + maybe_title: Option, + maybe_article_full_image_url: Option, + maybe_article_thumbnail_image_url: Option, + scraping_status: ScrapingStatus, + scrape_attempts: i64, +} diff --git a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs index 41156e612..a0a50cdf3 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs @@ -1 +1,2 @@ -pub mod insert_web_scraping_target; \ No newline at end of file +pub mod insert_web_scraping_target; +pub mod list_web_scraping_targets; \ No newline at end of file