list query, enum macros for web_scraping_targets (sqlite)

crates/_types/enums/src/by_table/web_scraping_targets/mod.rs

@@ -1 +1,2 @@
+pub mod scraping_status;
 pub mod web_content_type;

crates/_types/enums/src/by_table/web_scraping_targets/scraping_status.rs

@@ -0,0 +1,89 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`.
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub enum ScrapingStatus {
+  #[serde(rename = "new")]
+  New,
+
+  #[serde(rename = "failed")]
+  Failed,
+
+  #[serde(rename = "permanently_failed")]
+  PermanentlyFailed,
+
+  #[serde(rename = "success")]
+  Success,
+}
+
+// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests!
+impl_enum_display_and_debug_using_to_str!(ScrapingStatus);
+impl_enum_sqlx_coders!(ScrapingStatus);
+
+/// NB: Legacy API for older code.
+impl ScrapingStatus {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::New => "new",
+      Self::Failed => "failed",
+      Self::PermanentlyFailed => "permanently_failed",
+      Self::Success => "success",
+    }
+  }
+
+  pub fn from_str(value: &str) -> Result<Self, String> {
+    match value {
+      "new" => Ok(Self::New),
+      "failed" => Ok(Self::Failed),
+      "permanently_failed" => Ok(Self::PermanentlyFailed),
+      "success" => Ok(Self::Success),
+      _ => Err(format!("invalid value: {:?}", value)),
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
+  use crate::test_helpers::assert_serialization;
+
+  mod serde {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(ScrapingStatus::New, "new");
+      assert_serialization(ScrapingStatus::Failed, "failed");
+      assert_serialization(ScrapingStatus::PermanentlyFailed, "permanently_failed");
+      assert_serialization(ScrapingStatus::Success, "success");
+    }
+  }
+
+  mod impl_methods {
+    use super::*;
+
+    #[test]
+    fn test_to_str() {
+      assert_eq!(ScrapingStatus::New.to_str(), "new");
+      assert_eq!(ScrapingStatus::Failed.to_str(), "failed");
+      assert_eq!(ScrapingStatus::PermanentlyFailed.to_str(), "permanently_failed");
+      assert_eq!(ScrapingStatus::Success.to_str(), "success");
+    }
+
+    #[test]
+    fn test_from_str() {
+      assert_eq!(ScrapingStatus::from_str("new").unwrap(), ScrapingStatus::New);
+      assert_eq!(ScrapingStatus::from_str("failed").unwrap(), ScrapingStatus::Failed);
+      assert_eq!(ScrapingStatus::from_str("permanently_failed").unwrap(), ScrapingStatus::PermanentlyFailed);
+      assert_eq!(ScrapingStatus::from_str("success").unwrap(), ScrapingStatus::Success);
+      assert!(ScrapingStatus::from_str("foo").is_err());
+    }
+  }
+}

crates/_types/enums/src/by_table/web_scraping_targets/web_content_type.rs

@@ -34,7 +34,7 @@ pub enum WebContentType {
 
 // TODO(bt, 2023-01-17): This desperately needs MySQL integration tests!
 impl_enum_display_and_debug_using_to_str!(WebContentType);
-//impl_enum_sqlx_coders!(WebContentType); // TODO(bt, 2023-02-05): This doesn't exist for sqlite
+impl_enum_sqlx_coders!(WebContentType);
 
 /// NB: Legacy API for older code.
 impl WebContentType {

crates/_types/enums/src/lib.rs

@@ -28,6 +28,9 @@
 #[macro_use]
 mod macros;
 
+#[macro_use]
+mod macros2;
+
 #[cfg(test)] pub mod test_helpers;
 
 pub mod by_table;

crates/_types/enums/src/macros2/impl_sqlite_enum_coders.rs

@@ -0,0 +1,48 @@
+/// This overt approach is being taken because of the following error:
+///
+///   `MySqlDatabaseError { code: Some("HY000"), number: 1210, message: "Incorrect arguments to mysqld_stmt_execute" }`
+///
+/// Basically, sqlx can't turn our enum into a VARCHAR when using #[derive(sqlx::Type)].
+/// Unfortunately, by not using this, we also lose the ability to `#[sqlx(rename_all="lowercase")]`, etc.,
+/// so our encoder/decoder need to set the rules.
+///
+/// Solution adapted from https://github.com/launchbadge/sqlx/discussions/1502
+/// The 0.6.2 series solution adapted from https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66
+///
+macro_rules! impl_enum_sqlx_coders {
+  ($t:ident) => {
+
+    impl sqlx::Type<sqlx_core::sqlite::Sqlite> for $t {
+      fn type_info() -> sqlx_core::sqlite::SqliteTypeInfo {
+        // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 serves as an example
+        <str as sqlx::Type<sqlx_core::sqlite::Sqlite>>::type_info()
+      }
+    }
+
+    impl<'q> sqlx::Encode<'q, sqlx_core::sqlite::Sqlite> for $t {
+      fn encode_by_ref(
+        &self,
+        buf: &mut <sqlx_core::sqlite::Sqlite as sqlx_core::database::HasArguments<'q>>::ArgumentBuffer
+      ) -> sqlx_core::encode::IsNull {
+        // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 and
+        //  https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/str.rs.html#75-78 serves as examples
+        let value = self.to_str();
+        <&str as sqlx::Encode<sqlx_core::sqlite::Sqlite>>::encode(&*value, buf)
+      }
+    }
+
+    impl<'r> sqlx::Decode<'r, sqlx_core::sqlite::Sqlite> for $t {
+      fn decode(
+        value: sqlx_core::sqlite::SqliteValueRef<'r>,
+      ) -> Result<Self, sqlx_core::error::BoxDynError> {
+        // NB: https://docs.rs/sqlx-core/0.6.2/src/sqlx_core/mysql/types/uuid.rs.html#38-66 serves as an example
+        // delegate to the &str type to decode from MySQL
+        let text = <&str as sqlx::Decode<sqlx_core::sqlite::Sqlite>>::decode(value)?;
+        let value = $t::from_str(&text)?;
+        Ok(value)
+      }
+    }
+
+  }
+}
+

crates/_types/enums/src/macros2/mod.rs

@@ -0,0 +1,2 @@
+#[macro_use]
+pub mod impl_sqlite_enum_coders;

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/insert_web_scraping_target.rs

@@ -1,6 +1,6 @@
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 use errors::{anyhow, AnyhowResult};
 use sqlx::SqlitePool;
-use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
 
 pub struct Args <'a> {
   pub canonical_url: &'a str,
@@ -15,9 +15,7 @@ pub struct Args <'a> {
 }
 
 pub async fn insert_web_scraping_target(args: Args<'_>) -> AnyhowResult<()> {
-
   let web_content_type = args.web_content_type.to_str().to_string();
-
   let query = sqlx::query!(
         r#"
 INSERT INTO web_scraping_targets(

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/list_web_scraping_targets.rs

@@ -0,0 +1,74 @@
+use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
+use errors::{anyhow, AnyhowResult};
+use sqlx::SqlitePool;
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+
+pub struct WebScrapingTarget {
+  pub canonical_url: String,
+  pub web_content_type: WebContentType,
+  pub maybe_title: Option<String>,
+  pub maybe_article_full_image_url: Option<String>,
+  pub maybe_article_thumbnail_image_url: Option<String>,
+  pub scraping_status: ScrapingStatus,
+  pub scrape_attempts: i64,
+}
+
+pub async fn insert_web_scraping_target(
+  scraping_status: ScrapingStatus,
+  limit: i64,
+  sqlite_pool: &SqlitePool,
+) -> AnyhowResult<Vec<WebScrapingTarget>> {
+
+  let scraping_status = scraping_status.to_str().to_string();
+
+  let query = sqlx::query_as!(
+    RawInternalWebScrapingTarget,
+        r#"
+SELECT
+  canonical_url,
+  web_content_type as `web_content_type: enums::by_table::web_scraping_targets::web_content_type::WebContentType`,
+  maybe_title,
+  maybe_article_full_image_url,
+  maybe_article_thumbnail_image_url,
+  scraping_status as `scraping_status: enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus`,
+  scrape_attempts
+FROM web_scraping_targets
+WHERE
+  scraping_status = ?
+ORDER BY id DESC
+LIMIT ?
+        "#,
+        scraping_status,
+        limit,
+    );
+
+  let records = query.fetch_all(sqlite_pool)
+      .await?;
+
+
+  let records = records.into_iter()
+      .map(|record : RawInternalWebScrapingTarget| {
+        WebScrapingTarget {
+          canonical_url: record.canonical_url,
+          web_content_type: record.web_content_type,
+          maybe_title: record.maybe_title,
+          maybe_article_full_image_url: record.maybe_article_full_image_url,
+          maybe_article_thumbnail_image_url: record.maybe_article_thumbnail_image_url,
+          scraping_status: record.scraping_status,
+          scrape_attempts: record.scrape_attempts,
+        }
+      })
+      .collect::<Vec<WebScrapingTarget>>();
+
+  Ok(records)
+}
+
+struct RawInternalWebScrapingTarget {
+  canonical_url: String,
+  web_content_type: WebContentType,
+  maybe_title: Option<String>,
+  maybe_article_full_image_url: Option<String>,
+  maybe_article_thumbnail_image_url: Option<String>,
+  scraping_status: ScrapingStatus,
+  scrape_attempts: i64,
+}

crates/lib/sqlite_queries/src/queries/by_table/web_scraping_targets/mod.rs

@@ -1 +1,2 @@
-pub mod insert_web_scraping_target;
+pub mod insert_web_scraping_target;
+pub mod list_web_scraping_targets;