rendition schema

crates/_types/enums/src/by_table/mod.rs

@@ -2,4 +2,5 @@ pub mod media_uploads;
 pub mod model_categories;
 pub mod trending_model_analytics;
 pub mod user_ratings;
-pub mod web_scraping_targets; // NB: SqLite
+pub mod web_rendition_targets; // NB: SqLite
+pub mod web_scraping_targets; // NB: Sqlite

crates/_types/enums/src/by_table/web_rendition_targets/mod.rs

@@ -0,0 +1 @@
+pub mod rendition_status;

crates/_types/enums/src/by_table/web_rendition_targets/rendition_status.rs

@@ -0,0 +1,89 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`.
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub enum RenditionStatus {
+  #[serde(rename = "new")]
+  New,
+
+  #[serde(rename = "failed")]
+  Failed,
+
+  #[serde(rename = "permanently_failed")]
+  PermanentlyFailed,
+
+  #[serde(rename = "success")]
+  Success,
+}
+
+// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests!
+impl_enum_display_and_debug_using_to_str!(RenditionStatus);
+impl_sqlite_enum_coders!(RenditionStatus);
+
+/// NB: Legacy API for older code.
+impl RenditionStatus {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::New => "new",
+      Self::Failed => "failed",
+      Self::PermanentlyFailed => "permanently_failed",
+      Self::Success => "success",
+    }
+  }
+
+  pub fn from_str(value: &str) -> Result<Self, String> {
+    match value {
+      "new" => Ok(Self::New),
+      "failed" => Ok(Self::Failed),
+      "permanently_failed" => Ok(Self::PermanentlyFailed),
+      "success" => Ok(Self::Success),
+      _ => Err(format!("invalid value: {:?}", value)),
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::by_table::web_rendition_targets::rendition_status::RenditionStatus;
+  use crate::test_helpers::assert_serialization;
+
+  mod serde {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(RenditionStatus::New, "new");
+      assert_serialization(RenditionStatus::Failed, "failed");
+      assert_serialization(RenditionStatus::PermanentlyFailed, "permanently_failed");
+      assert_serialization(RenditionStatus::Success, "success");
+    }
+  }
+
+  mod impl_methods {
+    use super::*;
+
+    #[test]
+    fn test_to_str() {
+      assert_eq!(RenditionStatus::New.to_str(), "new");
+      assert_eq!(RenditionStatus::Failed.to_str(), "failed");
+      assert_eq!(RenditionStatus::PermanentlyFailed.to_str(), "permanently_failed");
+      assert_eq!(RenditionStatus::Success.to_str(), "success");
+    }
+
+    #[test]
+    fn test_from_str() {
+      assert_eq!(RenditionStatus::from_str("new").unwrap(), RenditionStatus::New);
+      assert_eq!(RenditionStatus::from_str("failed").unwrap(), RenditionStatus::Failed);
+      assert_eq!(RenditionStatus::from_str("permanently_failed").unwrap(), RenditionStatus::PermanentlyFailed);
+      assert_eq!(RenditionStatus::from_str("success").unwrap(), RenditionStatus::Success);
+      assert!(RenditionStatus::from_str("foo").is_err());
+    }
+  }
+}

crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs

@@ -39,11 +39,6 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult<WebScrapingResult> {
   let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string();
   let document = Html::parse_document(&downloaded_document);
 
-  //println!("Document: {:?}", document);
-
-  //let selector = Selector::parse(".article__content")
-  //let selector = Selector::parse("p.paragraph")
-
   let mut paragraphs = Vec::new();
 
   if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() {

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE web_rendition_targets;

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/up.sql

@@ -0,0 +1,32 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These tables are designed for sqlite, not MySQL!
+
+CREATE TABLE web_rendition_targets (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+
+  -- Canonical URLs help us avoid aliases of the same content.
+  -- We'll need to figure this out for each site.
+  canonical_url TEXT NOT NULL UNIQUE,
+
+  -- TODO: Once we have more types of content being ingested and more output modalities:
+  -- Effectively an enum-like: NewsArticle, WritingPrompt, SocialMediaDiscussion,  etc.
+  -- This controls the template that we use to query against GPT / LLM.
+  -- rephrasing_type TEXT NOT NULL,
+
+  -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
+  web_content_type TEXT NOT NULL,
+
+  -- Scraping status: "new", "failed", "permanently_failed", "success"
+  rendition_status TEXT DEFAULT "new" NOT NULL,
+  rendition_attempts INT DEFAULT 0 NOT NULL,
+
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+
+  version INT DEFAULT 0 NOT NULL
+);
+
+CREATE INDEX index_rendition_status ON web_rendition_targets(rendition_status);

crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/down.sql

@@ -1 +0,0 @@
--- This file should undo anything in `up.sql`

crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/up.sql

@@ -1 +0,0 @@
--- Your SQL goes here

crates/programs/aichatbot-sidecar/src/main.rs

@@ -40,6 +40,10 @@ use workers::web_content_scraping::single_target::ingest_url_scrape_and_save::in
 
 #[tokio::main]
 pub async fn main() -> AnyhowResult<()> {
+  test().await
+}
+
+async fn test() -> AnyhowResult<()> {
   let database_url = easyenv::get_env_string_required("DATABASE_URL")?;
   let pool = SqlitePoolOptions::new()
       .max_connections(5)

crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs

@@ -13,10 +13,6 @@ pub async fn ingest_url_scrape_and_save(url: &str, web_content_type: WebContentT
     _ => return Ok(()), // TODO: Implement the rest
   };
 
-  println!("\n\nScraping result: {:?}", scraping_result.result);
-
-  println!("\n\nScraping result: {}", scraping_result.result.body_text);
-
   {
     let directory = save_directory.directory_for_webpage_url(url)?;
     std::fs::create_dir_all(&directory)?;