From 5878224f7eb9d79bcbf320d2806d29378fde9333 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 7 Feb 2023 22:40:35 -0500 Subject: [PATCH] rendition schema --- crates/_types/enums/src/by_table/mod.rs | 3 +- .../src/by_table/web_rendition_targets/mod.rs | 1 + .../web_rendition_targets/rendition_status.rs | 89 +++++++++++++++++++ .../src/sites/cnn/cnn_article_scraper.rs | 5 -- .../down.sql | 5 ++ .../up.sql | 32 +++++++ .../down.sql | 1 - .../up.sql | 1 - crates/programs/aichatbot-sidecar/src/main.rs | 4 + .../ingest_url_scrape_and_save.rs | 4 - 10 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 crates/_types/enums/src/by_table/web_rendition_targets/mod.rs create mode 100644 crates/_types/enums/src/by_table/web_rendition_targets/rendition_status.rs create mode 100644 crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/down.sql create mode 100644 crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/up.sql delete mode 100644 crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/down.sql delete mode 100644 crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/up.sql diff --git a/crates/_types/enums/src/by_table/mod.rs b/crates/_types/enums/src/by_table/mod.rs index a071d1bf0c..230f8906bd 100644 --- a/crates/_types/enums/src/by_table/mod.rs +++ b/crates/_types/enums/src/by_table/mod.rs @@ -2,4 +2,5 @@ pub mod media_uploads; pub mod model_categories; pub mod trending_model_analytics; pub mod user_ratings; -pub mod web_scraping_targets; // NB: SqLite +pub mod web_rendition_targets; // NB: SqLite +pub mod web_scraping_targets; // NB: Sqlite diff --git a/crates/_types/enums/src/by_table/web_rendition_targets/mod.rs b/crates/_types/enums/src/by_table/web_rendition_targets/mod.rs new file mode 100644 index 0000000000..371fe4d32f --- /dev/null +++ b/crates/_types/enums/src/by_table/web_rendition_targets/mod.rs @@ -0,0 +1 @@ +pub mod rendition_status; diff --git a/crates/_types/enums/src/by_table/web_rendition_targets/rendition_status.rs b/crates/_types/enums/src/by_table/web_rendition_targets/rendition_status.rs new file mode 100644 index 0000000000..a59639b974 --- /dev/null +++ b/crates/_types/enums/src/by_table/web_rendition_targets/rendition_status.rs @@ -0,0 +1,89 @@ +use serde::Deserialize; +use serde::Serialize; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum RenditionStatus { + #[serde(rename = "new")] + New, + + #[serde(rename = "failed")] + Failed, + + #[serde(rename = "permanently_failed")] + PermanentlyFailed, + + #[serde(rename = "success")] + Success, +} + +// TODO(bt, 2023-01-17): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(RenditionStatus); +impl_sqlite_enum_coders!(RenditionStatus); + +/// NB: Legacy API for older code. +impl RenditionStatus { + pub fn to_str(&self) -> &'static str { + match self { + Self::New => "new", + Self::Failed => "failed", + Self::PermanentlyFailed => "permanently_failed", + Self::Success => "success", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "new" => Ok(Self::New), + "failed" => Ok(Self::Failed), + "permanently_failed" => Ok(Self::PermanentlyFailed), + "success" => Ok(Self::Success), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::web_rendition_targets::rendition_status::RenditionStatus; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(RenditionStatus::New, "new"); + assert_serialization(RenditionStatus::Failed, "failed"); + assert_serialization(RenditionStatus::PermanentlyFailed, "permanently_failed"); + assert_serialization(RenditionStatus::Success, "success"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(RenditionStatus::New.to_str(), "new"); + assert_eq!(RenditionStatus::Failed.to_str(), "failed"); + assert_eq!(RenditionStatus::PermanentlyFailed.to_str(), "permanently_failed"); + assert_eq!(RenditionStatus::Success.to_str(), "success"); + } + + #[test] + fn test_from_str() { + assert_eq!(RenditionStatus::from_str("new").unwrap(), RenditionStatus::New); + assert_eq!(RenditionStatus::from_str("failed").unwrap(), RenditionStatus::Failed); + assert_eq!(RenditionStatus::from_str("permanently_failed").unwrap(), RenditionStatus::PermanentlyFailed); + assert_eq!(RenditionStatus::from_str("success").unwrap(), RenditionStatus::Success); + assert!(RenditionStatus::from_str("foo").is_err()); + } + } +} diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs index e48307e2d5..fb7f4e0eef 100644 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs +++ b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs @@ -39,11 +39,6 @@ pub async fn cnn_article_scraper(url: &str) -> AnyhowResult { let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); let document = Html::parse_document(&downloaded_document); - //println!("Document: {:?}", document); - - //let selector = Selector::parse(".article__content") - //let selector = Selector::parse("p.paragraph") - let mut paragraphs = Vec::new(); if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/down.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/down.sql new file mode 100644 index 0000000000..051e55aaa4 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE web_rendition_targets; diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/up.sql new file mode 100644 index 0000000000..4416ee6b9a --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-05-064422_web_rendition_targets/up.sql @@ -0,0 +1,32 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: These tables are designed for sqlite, not MySQL! + +CREATE TABLE web_rendition_targets ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + + -- Canonical URLs help us avoid aliases of the same content. + -- We'll need to figure this out for each site. + canonical_url TEXT NOT NULL UNIQUE, + + -- TODO: Once we have more types of content being ingested and more output modalities: + -- Effectively an enum-like: NewsArticle, WritingPrompt, SocialMediaDiscussion, etc. + -- This controls the template that we use to query against GPT / LLM. + -- rephrasing_type TEXT NOT NULL, + + -- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc. + web_content_type TEXT NOT NULL, + + -- Scraping status: "new", "failed", "permanently_failed", "success" + rendition_status TEXT DEFAULT "new" NOT NULL, + rendition_attempts INT DEFAULT 0 NOT NULL, + + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + + version INT DEFAULT 0 NOT NULL +); + +CREATE INDEX index_rendition_status ON web_rendition_targets(rendition_status); diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/down.sql b/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/down.sql deleted file mode 100644 index 291a97c5ce..0000000000 --- a/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/down.sql +++ /dev/null @@ -1 +0,0 @@ --- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/up.sql deleted file mode 100644 index 33cf74b578..0000000000 --- a/crates/programs/aichatbot-sidecar/_sql/migrations_todo/2023-02-05-064422_web_rephrasing_targets/up.sql +++ /dev/null @@ -1 +0,0 @@ --- Your SQL goes here \ No newline at end of file diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index c7ca023921..fc85ad6f49 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -40,6 +40,10 @@ use workers::web_content_scraping::single_target::ingest_url_scrape_and_save::in #[tokio::main] pub async fn main() -> AnyhowResult<()> { + test().await +} + +async fn test() -> AnyhowResult<()> { let database_url = easyenv::get_env_string_required("DATABASE_URL")?; let pool = SqlitePoolOptions::new() .max_connections(5) diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs index db641b9a52..020cbfda91 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs @@ -13,10 +13,6 @@ pub async fn ingest_url_scrape_and_save(url: &str, web_content_type: WebContentT _ => return Ok(()), // TODO: Implement the rest }; - println!("\n\nScraping result: {:?}", scraping_result.result); - - println!("\n\nScraping result: {}", scraping_result.result.body_text); - { let directory = save_directory.directory_for_webpage_url(url)?; std::fs::create_dir_all(&directory)?;