WIP - tables for stories

_sql/analytics/tts/tts_results.sql

@@ -49,6 +49,21 @@ where model_token='TM:yt4gfbkngsjj'
 order by id desc
 limit 150;
 
+-- Search for results by eminem
+select
+    token,
+    created_at,
+    raw_inference_text
+from  tts_results
+where
+    model_token IN ('TM:27fj0gsh11pd', 'TM:8h9bfjgabeer', 'TM:pdf9c1anbdjq')
+    AND raw_inference_text LIKE "%rave%"
+  AND raw_inference_text LIKE "%sound%"
+  AND created_at > ( CURDATE() - INTERVAL 5 MONTH )
+ORDER BY created_at DESC
+
+
+
 -- Delete TTS results for a single model
 -- Do this in short batches so a lock isn't held for prohibitively long.
 update tts_results
@@ -56,3 +71,4 @@ set mod_deleted_at = NOW()
 where model_token = 'TM:ztt5s1be5tq6'
 and mod_deleted_at IS NULL
 limit 5000;
+

crates/_types/enums/src/common/sqlite/chatbot_job_status.rs

@@ -0,0 +1,97 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`.
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub enum ChatbotJobStatus {
+  #[serde(rename = "new")]
+  New,
+
+  #[serde(rename = "skipped")]
+  Skipped,
+
+  #[serde(rename = "failed")]
+  Failed,
+
+  #[serde(rename = "permanently_failed")]
+  PermanentlyFailed,
+
+  #[serde(rename = "success")]
+  Success,
+}
+
+// TODO(bt, 2023-02-10): This desperately needs Sqlite integration tests!
+impl_enum_display_and_debug_using_to_str!(ChatbotJobStatus);
+impl_sqlite_enum_coders!(ChatbotJobStatus);
+
+/// NB: Legacy API for older code.
+impl ChatbotJobStatus {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::New => "new",
+      Self::Skipped => "skipped",
+      Self::Failed => "failed",
+      Self::PermanentlyFailed => "permanently_failed",
+      Self::Success => "success",
+    }
+  }
+
+  pub fn from_str(value: &str) -> Result<Self, String> {
+    match value {
+      "new" => Ok(Self::New),
+      "skipped" => Ok(Self::Skipped),
+      "failed" => Ok(Self::Failed),
+      "permanently_failed" => Ok(Self::PermanentlyFailed),
+      "success" => Ok(Self::Success),
+      _ => Err(format!("invalid value: {:?}", value)),
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::common::sqlite::chatbot_job_status::ChatbotJobStatus;
+  use crate::test_helpers::assert_serialization;
+
+  mod serde {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(ChatbotJobStatus::New, "new");
+      assert_serialization(ChatbotJobStatus::Skipped, "skipped");
+      assert_serialization(ChatbotJobStatus::Failed, "failed");
+      assert_serialization(ChatbotJobStatus::PermanentlyFailed, "permanently_failed");
+      assert_serialization(ChatbotJobStatus::Success, "success");
+    }
+  }
+
+  mod impl_methods {
+    use super::*;
+
+    #[test]
+    fn test_to_str() {
+      assert_eq!(ChatbotJobStatus::New.to_str(), "new");
+      assert_eq!(ChatbotJobStatus::Skipped.to_str(), "skipped");
+      assert_eq!(ChatbotJobStatus::Failed.to_str(), "failed");
+      assert_eq!(ChatbotJobStatus::PermanentlyFailed.to_str(), "permanently_failed");
+      assert_eq!(ChatbotJobStatus::Success.to_str(), "success");
+    }
+
+    #[test]
+    fn test_from_str() {
+      assert_eq!(ChatbotJobStatus::from_str("new").unwrap(), ChatbotJobStatus::New);
+      assert_eq!(ChatbotJobStatus::from_str("skipped").unwrap(), ChatbotJobStatus::Skipped);
+      assert_eq!(ChatbotJobStatus::from_str("failed").unwrap(), ChatbotJobStatus::Failed);
+      assert_eq!(ChatbotJobStatus::from_str("permanently_failed").unwrap(), ChatbotJobStatus::PermanentlyFailed);
+      assert_eq!(ChatbotJobStatus::from_str("success").unwrap(), ChatbotJobStatus::Success);
+      assert!(ChatbotJobStatus::from_str("foo").is_err());
+    }
+  }
+}

crates/_types/enums/src/common/sqlite/mod.rs

@@ -1 +1,2 @@
+pub mod chatbot_job_status;
 pub mod skip_reason;

crates/_types/tokens/src/prefixes.rs

@@ -17,6 +17,7 @@ pub(crate) enum EntityType {
   InferenceJob,
   MediaUpload,
   ModelCategory,
+  NewsStory, // NB: aichatbot / sqlite
   TtsModel,
   User,
   VoiceConversionModel,
@@ -31,6 +32,7 @@ impl EntityType {
       Self::InferenceJob => "jinf_",
       Self::MediaUpload => "mu_",
       Self::ModelCategory => "CAT:", // NB: Old-style prefix, do not use for future tokens.
+      Self::NewsStory => "news_story_",
       Self::TtsModel => "TM:", // NB: Old-style prefix, do not use for future tokens.
       Self::User => "U:", // NB: Old-style prefix, do not use for future tokens.
       Self::VoiceConversionModel => "voco_",

crates/_types/tokens/src/tokens/mod.rs

@@ -1,3 +1,4 @@
 pub mod model_categories;
 pub mod tts_models;
 pub mod w2l_templates;
+pub mod news_stories;

crates/_types/tokens/src/tokens/news_stories.rs

@@ -0,0 +1,12 @@
+use crate::prefixes::EntityType;
+use serde::Deserialize;
+use serde::Serialize;
+use std::fmt::Debug;
+
+/// The primary key for "generic" inference jobs.
+#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
+#[sqlx(transparent)]
+pub struct NewsStoryToken(pub String);
+
+impl_string_token!(NewsStoryToken);
+impl_crockford_generator!(NewsStoryToken, 32usize, EntityType::NewsStory, CrockfordMixed);

crates/lib/sqlite_queries/src/queries/by_table/mod.rs

@@ -1,2 +1,3 @@
+pub mod news_story_preproductions;
 pub mod web_rendition_targets;
 pub mod web_scraping_targets;

crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/insert_news_story_preproduction.rs

@@ -0,0 +1,60 @@
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use enums::common::sqlite::skip_reason::SkipReason;
+use errors::{anyhow, AnyhowResult};
+use sqlx::SqlitePool;
+use tokens::tokens::news_stories::NewsStoryToken;
+use tokens::tokens::tts_models::TtsModelToken;
+
+pub struct Args <'a> {
+  pub canonical_url: &'a str,
+
+  pub news_story_token: &'a NewsStoryToken,
+
+  pub original_news_canonical_url: &'a str,
+
+  pub tts_voice_identifier: &'a TtsModelToken,
+
+  pub full_text: &'a str,
+
+  pub sqlite_pool: &'a SqlitePool,
+}
+
+pub async fn insert_news_story_preproductions(args: Args<'_>) -> AnyhowResult<()> {
+  let news_story_token = args.news_story_token.as_str();
+  let tts_model_token = args.tts_voice_identifier.as_str();
+
+  let query = sqlx::query!(
+        r#"
+INSERT INTO news_story_preproductions (
+  news_story_token,
+  original_news_canonical_url,
+  tts_service,
+  tts_voice_identifier,
+  full_text
+)
+VALUES (
+  ?,
+  ?,
+  "fakeyou",
+  ?,
+  ?
+)
+        "#,
+        news_story_token,
+        args.original_news_canonical_url,
+        tts_model_token,
+        args.full_text
+    );
+
+  let query_result = query.execute(args.sqlite_pool)
+      .await;
+
+  let _record_id = match query_result {
+    Ok(res) => res.last_insert_rowid(),
+    Err(err) => {
+      return Err(anyhow!("error inserting: {:?}", err));
+    }
+  };
+
+  Ok(())
+}

crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/list_news_story_preproductions.rs

@@ -0,0 +1,87 @@
+use enums::common::sqlite::chatbot_job_status::ChatbotJobStatus;
+use errors::{anyhow, AnyhowResult};
+use sqlx::SqlitePool;
+use tokens::tokens::news_stories::NewsStoryToken;
+use tokens::tokens::tts_models::TtsModelToken;
+
+pub struct NewsStoryPreproduction {
+  id: i64,
+  news_story_token: NewsStoryToken,
+  original_news_canonical_url: String,
+  tts_service: String,
+  tts_voice_identifier: TtsModelToken,
+  full_text: String,
+  maybe_result_url: Option<String>,
+  maybe_filesystem_relative_location: Option<String>,
+  news_story_production_status: ChatbotJobStatus,
+}
+
+pub async fn list_news_story_preproductions(
+  job_status: ChatbotJobStatus,
+  last_id: i64,
+  limit: i64,
+  sqlite_pool: &SqlitePool,
+) -> AnyhowResult<Vec<NewsStoryPreproduction>> {
+
+  // NB: Sqlx doesn't support `WHERE ... IN (...)` "yet". :(
+  // https://github.com/launchbadge/sqlx/blob/6d0d7402c8a9cbea2676a1795e9fb50b0cf60c03/FAQ.md?plain=1#L73
+  let job_status = job_status.to_str().to_string();
+
+  let query = sqlx::query_as!(
+    RawInternalNewsStoryPreproduction,
+        r#"
+SELECT
+  id,
+  news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`,
+  original_news_canonical_url,
+  tts_service,
+  tts_voice_identifier as `tts_voice_identifier: tokens::tokens::tts_models::TtsModelToken`,
+  full_text,
+  maybe_result_url,
+  maybe_filesystem_relative_location,
+  news_story_production_status as `news_story_production_status: enums::common::sqlite::chatbot_job_status::ChatbotJobStatus`
+FROM news_story_preproductions
+WHERE
+  news_story_production_status = ?
+  AND id > ?
+ORDER BY id ASC
+LIMIT ?
+        "#,
+        job_status,
+        last_id,
+        limit,
+    );
+
+  let records = query.fetch_all(sqlite_pool)
+      .await?;
+
+  let records = records.into_iter()
+      .map(|record : RawInternalNewsStoryPreproduction| {
+        NewsStoryPreproduction {
+          id: record.id,
+          news_story_token: record.news_story_token,
+          original_news_canonical_url: record.original_news_canonical_url,
+          tts_service: record.tts_service,
+          tts_voice_identifier: record.tts_voice_identifier,
+          full_text: record.full_text,
+          maybe_result_url: record.maybe_result_url,
+          maybe_filesystem_relative_location: record.maybe_filesystem_relative_location,
+          news_story_production_status: record.news_story_production_status,
+        }
+      })
+      .collect::<Vec<NewsStoryPreproduction>>();
+
+  Ok(records)
+}
+
+struct RawInternalNewsStoryPreproduction {
+  id: i64,
+  news_story_token: NewsStoryToken,
+  original_news_canonical_url: String,
+  tts_service: String,
+  tts_voice_identifier: TtsModelToken,
+  full_text: String,
+  maybe_result_url: Option<String>,
+  maybe_filesystem_relative_location: Option<String>,
+  news_story_production_status: ChatbotJobStatus,
+}

crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/mod.rs

@@ -0,0 +1,3 @@
+pub mod insert_news_story_preproduction;
+pub mod list_news_story_preproductions;
+pub mod update_news_story_preproductions;

crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/update_news_story_preproductions.rs

@@ -0,0 +1,38 @@
+use enums::by_table::web_scraping_targets::web_content_type::WebContentType;
+use errors::{anyhow, AnyhowResult};
+use sqlx::SqlitePool;
+use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
+use enums::common::sqlite::chatbot_job_status::ChatbotJobStatus;
+use tokens::tokens::news_stories::NewsStoryToken;
+
+pub struct Args <'a> {
+  pub news_story_token: &'a NewsStoryToken,
+
+  pub news_story_production_status: ChatbotJobStatus,
+
+  pub sqlite_pool: &'a SqlitePool,
+}
+
+pub async fn update_news_story_preproductions(args: Args<'_>) -> AnyhowResult<()> {
+  let news_story_production_status= args.news_story_production_status.to_str().to_string();
+  let query = sqlx::query!(
+        r#"
+UPDATE news_story_preproductions
+SET
+  news_story_production_status = ?,
+  version = version + 1
+WHERE
+  news_story_token = ?
+        "#,
+        news_story_production_status,
+        args.news_story_token,
+    );
+
+  let query_result = query.execute(args.sqlite_pool)
+      .await;
+
+  match query_result {
+    Ok(_) => Ok(()),
+    Err(err) => Err(anyhow!("error updating: {:?}", err)),
+  }
+}

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE tts_render_targets;

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/up.sql

@@ -0,0 +1,42 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These tables are designed for sqlite, not MySQL!
+
+CREATE TABLE tts_render_targets (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+
+  -- If the source came from somewhere, this is it
+  -- maybe_canonical_url TEXT NOT NULL UNIQUE,
+
+  -- The name of the service used to generate TTS.
+  -- I expect this will be FakeYou for a long time, but perhaps that will one day change.
+  -- Enum-like value, eg. "fakeyou".
+  tts_service TEXT NOT NULL,
+
+  -- The identifier of the voice. For FakeYou, this is the tts_token.
+  tts_voice_identifier TEXT NOT NULL,
+
+  -- Original text of the TTS result
+  full_text TEXT NOT NULL,
+
+  -- For audio results, the URL to the result.
+  maybe_result_url TEXT,
+
+  -- For audio results that get downloaded, they live on the filesystem here
+  -- This is relative to the runtime data directory.
+  maybe_filesystem_relative_location TEXT,
+
+  -- TTS render status: "new", "processing", "failed", "permanently_failed", "success", "skipped"
+  tts_render_status TEXT DEFAULT "new" NOT NULL,
+  tts_render_attempts INT DEFAULT 0 NOT NULL,
+
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  -- fetched_at DATETIME DEFAULT NULL,
+
+  version INT DEFAULT 0 NOT NULL
+);
+
+CREATE INDEX index_tts_render_status ON tts_render_targets(tts_render_status);

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE news_story_preproductions;

crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/up.sql

@@ -0,0 +1,47 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE news_story_preproductions (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+
+  -- TODO: Generic "story_token"
+  -- Entropic token that follows a single news story through the phases of production
+  -- This isn't a primary key that belongs to this table alone: it tracks several 1:1 and 1:n entities.
+  news_story_token TEXT NOT NULL UNIQUE,
+
+  -- The original URL for the news item
+  original_news_canonical_url TEXT NOT NULL UNIQUE,
+
+  -- If the source came from somewhere, this is it
+  -- maybe_canonical_url TEXT NOT NULL UNIQUE,
+
+  -- The name of the service used to generate TTS.
+  -- I expect this will be FakeYou for a long time, but perhaps that will one day change.
+  -- Enum-like value, eg. "fakeyou".
+  tts_service TEXT NOT NULL,
+
+  -- The identifier of the voice. For FakeYou, this is the tts_token.
+  tts_voice_identifier TEXT NOT NULL,
+
+  -- Original text of the TTS result
+  full_text TEXT NOT NULL,
+
+  -- For audio results, the URL to the result.
+  maybe_result_url TEXT,
+
+  -- For audio results that get downloaded, they live on the filesystem here
+  -- This is relative to the runtime data directory.
+  maybe_filesystem_relative_location TEXT,
+
+  -- TTS render status: "new", "processing", "failed", "permanently_failed", "success", "skipped"
+  news_story_production_status TEXT DEFAULT "new" NOT NULL,
+
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  -- fetched_at DATETIME DEFAULT NULL,
+
+  version INT DEFAULT 0 NOT NULL
+);
+
+CREATE INDEX index_news_story_production_status ON news_story_preproductions(news_story_production_status);