diff --git a/_sql/analytics/tts_category_assignments.sql b/_sql/analytics/tts/tts_category_assignments.sql similarity index 100% rename from _sql/analytics/tts_category_assignments.sql rename to _sql/analytics/tts/tts_category_assignments.sql diff --git a/_sql/analytics/tts_inference_jobs.sql b/_sql/analytics/tts/tts_inference_jobs.sql similarity index 100% rename from _sql/analytics/tts_inference_jobs.sql rename to _sql/analytics/tts/tts_inference_jobs.sql diff --git a/_sql/analytics/tts_models.sql b/_sql/analytics/tts/tts_models.sql similarity index 100% rename from _sql/analytics/tts_models.sql rename to _sql/analytics/tts/tts_models.sql diff --git a/_sql/analytics/tts_results.sql b/_sql/analytics/tts/tts_results.sql similarity index 80% rename from _sql/analytics/tts_results.sql rename to _sql/analytics/tts/tts_results.sql index 935a84300a..98312ad7c1 100644 --- a/_sql/analytics/tts_results.sql +++ b/_sql/analytics/tts/tts_results.sql @@ -49,6 +49,21 @@ where model_token='TM:yt4gfbkngsjj' order by id desc limit 150; +-- Search for results by eminem +select + token, + created_at, + raw_inference_text +from tts_results +where + model_token IN ('TM:27fj0gsh11pd', 'TM:8h9bfjgabeer', 'TM:pdf9c1anbdjq') + AND raw_inference_text LIKE "%rave%" + AND raw_inference_text LIKE "%sound%" + AND created_at > ( CURDATE() - INTERVAL 5 MONTH ) +ORDER BY created_at DESC + + + -- Delete TTS results for a single model -- Do this in short batches so a lock isn't held for prohibitively long. update tts_results @@ -56,3 +71,4 @@ set mod_deleted_at = NOW() where model_token = 'TM:ztt5s1be5tq6' and mod_deleted_at IS NULL limit 5000; + diff --git a/crates/_types/enums/src/common/sqlite/chatbot_job_status.rs b/crates/_types/enums/src/common/sqlite/chatbot_job_status.rs new file mode 100644 index 0000000000..9c5bffed70 --- /dev/null +++ b/crates/_types/enums/src/common/sqlite/chatbot_job_status.rs @@ -0,0 +1,97 @@ +use serde::Deserialize; +use serde::Serialize; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// Used in the SqLite `web_scraping_targets` table in a `TEXT` field named `scraping_status`. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)] +pub enum ChatbotJobStatus { + #[serde(rename = "new")] + New, + + #[serde(rename = "skipped")] + Skipped, + + #[serde(rename = "failed")] + Failed, + + #[serde(rename = "permanently_failed")] + PermanentlyFailed, + + #[serde(rename = "success")] + Success, +} + +// TODO(bt, 2023-02-10): This desperately needs Sqlite integration tests! +impl_enum_display_and_debug_using_to_str!(ChatbotJobStatus); +impl_sqlite_enum_coders!(ChatbotJobStatus); + +/// NB: Legacy API for older code. +impl ChatbotJobStatus { + pub fn to_str(&self) -> &'static str { + match self { + Self::New => "new", + Self::Skipped => "skipped", + Self::Failed => "failed", + Self::PermanentlyFailed => "permanently_failed", + Self::Success => "success", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "new" => Ok(Self::New), + "skipped" => Ok(Self::Skipped), + "failed" => Ok(Self::Failed), + "permanently_failed" => Ok(Self::PermanentlyFailed), + "success" => Ok(Self::Success), + _ => Err(format!("invalid value: {:?}", value)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::common::sqlite::chatbot_job_status::ChatbotJobStatus; + use crate::test_helpers::assert_serialization; + + mod serde { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(ChatbotJobStatus::New, "new"); + assert_serialization(ChatbotJobStatus::Skipped, "skipped"); + assert_serialization(ChatbotJobStatus::Failed, "failed"); + assert_serialization(ChatbotJobStatus::PermanentlyFailed, "permanently_failed"); + assert_serialization(ChatbotJobStatus::Success, "success"); + } + } + + mod impl_methods { + use super::*; + + #[test] + fn test_to_str() { + assert_eq!(ChatbotJobStatus::New.to_str(), "new"); + assert_eq!(ChatbotJobStatus::Skipped.to_str(), "skipped"); + assert_eq!(ChatbotJobStatus::Failed.to_str(), "failed"); + assert_eq!(ChatbotJobStatus::PermanentlyFailed.to_str(), "permanently_failed"); + assert_eq!(ChatbotJobStatus::Success.to_str(), "success"); + } + + #[test] + fn test_from_str() { + assert_eq!(ChatbotJobStatus::from_str("new").unwrap(), ChatbotJobStatus::New); + assert_eq!(ChatbotJobStatus::from_str("skipped").unwrap(), ChatbotJobStatus::Skipped); + assert_eq!(ChatbotJobStatus::from_str("failed").unwrap(), ChatbotJobStatus::Failed); + assert_eq!(ChatbotJobStatus::from_str("permanently_failed").unwrap(), ChatbotJobStatus::PermanentlyFailed); + assert_eq!(ChatbotJobStatus::from_str("success").unwrap(), ChatbotJobStatus::Success); + assert!(ChatbotJobStatus::from_str("foo").is_err()); + } + } +} diff --git a/crates/_types/enums/src/common/sqlite/mod.rs b/crates/_types/enums/src/common/sqlite/mod.rs index db9087b917..0a6cdc79b7 100644 --- a/crates/_types/enums/src/common/sqlite/mod.rs +++ b/crates/_types/enums/src/common/sqlite/mod.rs @@ -1 +1,2 @@ +pub mod chatbot_job_status; pub mod skip_reason; \ No newline at end of file diff --git a/crates/_types/tokens/src/prefixes.rs b/crates/_types/tokens/src/prefixes.rs index 8e5ed09038..7bfe7fc5d6 100644 --- a/crates/_types/tokens/src/prefixes.rs +++ b/crates/_types/tokens/src/prefixes.rs @@ -17,6 +17,7 @@ pub(crate) enum EntityType { InferenceJob, MediaUpload, ModelCategory, + NewsStory, // NB: aichatbot / sqlite TtsModel, User, VoiceConversionModel, @@ -31,6 +32,7 @@ impl EntityType { Self::InferenceJob => "jinf_", Self::MediaUpload => "mu_", Self::ModelCategory => "CAT:", // NB: Old-style prefix, do not use for future tokens. + Self::NewsStory => "news_story_", Self::TtsModel => "TM:", // NB: Old-style prefix, do not use for future tokens. Self::User => "U:", // NB: Old-style prefix, do not use for future tokens. Self::VoiceConversionModel => "voco_", diff --git a/crates/_types/tokens/src/tokens/mod.rs b/crates/_types/tokens/src/tokens/mod.rs index 7c9ff9a784..871a0580b4 100644 --- a/crates/_types/tokens/src/tokens/mod.rs +++ b/crates/_types/tokens/src/tokens/mod.rs @@ -1,3 +1,4 @@ pub mod model_categories; pub mod tts_models; pub mod w2l_templates; +pub mod news_stories; diff --git a/crates/_types/tokens/src/tokens/news_stories.rs b/crates/_types/tokens/src/tokens/news_stories.rs new file mode 100644 index 0000000000..aae770d235 --- /dev/null +++ b/crates/_types/tokens/src/tokens/news_stories.rs @@ -0,0 +1,12 @@ +use crate::prefixes::EntityType; +use serde::Deserialize; +use serde::Serialize; +use std::fmt::Debug; + +/// The primary key for "generic" inference jobs. +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[sqlx(transparent)] +pub struct NewsStoryToken(pub String); + +impl_string_token!(NewsStoryToken); +impl_crockford_generator!(NewsStoryToken, 32usize, EntityType::NewsStory, CrockfordMixed); diff --git a/crates/lib/sqlite_queries/src/queries/by_table/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/mod.rs index 54c2926ad8..b0678d225b 100644 --- a/crates/lib/sqlite_queries/src/queries/by_table/mod.rs +++ b/crates/lib/sqlite_queries/src/queries/by_table/mod.rs @@ -1,2 +1,3 @@ +pub mod news_story_preproductions; pub mod web_rendition_targets; pub mod web_scraping_targets; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/insert_news_story_preproduction.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/insert_news_story_preproduction.rs new file mode 100644 index 0000000000..d0d50bdd88 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/insert_news_story_preproduction.rs @@ -0,0 +1,60 @@ +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use enums::common::sqlite::skip_reason::SkipReason; +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use tokens::tokens::news_stories::NewsStoryToken; +use tokens::tokens::tts_models::TtsModelToken; + +pub struct Args <'a> { + pub canonical_url: &'a str, + + pub news_story_token: &'a NewsStoryToken, + + pub original_news_canonical_url: &'a str, + + pub tts_voice_identifier: &'a TtsModelToken, + + pub full_text: &'a str, + + pub sqlite_pool: &'a SqlitePool, +} + +pub async fn insert_news_story_preproductions(args: Args<'_>) -> AnyhowResult<()> { + let news_story_token = args.news_story_token.as_str(); + let tts_model_token = args.tts_voice_identifier.as_str(); + + let query = sqlx::query!( + r#" +INSERT INTO news_story_preproductions ( + news_story_token, + original_news_canonical_url, + tts_service, + tts_voice_identifier, + full_text +) +VALUES ( + ?, + ?, + "fakeyou", + ?, + ? +) + "#, + news_story_token, + args.original_news_canonical_url, + tts_model_token, + args.full_text + ); + + let query_result = query.execute(args.sqlite_pool) + .await; + + let _record_id = match query_result { + Ok(res) => res.last_insert_rowid(), + Err(err) => { + return Err(anyhow!("error inserting: {:?}", err)); + } + }; + + Ok(()) +} diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/list_news_story_preproductions.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/list_news_story_preproductions.rs new file mode 100644 index 0000000000..c213f47a88 --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/list_news_story_preproductions.rs @@ -0,0 +1,87 @@ +use enums::common::sqlite::chatbot_job_status::ChatbotJobStatus; +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use tokens::tokens::news_stories::NewsStoryToken; +use tokens::tokens::tts_models::TtsModelToken; + +pub struct NewsStoryPreproduction { + id: i64, + news_story_token: NewsStoryToken, + original_news_canonical_url: String, + tts_service: String, + tts_voice_identifier: TtsModelToken, + full_text: String, + maybe_result_url: Option, + maybe_filesystem_relative_location: Option, + news_story_production_status: ChatbotJobStatus, +} + +pub async fn list_news_story_preproductions( + job_status: ChatbotJobStatus, + last_id: i64, + limit: i64, + sqlite_pool: &SqlitePool, +) -> AnyhowResult> { + + // NB: Sqlx doesn't support `WHERE ... IN (...)` "yet". :( + // https://github.com/launchbadge/sqlx/blob/6d0d7402c8a9cbea2676a1795e9fb50b0cf60c03/FAQ.md?plain=1#L73 + let job_status = job_status.to_str().to_string(); + + let query = sqlx::query_as!( + RawInternalNewsStoryPreproduction, + r#" +SELECT + id, + news_story_token as `news_story_token: tokens::tokens::news_stories::NewsStoryToken`, + original_news_canonical_url, + tts_service, + tts_voice_identifier as `tts_voice_identifier: tokens::tokens::tts_models::TtsModelToken`, + full_text, + maybe_result_url, + maybe_filesystem_relative_location, + news_story_production_status as `news_story_production_status: enums::common::sqlite::chatbot_job_status::ChatbotJobStatus` +FROM news_story_preproductions +WHERE + news_story_production_status = ? + AND id > ? +ORDER BY id ASC +LIMIT ? + "#, + job_status, + last_id, + limit, + ); + + let records = query.fetch_all(sqlite_pool) + .await?; + + let records = records.into_iter() + .map(|record : RawInternalNewsStoryPreproduction| { + NewsStoryPreproduction { + id: record.id, + news_story_token: record.news_story_token, + original_news_canonical_url: record.original_news_canonical_url, + tts_service: record.tts_service, + tts_voice_identifier: record.tts_voice_identifier, + full_text: record.full_text, + maybe_result_url: record.maybe_result_url, + maybe_filesystem_relative_location: record.maybe_filesystem_relative_location, + news_story_production_status: record.news_story_production_status, + } + }) + .collect::>(); + + Ok(records) +} + +struct RawInternalNewsStoryPreproduction { + id: i64, + news_story_token: NewsStoryToken, + original_news_canonical_url: String, + tts_service: String, + tts_voice_identifier: TtsModelToken, + full_text: String, + maybe_result_url: Option, + maybe_filesystem_relative_location: Option, + news_story_production_status: ChatbotJobStatus, +} diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/mod.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/mod.rs new file mode 100644 index 0000000000..e7c3d2d30e --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/mod.rs @@ -0,0 +1,3 @@ +pub mod insert_news_story_preproduction; +pub mod list_news_story_preproductions; +pub mod update_news_story_preproductions; \ No newline at end of file diff --git a/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/update_news_story_preproductions.rs b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/update_news_story_preproductions.rs new file mode 100644 index 0000000000..c51f4545cf --- /dev/null +++ b/crates/lib/sqlite_queries/src/queries/by_table/news_story_preproductions/update_news_story_preproductions.rs @@ -0,0 +1,38 @@ +use enums::by_table::web_scraping_targets::web_content_type::WebContentType; +use errors::{anyhow, AnyhowResult}; +use sqlx::SqlitePool; +use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus; +use enums::common::sqlite::chatbot_job_status::ChatbotJobStatus; +use tokens::tokens::news_stories::NewsStoryToken; + +pub struct Args <'a> { + pub news_story_token: &'a NewsStoryToken, + + pub news_story_production_status: ChatbotJobStatus, + + pub sqlite_pool: &'a SqlitePool, +} + +pub async fn update_news_story_preproductions(args: Args<'_>) -> AnyhowResult<()> { + let news_story_production_status= args.news_story_production_status.to_str().to_string(); + let query = sqlx::query!( + r#" +UPDATE news_story_preproductions +SET + news_story_production_status = ?, + version = version + 1 +WHERE + news_story_token = ? + "#, + news_story_production_status, + args.news_story_token, + ); + + let query_result = query.execute(args.sqlite_pool) + .await; + + match query_result { + Ok(_) => Ok(()), + Err(err) => Err(anyhow!("error updating: {:?}", err)), + } +} diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/down.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/down.sql new file mode 100644 index 0000000000..1dbdf341ee --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE tts_render_targets; diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/up.sql new file mode 100644 index 0000000000..da3f230c63 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-073239_tts_render_targets/up.sql @@ -0,0 +1,42 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: These tables are designed for sqlite, not MySQL! + +CREATE TABLE tts_render_targets ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + + -- If the source came from somewhere, this is it + -- maybe_canonical_url TEXT NOT NULL UNIQUE, + + -- The name of the service used to generate TTS. + -- I expect this will be FakeYou for a long time, but perhaps that will one day change. + -- Enum-like value, eg. "fakeyou". + tts_service TEXT NOT NULL, + + -- The identifier of the voice. For FakeYou, this is the tts_token. + tts_voice_identifier TEXT NOT NULL, + + -- Original text of the TTS result + full_text TEXT NOT NULL, + + -- For audio results, the URL to the result. + maybe_result_url TEXT, + + -- For audio results that get downloaded, they live on the filesystem here + -- This is relative to the runtime data directory. + maybe_filesystem_relative_location TEXT, + + -- TTS render status: "new", "processing", "failed", "permanently_failed", "success", "skipped" + tts_render_status TEXT DEFAULT "new" NOT NULL, + tts_render_attempts INT DEFAULT 0 NOT NULL, + + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + -- fetched_at DATETIME DEFAULT NULL, + + version INT DEFAULT 0 NOT NULL +); + +CREATE INDEX index_tts_render_status ON tts_render_targets(tts_render_status); diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/down.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/down.sql new file mode 100644 index 0000000000..a5444cd7e0 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE news_story_preproductions; diff --git a/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/up.sql b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/up.sql new file mode 100644 index 0000000000..cd4afe8911 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/_sql/migrations/2023-02-10-090057_news_story_preproductions/up.sql @@ -0,0 +1,47 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE news_story_preproductions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + + -- TODO: Generic "story_token" + -- Entropic token that follows a single news story through the phases of production + -- This isn't a primary key that belongs to this table alone: it tracks several 1:1 and 1:n entities. + news_story_token TEXT NOT NULL UNIQUE, + + -- The original URL for the news item + original_news_canonical_url TEXT NOT NULL UNIQUE, + + -- If the source came from somewhere, this is it + -- maybe_canonical_url TEXT NOT NULL UNIQUE, + + -- The name of the service used to generate TTS. + -- I expect this will be FakeYou for a long time, but perhaps that will one day change. + -- Enum-like value, eg. "fakeyou". + tts_service TEXT NOT NULL, + + -- The identifier of the voice. For FakeYou, this is the tts_token. + tts_voice_identifier TEXT NOT NULL, + + -- Original text of the TTS result + full_text TEXT NOT NULL, + + -- For audio results, the URL to the result. + maybe_result_url TEXT, + + -- For audio results that get downloaded, they live on the filesystem here + -- This is relative to the runtime data directory. + maybe_filesystem_relative_location TEXT, + + -- TTS render status: "new", "processing", "failed", "permanently_failed", "success", "skipped" + news_story_production_status TEXT DEFAULT "new" NOT NULL, + + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + -- fetched_at DATETIME DEFAULT NULL, + + version INT DEFAULT 0 NOT NULL +); + +CREATE INDEX index_news_story_production_status ON news_story_preproductions(news_story_production_status);