mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-05 01:38:44 +00:00
Cleanup a lot of old junk; make repo more discoverable
This commit is contained in:
Brandon Thomas committed
1 parent
d9dcb76c63
commit
093525d04f
154 files changed
-293551
No files matched your search
@@ -72,9 +72,6 @@ members = [
|
||||
#"crates/lib/api_clients/fakeyou_client",
|
||||
#"crates/lib/sqlite_queries",
|
||||
|
||||
# Programs: Desktop apps
|
||||
#"crates/programs/aichatbot-sidecar",
|
||||
|
||||
# Services: API servers, jobs, workers, etc.
|
||||
"crates/service/job/analytics_job",
|
||||
"crates/service/job/download_job",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE twitch_event_match_logs;
|
||||
@@ -1,23 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- Log any event matching (bits, channel points, etc.)
|
||||
CREATE TABLE twitch_event_match_logs(
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Type of event, eg.
|
||||
event_type VARCHAR(32) NOT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
deleted_at TIMESTAMP DEFAULT NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -1,9 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE paypal_payment_events;
|
||||
DROP TABLE stripe_payment_events;
|
||||
DROP TABLE credit_spend_events;
|
||||
DROP TABLE credit_refill_events;
|
||||
DROP TABLE credit_balances;
|
||||
@@ -1,253 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- Current credit balances
|
||||
CREATE TABLE credit_balances (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- The type of credit
|
||||
-- Each credit type is its own logical "bucket" that cannot be mixed with the other types.
|
||||
-- eg. "image_generation"
|
||||
-- We'll have a composite unique key on this, so there can only be one balance per user per type.
|
||||
credit_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user
|
||||
-- We'll have a composite unique key on this, so there can only be one balance per user per type.
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== BALANCE INFO ==========
|
||||
|
||||
-- The credit balance
|
||||
-- (for the user within this credit_type bucket)
|
||||
-- Maximum value signed: 4,294,967,295
|
||||
balance INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== VECTOR CLOCK ==========
|
||||
|
||||
-- Vector clock incremented on update
|
||||
version INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
last_refilled_at TIMESTAMP DEFAULT NULL,
|
||||
last_spent_at TIMESTAMP DEFAULT NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (credit_type, user_token),
|
||||
KEY index_credit_type (credit_type),
|
||||
KEY fk_user_token (user_token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
|
||||
-- Record of credit refill (eg. purchase) events, some of which may be free
|
||||
-- At time of record creation, these are considered "already processed" and reflected in the "user_balances" table.
|
||||
-- For all intents and purposes, this is just a historical ledger of changes to active balance.
|
||||
CREATE TABLE credit_refill_events (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key"
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- The type of credit
|
||||
-- Each credit type is its own logical "bucket" that cannot be mixed with the other types.
|
||||
-- eg. "image_generation"
|
||||
credit_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user (not unique!)
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== REFILL INFO ==========
|
||||
|
||||
-- If we filled the balance for free, this will be set to true.
|
||||
-- These may be rewards for users, subscriptions that refill buckets, etc.
|
||||
is_free_event BOOL NOT NULL DEFAULT false,
|
||||
|
||||
-- Added funds
|
||||
-- Maximum value signed: 4,294,967,295
|
||||
amount_added INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- What the balance should be after the funds are added
|
||||
-- This is not the source of truth and is only used for debugging.
|
||||
balance_after_added INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- Where the balance increase came from.
|
||||
-- eg. "stripe", "paypal", "coinbase", "free", etc.
|
||||
funds_source VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== ABUSE TRACKING ==========
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
maybe_creator_ip_address VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY index_credit_type_and_user_token (credit_type, user_token),
|
||||
KEY index_credit_type (credit_type),
|
||||
KEY fk_user_token (user_token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
|
||||
-- Record of credit use events (some of which may be zero-credit "free" uses)
|
||||
-- At time of record creation, these are considered "already processed" and reflected in the "user_balances" table.
|
||||
-- For all intents and purposes, this is just a historical ledger of changes to active balance.
|
||||
CREATE TABLE credit_spend_events (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key"
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- The type of credit.
|
||||
-- Each credit type is its own logical "bucket" that cannot be mixed with the other types.
|
||||
-- eg. "image_generation"
|
||||
credit_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user (not unique!)
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== SPEND INFO ==========
|
||||
|
||||
-- If we processed the transaction for free, this will be set to true.
|
||||
-- These may be rewards for users, subscriptions that refill, etc.
|
||||
is_free_event BOOL NOT NULL DEFAULT false,
|
||||
|
||||
-- Subtracted funds
|
||||
-- Maximum value signed: 4,294,967,295
|
||||
amount_subtracted INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- What the balance should be after the funds are subtracted
|
||||
-- This is not the source of truth and is only used for debugging.
|
||||
balance_after_subtracted INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- Where the funds were spent. Typically 1:1 with "credit_type".
|
||||
-- eg. "stable_diffusion", "voice_commission", etc.
|
||||
spent_on VARCHAR(32) NOT NULL,
|
||||
|
||||
-- If we can link it to whatever was purchased, do so.
|
||||
-- If it was a job run, it's set here.
|
||||
maybe_spent_on_job_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- If we can link it to whatever was purchased, do so.
|
||||
-- We may not have the entity token if the workload was async, but we can update this record later.
|
||||
maybe_spent_on_entity_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== ABUSE TRACKING ==========
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
maybe_creator_ip_address VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY index_credit_type_and_user_token (credit_type, user_token),
|
||||
KEY index_credit_type (credit_type),
|
||||
KEY fk_user_token (user_token),
|
||||
KEY fk_maybe_spent_on_job_token (maybe_spent_on_job_token),
|
||||
KEY fk_maybe_spent_on_entity_token (maybe_spent_on_entity_token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
|
||||
-- Stripe payments
|
||||
CREATE TABLE stripe_payment_events (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key"
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user (not unique!)
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== PAYMENT INFO ==========
|
||||
|
||||
-- Potentially an internal event token (eg. if this goes to a balance, not a subscription)
|
||||
maybe_balance_refill_event_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- The stripe payment ID
|
||||
maybe_payment_id VARCHAR(64) NOT NULL,
|
||||
|
||||
-- The stripe customer ID
|
||||
maybe_customer_id VARCHAR(64) NOT NULL,
|
||||
|
||||
-- ========== ABUSE TRACKING ==========
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
maybe_creator_ip_address VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY fk_user_token (user_token),
|
||||
KEY fk_maybe_balance_refill_event_token (maybe_balance_refill_event_token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
-- Paypal payments
|
||||
CREATE TABLE paypal_payment_events (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key"
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user (not unique!)
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== PAYMENT INFO ==========
|
||||
|
||||
-- Potentially an internal event token (eg. if this goes to a balance, not a subscription)
|
||||
maybe_balance_refill_event_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- The paypal payment ID
|
||||
maybe_payment_id VARCHAR(64) NOT NULL,
|
||||
|
||||
-- The paypal customer ID
|
||||
maybe_customer_id VARCHAR(64) NOT NULL,
|
||||
|
||||
-- ========== ABUSE TRACKING ==========
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
maybe_creator_ip_address VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY fk_user_token (user_token),
|
||||
KEY fk_maybe_balance_refill_event_token (maybe_balance_refill_event_token)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE generic_workloads;
|
||||
@@ -1,108 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE generic_workloads (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Idempotency token
|
||||
-- This is so the frontend client doesn't submit duplicate jobs.
|
||||
uuid_idempotency_token VARCHAR(36) NOT NULL,
|
||||
|
||||
-- ========== SUCCESS CASE ==========
|
||||
|
||||
-- If the job completes successfully, this is the result token.
|
||||
-- This is only populated on a successful result.
|
||||
on_success_finished_entity_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- The type of the object will vary based on the type of the workload,
|
||||
-- and we may include heuristics that auto-detect types in the future
|
||||
on_success_finished_entity_type VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== WORKLOAD DETAILS ==========
|
||||
|
||||
-- A "well defined" type of workload
|
||||
-- The first workload type will be "image_generation_request"
|
||||
workload_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- JSON serialized details about the workload
|
||||
-- The schema varies and is controlled by the "workload_type"
|
||||
-- TEXT – 64KB (65,535 characters); TEXT also requires 2 bytes overhead.
|
||||
workload_details TEXT,
|
||||
|
||||
-- ========== CREATOR DETAILS AND PREFERENCES ==========
|
||||
|
||||
-- Foreign key to user
|
||||
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Based on a cookie sent by the frontend.
|
||||
-- We'll save this even if the user is logged in.
|
||||
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
-- The creator can set a desired visibility for their data.
|
||||
-- This does not always apply to every workload type.
|
||||
-- Additionally, some workload types may require moderator approval prior
|
||||
-- to being publicly listed, and this field has no bearing on that.
|
||||
-- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- ========== JOB SYSTEM DETAILS ==========
|
||||
|
||||
-- Jobs begin as "pending", then transition to other states.
|
||||
--
|
||||
-- * Pending = job is ready to go
|
||||
-- * Started = job is running
|
||||
-- * Complete_Success = job is done (success)
|
||||
-- * Complete_Failure = job is done (failure)
|
||||
-- * Attempt_Failed = job failed but may retry.
|
||||
-- * Dead = job failed permanently.
|
||||
--
|
||||
-- Pending -> Started -> Complete_Success
|
||||
-- |-> Complete_Failure
|
||||
-- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
|
||||
status ENUM(
|
||||
'pending',
|
||||
'started',
|
||||
'complete_success',
|
||||
'complete_failure',
|
||||
'attempt_failed',
|
||||
'dead') NOT NULL DEFAULT 'pending',
|
||||
|
||||
-- We can track this against a "max_attempt_count"
|
||||
attempt_count INT(3) NOT NULL DEFAULT 0,
|
||||
|
||||
-- If there is a failure, tell the user why.
|
||||
failure_reason VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Failed jobs will set a next attempt time.
|
||||
-- Subsequent tries can increase the timeout.
|
||||
-- Failures because of permissions require human intervention => [retry_at=null].
|
||||
-- Failures because of invalid files are dead => [status=dead].
|
||||
retry_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (uuid_idempotency_token),
|
||||
KEY index_on_workload_type (workload_type),
|
||||
KEY fk_maybe_creator_user_token (maybe_creator_user_token),
|
||||
KEY index_status (status),
|
||||
KEY index_creator_ip_address (creator_ip_address)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -1,7 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE generated_images;
|
||||
DROP TABLE generated_image_batches;
|
||||
DROP TABLE generated_image_request_logs;
|
||||
@@ -1,111 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- Sometimes we'll block image generation requests.
|
||||
-- This is a log of *everything* that gets sent to us so that we can track abuse
|
||||
-- of the system and identify bad actors.
|
||||
CREATE TABLE generated_image_request_logs(
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== REQUEST DETAILS ==========
|
||||
|
||||
-- The text prompt
|
||||
text_prompt VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
-- If the request consumed credits
|
||||
request_was_paid BOOL NOT NULL DEFAULT false,
|
||||
|
||||
-- If we determined the request was NSFW
|
||||
request_was_nsfw BOOL NOT NULL DEFAULT false,
|
||||
|
||||
-- If we determined we should block the request upfront
|
||||
request_was_blocked BOOL NOT NULL DEFAULT false,
|
||||
|
||||
-- ========== CREATOR DETAILS ==========
|
||||
|
||||
-- Foreign key to user
|
||||
-- If no user is logged in, this is null.
|
||||
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Based on a cookie sent by the frontend.
|
||||
-- We'll save this even if the user is logged in.
|
||||
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
-- Each image generation request will produce multiple images.
|
||||
-- This is the parent record for multiple images.
|
||||
CREATE TABLE generated_image_batches (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== CREATOR DETAILS ==========
|
||||
|
||||
-- Foreign key to user
|
||||
-- If no user is logged in, this is null.
|
||||
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Based on a cookie sent by the frontend.
|
||||
-- We'll save this even if the user is logged in.
|
||||
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
-- This is the child record for single images within a generated batch
|
||||
CREATE TABLE generated_images (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Reference to the batch this image belongs to.
|
||||
batch_token VARCHAR(32) NOT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE voice_survey_answers;
|
||||
@@ -1,53 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- Voice survey answers
|
||||
CREATE TABLE voice_survey_answers (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Idempotency token from client
|
||||
-- This is so the frontend client doesn't submit duplicate items.
|
||||
uuid_idempotency_token VARCHAR(36) NOT NULL,
|
||||
|
||||
-- User making the comment
|
||||
maybe_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- The voice that is being requested
|
||||
voice_name VARCHAR(128) NOT NULL,
|
||||
|
||||
-- If the person would pay for FakeYou premium
|
||||
would_pay BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- How much the user would hypothetically pay per month
|
||||
would_pay_amount DOUBLE DEFAULT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
-- ========== VECTOR CLOCK ==========
|
||||
|
||||
-- Incremented with every update.
|
||||
version INT NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (uuid_idempotency_token),
|
||||
KEY index_maybe_user_token (maybe_user_token),
|
||||
KEY index_voice_name (voice_name),
|
||||
KEY index_would_pay (would_pay),
|
||||
KEY index_would_pay_amount (would_pay_amount),
|
||||
KEY index_created_at (created_at)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,93 +0,0 @@
|
||||
Copyright (c) 2012-2015, The Mozilla Foundation and Telefonica S.A.
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
http://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,3 +0,0 @@
|
||||
# This is only set to run database migrations on Windows
|
||||
DATABASE_URL=sqlite://../../../runtime_data/database.db
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
[package]
|
||||
name = "aichatbot-sidecar"
|
||||
publish = false
|
||||
edition = "2021"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
|
||||
[[bin]]
|
||||
name = "aichatbot-sidecar"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
actix_helpers = { path = "../../lib/actix_helpers" }
|
||||
concurrency = { path = "../../lib/concurrency" }
|
||||
easyenv = { path = "../../lib/easyenv" }
|
||||
enums = { path = "../../_types/enums" }
|
||||
errors = { path = "../../lib/errors" }
|
||||
fakeyou_client = { path = "../../lib/api_clients/fakeyou_client" }
|
||||
filesys = { path = "../../lib/files/filesys" }
|
||||
hashing = { path = "../../lib/files/hashing" }
|
||||
http_server_common = { path = "../../lib/deprecated/http_server_common" }
|
||||
idempotency = { path = "../../lib/idempotency" }
|
||||
media = { path = "../../lib/files/media" }
|
||||
sqlite_queries = { path = "../../lib/sqlite_queries" }
|
||||
tokens = { path = "../../_types/tokens" }
|
||||
web_scrapers = { path = "../../lib/web_scrapers" }
|
||||
|
||||
# External
|
||||
base64 = "0.21.0"
|
||||
chrono = { version = "0.4.22", features = ["serde"] }
|
||||
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
|
||||
dotenv = "0.15.0"
|
||||
log = "0.4.14"
|
||||
once_cell = "1.9.0"
|
||||
path-absolutize = "3.0.14" # Absolute paths have not stabilized in Rust yet
|
||||
rand = "0.8.3"
|
||||
regex = "1.5.4"
|
||||
strum = { version = "0.24", features = ["derive"] } # Fancy enum iteration, etc.
|
||||
|
||||
sqlx = { version = "0.7.2", features = [ "sqlite", "runtime-tokio-native-tls", "chrono" ] }
|
||||
|
||||
# Async Runtime
|
||||
tokio = { version = "1.6.0", features = ["macros"] }
|
||||
|
||||
# Content downloads
|
||||
reqwest = { version = "0.11", features = ["json"] } # HTTP Client
|
||||
rss = "2.0.2"
|
||||
|
||||
# Webserver
|
||||
actix-cors = "0.6.0-beta.3"
|
||||
actix-files= "0.6.0-beta.8"
|
||||
actix-http = "3.0.0-beta.11"
|
||||
actix-multipart = "0.4.0-beta.7"
|
||||
actix-rt = "2.4.0"
|
||||
actix-service = "2.0.1"
|
||||
actix-utils = "3.0.0"
|
||||
actix-web = "4.0.0-beta.10"
|
||||
actix-web-actors = "4.0.0-beta.7"
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
serde_derive = "1.0.125"
|
||||
serde_json = "1.0.64"
|
||||
serde_yaml = "0.9.1"
|
||||
|
||||
# Gui (Egui)
|
||||
eframe = "0.20.1"
|
||||
egui = "0.20.1"
|
||||
|
||||
# API integrations
|
||||
async-openai = "0.6.0"
|
||||
@@ -1,54 +0,0 @@
|
||||
aichatbot-sidecar (Sidecar companion to AiChatBot)
|
||||
=================
|
||||
|
||||
This is the first app that doesn't run as a part of our SoA API architecture. (Though it is written such that we
|
||||
could port it.) This app is responsible for generating all the content that goes into our streaming AI-powered TV
|
||||
programs.
|
||||
|
||||
### Installation on Windows
|
||||
|
||||
The application uses sqlite, which is tricky to install on Windows.
|
||||
|
||||
1. Put together an sqlite build directory containing the sources and dll file. One is checked in under this monorepo
|
||||
in `./windows/sqlite` at the time of this writing, though that may change in the future.
|
||||
|
||||
|
||||
2. Add environment variables for the sqlite build directory (see https://github.com/diesel-rs/diesel/issues/487)
|
||||
|
||||
* PATH (append the sqlite build directory to the end)
|
||||
* SQLITE3_LIB_DIR (a new environment variable; set it to the sqllite build path)
|
||||
|
||||
|
||||
3. Build the `sqlite3.lib` file with the following:
|
||||
|
||||
```
|
||||
lib /def:sqlite3.def /out:sqlite3.lib /MACHINE:amd64
|
||||
```
|
||||
|
||||
This must be done in "Developer Command Prompt for VS 2022" (or similar), not `cmd` or `PowerShell`.
|
||||
|
||||
|
||||
4. Run the diesel_cli tool build
|
||||
```
|
||||
cargo install diesel_cli --no-default-features --features sqlite
|
||||
```
|
||||
|
||||
If it still complains about the `sqlite3.lib` file not being found, be sure to double check that `echo %PATH%` shows
|
||||
the updated PATH from the steps above.
|
||||
|
||||
|
||||
### Installation on Linux
|
||||
|
||||
The egui/eframe libraries require the installation of a metric ton of dependencies, so have at it:
|
||||
|
||||
```
|
||||
sudo apt-get install \
|
||||
cmake \
|
||||
libfontconfig1-dev \
|
||||
libspeechd-dev \
|
||||
libssl-dev \
|
||||
libxcb-render0-dev \
|
||||
libxcb-shape0-dev \
|
||||
libxcb-xfixes0-dev \
|
||||
libxkbcommon-dev
|
||||
```
|
||||
Whitespace-only changes.
-5
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE web_scraping_targets;
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- NB: These tables are designed for sqlite, not MySQL!
|
||||
|
||||
CREATE TABLE web_scraping_targets (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
-- Canonical URLs help us avoid aliases of the same content.
|
||||
-- We'll need to figure this out for each site.
|
||||
canonical_url TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
|
||||
web_content_type TEXT NOT NULL,
|
||||
|
||||
-- Optional title of the target, which might help downstream.
|
||||
maybe_title TEXT,
|
||||
|
||||
-- A full image will only be set at this stage if the RSS feed or HTML specifies one.
|
||||
-- This can be useful downstream during the actual scraping.
|
||||
maybe_article_full_image_url TEXT,
|
||||
|
||||
-- A thumbnail will only be set at this stage if the RSS feed or HTML specifies one.
|
||||
-- This can be useful downstream during the actual scraping.
|
||||
maybe_article_thumbnail_image_url TEXT,
|
||||
|
||||
-- If we skip the target (a priori or after scraping), this is why.
|
||||
-- Enum-like value, eg. "predominantly_video", "empty", "filtered_topic", "nobody_cares", etc.
|
||||
maybe_skip_reason TEXT,
|
||||
|
||||
-- Scraping status: "new", "failed", "permanently_failed", "success", "skipped"
|
||||
scraping_status TEXT DEFAULT "new" NOT NULL,
|
||||
scrape_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
-- fetched_at DATETIME DEFAULT NULL,
|
||||
|
||||
version INT DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX index_scraping_status ON web_scraping_targets(scraping_status);
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE tts_render_tasks;
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- NB: These tables are designed for sqlite, not MySQL!
|
||||
|
||||
CREATE TABLE tts_render_tasks (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
-- Effective primary key.
|
||||
-- Follows a single TTS audio through the phases of rendering
|
||||
token TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- Composite primary key for what this TTS render job belongs to (eg. news story)
|
||||
story_type TEXT NOT NULL,
|
||||
story_token TEXT NOT NULL,
|
||||
|
||||
-- For each "sequence" of audio, this is the order.
|
||||
-- This is 1-indexed
|
||||
sequence_order INTEGER NOT NULL,
|
||||
|
||||
-- The total number of audio files in the entire speech.
|
||||
sequence_length INTEGER NOT NULL,
|
||||
|
||||
-- If the source came from somewhere, this is it
|
||||
-- maybe_canonical_url TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- The name of the service used to generate TTS.
|
||||
-- I expect this will be FakeYou for a long time, but perhaps that will one day change.
|
||||
-- Enum-like value, eg. "fakeyou".
|
||||
tts_service TEXT NOT NULL,
|
||||
|
||||
-- The identifier of the voice. For FakeYou, this is the tts_token.
|
||||
tts_voice_identifier TEXT NOT NULL,
|
||||
|
||||
-- Original text of the TTS result
|
||||
full_text TEXT NOT NULL,
|
||||
|
||||
-- For pending audio results, this is the job token to check back later.
|
||||
maybe_inference_job_token TEXT,
|
||||
|
||||
-- For finished audio results, the token of the result.
|
||||
maybe_result_token TEXT,
|
||||
|
||||
-- For finished audio results, the URL to the result.
|
||||
maybe_result_url TEXT,
|
||||
|
||||
-- For audio results that get downloaded, they live on the filesystem here
|
||||
-- This is relative to the runtime data directory.
|
||||
maybe_result_relative_filesystem_location TEXT,
|
||||
|
||||
-- When the file is downloaded, the duration in milliseconds.
|
||||
maybe_audio_duration_millis INTEGER,
|
||||
|
||||
-- TTS render status: "new", "processing", "failed", "permanently_failed", "success", "skipped"
|
||||
tts_render_status TEXT DEFAULT "new" NOT NULL,
|
||||
tts_render_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
-- fetched_at DATETIME DEFAULT NULL,
|
||||
|
||||
version INT DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX index_tts_render_status ON tts_render_tasks(tts_render_status);
|
||||
CREATE INDEX index_tts_render_foreign_key ON tts_render_tasks(story_type, story_token);
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE news_story_productions;
|
||||
-80
@@ -1,80 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE news_story_productions (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
-- Entropic token that follows a single news story through the phases of production
|
||||
-- This isn't a primary key that belongs to this table alone: it tracks several 1:1 and 1:n entities.
|
||||
news_story_token TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- The original URL for the news item
|
||||
original_news_canonical_url TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
|
||||
web_content_type TEXT NOT NULL,
|
||||
|
||||
-- Original article title (not LLM renditioned)
|
||||
original_news_title TEXT NOT NULL,
|
||||
|
||||
-- LLM renditioned news title, short. Under 60 characters.
|
||||
maybe_summary_news_title TEXT,
|
||||
|
||||
-- Comma-separated categories for the story. Generated by LLM.
|
||||
maybe_categorization TEXT,
|
||||
|
||||
-- If we skip the production at any stage (after scraping, after rendition, etc.), for a non-failure
|
||||
-- reason, this is why.
|
||||
-- Enum-like value, eg. "predominantly_video", "empty", "world_leader", "nobody_cares", "advertisement", etc.
|
||||
maybe_skip_reason TEXT,
|
||||
|
||||
-- The overall status of the news story.
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
overall_production_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Renditioning the audio with LLM/GPT
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
llm_rendition_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Number of LLM/GPT rendition attempts that have been made
|
||||
llm_rendition_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
-- Summarizing the title with LLM/GPT
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
llm_title_summary_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Number of LLM/GPT title summary attempts that have been made
|
||||
llm_title_summary_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
-- Classifying the categories with LLM/GPT
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
llm_categorization_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Number of LLM/GPT category summary attempts that have been made
|
||||
llm_categorization_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
-- TTS for the audio with FakeYou
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
audio_generation_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Generating headline images
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
image_generation_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
-- Number of image generation attempts that have been made
|
||||
image_generation_attempts INT DEFAULT 0 NOT NULL,
|
||||
|
||||
-- Generating extra media (images, etc.)
|
||||
-- "not_ready", "ready_waiting", "processing", "retryably_failed", "permanently_failed", "skipped", "done"
|
||||
-- media_generation_status TEXT DEFAULT "not_ready" NOT NULL,
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
|
||||
version INT DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX index_overall_production_status ON news_story_productions(overall_production_status);
|
||||
CREATE INDEX index_llm_rendition_status ON news_story_productions(llm_rendition_status);
|
||||
CREATE INDEX index_audio_generation_status ON news_story_productions(audio_generation_status);
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE news_stories;
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE news_stories (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
-- Entropic token that follows a single news story through the phases of production
|
||||
-- This isn't a primary key that belongs to this table alone: it tracks several 1:1 and 1:n entities.
|
||||
news_story_token TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- The original URL for the news item
|
||||
original_news_canonical_url TEXT NOT NULL UNIQUE,
|
||||
|
||||
-- Effectively an enum-like: CnnArticle, SlashdotArticle, HackerNewsThread, etc.
|
||||
web_content_type TEXT NOT NULL,
|
||||
|
||||
-- Original article title (not LLM renditioned)
|
||||
original_news_title TEXT NOT NULL,
|
||||
|
||||
-- LLM renditioned news title, short. Under 60 characters.
|
||||
summary_news_title TEXT NOT NULL,
|
||||
|
||||
-- Comma-separated categories for the story. Generated by LLM.
|
||||
llm_categorization TEXT NOT NULL,
|
||||
|
||||
-- Total number of discrete audio files for this story
|
||||
audio_file_count INTEGER NOT NULL,
|
||||
|
||||
-- Total amount of time in the audio files.
|
||||
-- NB: This is seconds, not millis!
|
||||
audio_total_duration_seconds INTEGER NOT NULL,
|
||||
|
||||
-- When the story should stop being scheduled.
|
||||
replayable_until DATETIME NOT NULL,
|
||||
|
||||
-- A kill switch for news stories.
|
||||
is_playable BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
-- fetched_at DATETIME DEFAULT NULL,
|
||||
|
||||
version INT DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX index_replayable_until ON news_stories(replayable_until);
|
||||
CREATE INDEX index_is_playable ON news_stories(is_playable);
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE news_story_schedule_histories;
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE news_story_schedule_histories (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
-- We can have many entries for the same news story.
|
||||
news_story_token TEXT NOT NULL,
|
||||
|
||||
-- Records when the story was played.
|
||||
played_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX index_news_story_token ON news_story_schedule_histories(news_story_token);
|
||||
CREATE INDEX played_at ON news_story_schedule_histories(played_at);
|
||||
-1
@@ -1 +0,0 @@
|
||||
-- This file should undo anything in `up.sql`
|
||||
-1
@@ -1 +0,0 @@
|
||||
-- Your SQL goes here
|
||||
@@ -1,13 +0,0 @@
|
||||
# For documentation on how to configure this file,
|
||||
# see diesel.rs/guides/configuring-diesel-cli
|
||||
|
||||
[print_schema]
|
||||
file = "_ignored_schema.rs"
|
||||
|
||||
# I want to sort this directory first for easier reference
|
||||
# Not released yet, but added here:
|
||||
# https://github.com/diesel-rs/diesel/pull/2179
|
||||
# You'll need to install the a non-released diesel CLI
|
||||
# (See storyteller READMEs/docs on how to install)
|
||||
[migrations_directory]
|
||||
dir = "_sql/migrations"
|
||||
@@ -1,54 +0,0 @@
|
||||
use strum::EnumIter;
|
||||
use strum::IntoEnumIterator;
|
||||
use tokens::tokens::tts_models::TtsModelToken;
|
||||
|
||||
/// Static configurations for usable voices.
|
||||
/// These will be well-known character voices, test voices, etc.
|
||||
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
|
||||
#[derive(EnumIter)]
|
||||
pub enum FakeYouVoiceOption {
|
||||
AndersonCooper,
|
||||
HanashiV3,
|
||||
HanashiV2,
|
||||
// NB: Hanashi v1 has strokes.
|
||||
HanashiV1,
|
||||
// NB: John Madden is fantastic.
|
||||
JohnMadden,
|
||||
}
|
||||
|
||||
impl Default for FakeYouVoiceOption {
|
||||
fn default() -> Self {
|
||||
Self::AndersonCooper
|
||||
}
|
||||
}
|
||||
|
||||
impl FakeYouVoiceOption {
|
||||
|
||||
pub fn variant_name(&self) -> &'static str {
|
||||
match self {
|
||||
FakeYouVoiceOption::AndersonCooper => "Anderson Cooper",
|
||||
FakeYouVoiceOption::HanashiV3 => "Hanashi (v3)",
|
||||
FakeYouVoiceOption::HanashiV2 => "Hanashi (v2)",
|
||||
FakeYouVoiceOption::HanashiV1 => "Hanashi (v1)",
|
||||
FakeYouVoiceOption::JohnMadden => "John Madden",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tts_model_token_str(&self) -> &'static str {
|
||||
match self {
|
||||
FakeYouVoiceOption::AndersonCooper => "TM:5mk6h46emvhk",
|
||||
FakeYouVoiceOption::HanashiV3 => "TM:5hc38cg7cdtm",
|
||||
FakeYouVoiceOption::HanashiV2 => "TM:zcd4qzrwacq0",
|
||||
FakeYouVoiceOption::HanashiV1 => "TM:npxv6rgtddmv",
|
||||
FakeYouVoiceOption::JohnMadden => "TM:30ha2t6bxfn4",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tts_model_token(&self) -> TtsModelToken {
|
||||
TtsModelToken::new_from_str(self.tts_model_token_str())
|
||||
}
|
||||
|
||||
pub fn iterate() -> FakeYouVoiceOptionIter {
|
||||
Self::iter()
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
use strum::EnumIter;
|
||||
use strum::IntoEnumIterator;
|
||||
|
||||
/// Static configurations for which levels the Unreal Engine program can present.
|
||||
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
|
||||
#[derive(EnumIter)]
|
||||
pub enum LevelOption {
|
||||
BlankBlue,
|
||||
BlankRed,
|
||||
FakeCnn,
|
||||
FantasyStorytelling,
|
||||
CartoonNewsStation,
|
||||
VirtualMusician,
|
||||
AiUpscaledDeepFake,
|
||||
}
|
||||
|
||||
impl Default for LevelOption {
|
||||
fn default() -> Self {
|
||||
Self::FakeCnn
|
||||
}
|
||||
}
|
||||
|
||||
impl LevelOption {
|
||||
pub fn variant_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::BlankBlue => "Blank (Blue)",
|
||||
Self::BlankRed => "Blank (Red)",
|
||||
Self::FakeCnn => "Fake CNN",
|
||||
Self::FantasyStorytelling => "Fantasy Storytelling",
|
||||
Self::CartoonNewsStation => "Cartoon News Station",
|
||||
Self::VirtualMusician => "Virtual Musician",
|
||||
Self::AiUpscaledDeepFake => "AI Upscaled Deep Fake (Unreal -> DeepFaceLive)",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tts_model_token_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::BlankBlue => "blank_blue",
|
||||
Self::BlankRed => "blank_red",
|
||||
Self::FakeCnn => "fake_cnn",
|
||||
Self::CartoonNewsStation => "cartoon_news_station",
|
||||
Self::FantasyStorytelling => "fantasy_storytelling",
|
||||
Self::VirtualMusician => "virtual_musician",
|
||||
Self::AiUpscaledDeepFake => "ai_upscaled_deep_fake",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iterate() -> LevelOptionIter {
|
||||
Self::iter()
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod fakeyou_voice_option;
|
||||
pub mod level_option;
|
||||
@@ -1,23 +0,0 @@
|
||||
|
||||
pub struct CategorizationGptPrompt {
|
||||
// TODO: Add other details (title, etc.)
|
||||
paragraphs: Vec<String>,
|
||||
}
|
||||
|
||||
impl CategorizationGptPrompt {
|
||||
|
||||
pub fn new(paragraphs: &Vec<String>) -> Self {
|
||||
Self {
|
||||
paragraphs: paragraphs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_prompt(&self) -> String {
|
||||
// TODO: This will need a lot of work and experimentation.
|
||||
let start = "Come up with a comma separated list of keywords or concepts for this news article:\n\n-----\n\n";
|
||||
|
||||
let end = "\n\n-----\n\nThese keywords will classify the key points of the article. List the top five words and no more";
|
||||
|
||||
format!("{}{}{}", start, &self.paragraphs.join("\n\n"), end)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
|
||||
pub struct ConciseTitleGptPrompt {
|
||||
title: String,
|
||||
paragraphs: Vec<String>,
|
||||
}
|
||||
|
||||
impl ConciseTitleGptPrompt {
|
||||
|
||||
pub fn new(title: &str, paragraphs: &Vec<String>) -> Self {
|
||||
Self {
|
||||
title: title.to_string(),
|
||||
paragraphs: paragraphs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn make_prompt(&self) -> String {
|
||||
// TODO: This will need a lot of work and experimentation.
|
||||
let start = "The following is a news article:\n\n-----\n\n";
|
||||
|
||||
let end = "\n\n-----\n\nSummarize a headline news title for the article that is under 50 characters. The title must be 50 characters or less";
|
||||
|
||||
format!("{}{}\n\n{}{}", start, &self.title, &self.paragraphs.join("\n\n"), end)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
pub mod categorization_gpt_prompt;
|
||||
pub mod concise_title_gpt_prompt;
|
||||
pub mod news_article_prompt;
|
||||
@@ -1,25 +0,0 @@
|
||||
|
||||
// TODO: We'll add more types of prompt templates in the future.
|
||||
|
||||
pub struct NewsArticlePrompt {
|
||||
// TODO: Add other details (title, etc.)
|
||||
paragraphs: Vec<String>,
|
||||
}
|
||||
|
||||
impl NewsArticlePrompt {
|
||||
|
||||
pub fn new(paragraphs: &Vec<String>) -> Self {
|
||||
Self {
|
||||
paragraphs: paragraphs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: This will need a lot of work and experimentation.
|
||||
pub fn make_prompt(&self) -> String {
|
||||
let start = "The following is a news article:\n\n";
|
||||
|
||||
let end = "\n\nPlease reword the above article as if you were a newscaster presenting the news. Spell out any abbreviations and numbers";
|
||||
|
||||
format!("{}{}{}", start, &self.paragraphs.join("\n\n"), end)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use eframe::egui;
|
||||
use log::{error, info};
|
||||
|
||||
use crate::configs::fakeyou_voice_option::FakeYouVoiceOption;
|
||||
use crate::configs::level_option::LevelOption;
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
/// Initial state for the GUI
|
||||
pub struct AppGuiArgs {
|
||||
/// Where the program saves its database
|
||||
pub save_directory: String,
|
||||
|
||||
/// Shared state with the HTTP server and workers.
|
||||
pub control_state: Arc<AppControlState>,
|
||||
}
|
||||
|
||||
pub struct AppGui {
|
||||
// NB: This is not the authoritative set of state.
|
||||
// These will be copied into the shared server state.
|
||||
filename: String,
|
||||
default_directory: String,
|
||||
|
||||
is_unreal_paused : bool,
|
||||
is_scraping_paused: bool,
|
||||
is_openai_paused: bool,
|
||||
is_fakeyou_paused: bool,
|
||||
|
||||
current_level: LevelOption, // NB: Just a copy for the UI, not the authoritative copy
|
||||
fakeyou_voice: FakeYouVoiceOption, // NB: Just a copy for the UI, not the authoritative copy
|
||||
|
||||
/// Shared state with the HTTP server and workers.
|
||||
control_state: Arc<AppControlState>,
|
||||
}
|
||||
|
||||
impl AppGui {
|
||||
pub fn new(args: AppGuiArgs) -> Self {
|
||||
Self {
|
||||
filename: "".to_string(),
|
||||
default_directory: args.save_directory,
|
||||
is_unreal_paused: false,
|
||||
is_scraping_paused: false,
|
||||
is_openai_paused: false,
|
||||
is_fakeyou_paused: false,
|
||||
current_level: LevelOption::default(),
|
||||
fakeyou_voice: FakeYouVoiceOption::default(),
|
||||
control_state: args.control_state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl eframe::App for AppGui {
|
||||
fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
|
||||
//if let Err(e) = self.control_state.set_is_paused(self.is_paused) {
|
||||
// error!("Couldn't set paused state: {:?}", e);
|
||||
//}
|
||||
|
||||
self.control_state.set_is_unreal_paused(self.is_unreal_paused);
|
||||
self.control_state.set_is_scraping_paused(self.is_scraping_paused);
|
||||
self.control_state.set_is_openai_paused(self.is_openai_paused);
|
||||
self.control_state.set_is_fakeyou_paused(self.is_fakeyou_paused);
|
||||
|
||||
let _r = self.control_state.set_level(self.current_level); // NB: Ignore lock errors.
|
||||
let _r = self.control_state.set_fakeyou_voice(self.fakeyou_voice); // NB: Ignore lock errors.
|
||||
|
||||
egui::CentralPanel::default().show(ctx, |ui| {
|
||||
//ui.heading("AiChatBot Sidecar");
|
||||
//ui.horizontal(|ui| {
|
||||
// let name_label = ui.label("Filename: ");
|
||||
// ui.text_edit_singleline(&mut self.filename)
|
||||
// .labelled_by(name_label.id);
|
||||
//});
|
||||
//ui.horizontal(|ui| {
|
||||
// let name_label = ui.label("Default directory: ");
|
||||
// ui.text_edit_singleline(&mut self.default_directory)
|
||||
// .labelled_by(name_label.id);
|
||||
//});
|
||||
|
||||
//ui.horizontal(|ui| {
|
||||
// let name_label = ui.label("Is Paused?: ");
|
||||
// ui.checkbox(&mut self.is_paused, "Is Paused?")
|
||||
// .labelled_by(name_label.id);
|
||||
//});
|
||||
|
||||
ui.add_space(10.0);
|
||||
ui.heading("Unreal Level");
|
||||
ui.add_space(7.0);
|
||||
|
||||
egui::ComboBox::from_label("Control for what programming is generated and played.")
|
||||
.selected_text(format!("{:?}", self.current_level))
|
||||
.show_ui(ui, |ui| {
|
||||
for option in LevelOption::iterate() {
|
||||
ui.selectable_value(&mut self.current_level, option, option.variant_name());
|
||||
}
|
||||
});
|
||||
|
||||
ui.add_space(10.0);
|
||||
ui.heading("Sidecar Pause Controls");
|
||||
ui.add_space(7.0);
|
||||
|
||||
ui.horizontal(|ui| {
|
||||
ui.checkbox(&mut self.is_scraping_paused, "Web Scraping");
|
||||
});
|
||||
ui.horizontal(|ui| {
|
||||
ui.checkbox(&mut self.is_openai_paused, "OpenAI API");
|
||||
});
|
||||
ui.horizontal(|ui| {
|
||||
ui.checkbox(&mut self.is_fakeyou_paused, "FakeYou API");
|
||||
});
|
||||
|
||||
ui.add_space(20.0);
|
||||
ui.heading("Unreal Pause Controls");
|
||||
ui.add_space(7.0);
|
||||
ui.horizontal(|ui| {
|
||||
ui.checkbox(&mut self.is_unreal_paused, "TODO: This does not work yet.");
|
||||
});
|
||||
|
||||
ui.add_space(20.0);
|
||||
ui.heading("FakeYou Voice");
|
||||
ui.add_space(7.0);
|
||||
|
||||
egui::ComboBox::from_label("Note: once a story is generated, it will not be re-voiced!")
|
||||
.selected_text(format!("{:?}", self.fakeyou_voice))
|
||||
.show_ui(ui, |ui| {
|
||||
for option in FakeYouVoiceOption::iterate() {
|
||||
ui.selectable_value(&mut self.fakeyou_voice, option, option.variant_name());
|
||||
}
|
||||
});
|
||||
|
||||
ui.add_space(30.0);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use eframe::egui;
|
||||
use errors::AnyhowResult;
|
||||
|
||||
use crate::gui::gui::{AppGui, AppGuiArgs};
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::startup_args::StartupArgs;
|
||||
|
||||
pub fn launch_gui(startup_args: StartupArgs, control_state: Arc<AppControlState>) -> AnyhowResult<()> {
|
||||
// Log to stdout (if you run with `RUST_LOG=debug`).
|
||||
//tracing_subscriber::fmt::init();
|
||||
|
||||
// TODO: Is there a way for the program to determine size by itself? Default::default() yields a bad size too.
|
||||
let width = 440.0;
|
||||
let height = 400.0;
|
||||
|
||||
let options = eframe::NativeOptions {
|
||||
initial_window_size: Some(egui::vec2(width, height)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let app_state = AppGui::new(AppGuiArgs {
|
||||
save_directory: startup_args.save_directory,
|
||||
control_state,
|
||||
});
|
||||
|
||||
let result = eframe::run_native(
|
||||
"AiChatBot Sidecar",
|
||||
options,
|
||||
Box::new(|_cc| Box::new(app_state)),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod gui;
|
||||
pub mod launch_gui;
|
||||
@@ -1,211 +0,0 @@
|
||||
// NB: The following windows directive is cargo-culted from:
|
||||
// https://github.com/emilk/egui/blob/master/examples/hello_world/src/main.rs
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] // hide console window on Windows in release
|
||||
|
||||
#[macro_use] extern crate serde_derive;
|
||||
|
||||
use actix_web::{HttpResponse, HttpServer, web};
|
||||
use async_openai::Client;
|
||||
use clap::{App, Arg};
|
||||
use enums::common::sqlite::web_content_type::WebContentType;
|
||||
use errors::AnyhowResult;
|
||||
use fakeyou_client::credentials::FakeYouCredentials;
|
||||
use fakeyou_client::fakeyou_api_client::FakeYouApiClient;
|
||||
use log::info;
|
||||
use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
|
||||
use sqlx::sqlite::SqlitePoolOptions;
|
||||
use tokio::runtime::{Builder, Runtime};
|
||||
use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper;
|
||||
use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper;
|
||||
use workers::news_stories::phase_4::news_story_audio_final_verification::main_loop::news_story_audio_final_verification_main_loop;
|
||||
use workers::news_stories::phase_5::news_story_post_production_finalization::main_loop::news_story_post_production_finalization_main_loop;
|
||||
|
||||
use crate::gui::launch_gui::launch_gui;
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::startup_args::get_startup_args;
|
||||
use crate::web_server::launch_web_server::{launch_web_server, LaunchWebServerArgs};
|
||||
use crate::workers::audio::fakeyou_audio_create::main_loop::fakeyou_audio_create_main_loop;
|
||||
use crate::workers::audio::fakeyou_audio_download::main_loop::fakeyou_audio_download_main_loop;
|
||||
use crate::workers::news_stories::phase_1::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop;
|
||||
use crate::workers::news_stories::phase_2::news_story_image_generation::main_loop::news_story_image_generation_main_loop;
|
||||
use crate::workers::news_stories::phase_2::news_story_llm_category_summary::main_loop::news_story_llm_category_summary_main_loop;
|
||||
use crate::workers::news_stories::phase_2::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop;
|
||||
use crate::workers::news_stories::phase_2::news_story_llm_title_summary::main_loop::news_story_llm_title_summary_main_loop;
|
||||
use crate::workers::news_stories::phase_3::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop;
|
||||
use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop;
|
||||
use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save;
|
||||
use crate::workers::web::web_index_ingestion::main_loop::web_index_ingestion_main_loop;
|
||||
|
||||
pub mod configs;
|
||||
pub mod gpt_prompts;
|
||||
pub mod gui;
|
||||
pub mod persistence;
|
||||
pub mod shared_state;
|
||||
pub mod startup_args;
|
||||
pub mod test;
|
||||
pub mod web_server;
|
||||
pub mod workers;
|
||||
|
||||
pub const LOG_LEVEL: &str = concat!(
|
||||
"info,",
|
||||
"actix_web=info,",
|
||||
"symphonia_core=warn,", // Symphonia is spammy af.
|
||||
"sqlx::query=warn,", // SQLX logs all queries as "info", which is super spammy
|
||||
"hyper::proto::h1::io=warn,",
|
||||
"http_server_common::request::get_request_ip=info," // Debug spams Rust logs
|
||||
);
|
||||
|
||||
//#[tokio::main]
|
||||
//pub async fn main() -> AnyhowResult<()> {
|
||||
// test::test().await
|
||||
//}
|
||||
|
||||
#[actix_web::main]
|
||||
pub async fn main() -> AnyhowResult<()> {
|
||||
easyenv::init_all_with_default_logging(Some(LOG_LEVEL));
|
||||
|
||||
// NB: Do not check this secrets-containing dotenv file into VCS.
|
||||
// This file should only contain *development* secrets, never production.
|
||||
let _ = dotenv::from_filename(".env-aichatbot-secrets").ok();
|
||||
|
||||
let startup_args = get_startup_args()?;
|
||||
|
||||
// TODO: Fix this mess of Arc<T> wrapping other Arc<T> !
|
||||
let app_control_state_inner = AppControlState::new();
|
||||
let app_control_state = Arc::new(app_control_state_inner.clone());
|
||||
|
||||
let openai_client = Client::new()
|
||||
.with_api_key(startup_args.openai_secret_key.clone())
|
||||
//.with_backoff(backoff::ExponentialBackoff)
|
||||
;
|
||||
|
||||
let openai_client = Arc::new(openai_client);
|
||||
|
||||
let fakeyou_credentials = FakeYouCredentials::from_session_cookie_payload(
|
||||
&startup_args.fakeyou_session_cookie_payload);
|
||||
|
||||
let fakeyou_client =
|
||||
Arc::new(FakeYouApiClient::make_production_client_from_credentials(fakeyou_credentials)?);
|
||||
|
||||
let save_directory = SaveDirectory::new(&startup_args.save_directory);
|
||||
|
||||
let database_url = easyenv::get_env_string_required("DATABASE_URL")?;
|
||||
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(16)
|
||||
.connect(&database_url).await?;
|
||||
|
||||
let pool2 = pool.clone(); // NB: Clone-safe.
|
||||
|
||||
let job_state = Arc::new(JobState {
|
||||
openai_client: openai_client.clone(),
|
||||
fakeyou_client: fakeyou_client.clone(),
|
||||
save_directory: save_directory.clone(),
|
||||
sqlite_pool: pool,
|
||||
app_control_state: app_control_state_inner.clone(),
|
||||
});
|
||||
|
||||
info!("Starting worker threads and web server...");
|
||||
|
||||
let app_control_state2 = app_control_state.clone();
|
||||
let openai_client2 = openai_client.clone();
|
||||
|
||||
// NB: both egui and imgui (which we aren't using) complain about launching on a non-main thread.
|
||||
// They even complain that this is impossible on Windows (and our program aims to be multiplatform)
|
||||
// Thus, we launch everything else into its own thread.
|
||||
thread::spawn(move || {
|
||||
let tokio_runtime = Builder::new_multi_thread()
|
||||
.worker_threads(16)
|
||||
.thread_name("tokio-worker")
|
||||
.thread_stack_size(3 * 1024 * 1024)
|
||||
.enable_time()
|
||||
.enable_io()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let job_state2 = job_state.clone();
|
||||
let job_state3 = job_state.clone();
|
||||
let job_state4 = job_state.clone();
|
||||
let job_state5 = job_state.clone();
|
||||
let job_state6 = job_state.clone();
|
||||
let job_state7 = job_state.clone();
|
||||
let job_state8 = job_state.clone();
|
||||
let job_state9 = job_state.clone();
|
||||
let job_state10 = job_state.clone();
|
||||
let job_state11 = job_state.clone();
|
||||
let job_state12 = job_state.clone();
|
||||
let job_state13 = job_state.clone();
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = web_index_ingestion_main_loop(job_state2).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = web_content_scraping_main_loop(job_state3).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_greenlighting_main_loop(job_state4).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_llm_rendition_main_loop(job_state5).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_llm_title_summary_main_loop(job_state12).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_llm_category_summary_main_loop(job_state13).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_audio_preprocessing_main_loop(job_state6).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = fakeyou_audio_create_main_loop(job_state7).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = fakeyou_audio_download_main_loop(job_state8).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_audio_final_verification_main_loop(job_state9).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_post_production_finalization_main_loop(job_state10).await;
|
||||
});
|
||||
|
||||
tokio_runtime.spawn(async {
|
||||
let _r = news_story_image_generation_main_loop(job_state11).await;
|
||||
});
|
||||
|
||||
// TODO: Final scheduling thread
|
||||
//tokio_runtime.spawn(async {
|
||||
// // TODO...
|
||||
//});
|
||||
|
||||
let server_future = launch_web_server(LaunchWebServerArgs {
|
||||
app_control_state: app_control_state2,
|
||||
openai_client: openai_client2,
|
||||
save_directory,
|
||||
sqlite_pool: pool2.clone(),
|
||||
});
|
||||
|
||||
let runtime = actix_web::rt::System::new();
|
||||
|
||||
runtime.block_on(server_future)
|
||||
});
|
||||
|
||||
info!("Starting GUI ...");
|
||||
|
||||
let _r = launch_gui(startup_args.clone(), app_control_state.clone());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
|
||||
// NB: this is copied from crates/_types/buckets/src/util/hashed_directory_path_short_string.rs
|
||||
pub fn hashed_directory_path(file_hash: &str) -> String {
|
||||
match file_hash.len() {
|
||||
0 | 1 => "".to_string(),
|
||||
2 => format!("{}/", &file_hash[0..1]),
|
||||
3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
|
||||
_ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
/// For serializing Image Generation responses for debugging
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ImageGenerationDebugData {
|
||||
/// Prompt used to generate the image
|
||||
pub image_generation_prompt: String,
|
||||
|
||||
/// We might have created a prompt to generate a prompt.
|
||||
pub maybe_preconditioning_prompt: Option<String>,
|
||||
}
|
||||
|
||||
impl ImageGenerationDebugData {
|
||||
|
||||
//pub fn try_from_yaml_file<P: AsRef<Path>>(filename: P) -> AnyhowResult<Self> {
|
||||
// let file = std::fs::File::open(filename)?;
|
||||
// let object = serde_yaml::from_reader(file)?;
|
||||
// Ok(object)
|
||||
//}
|
||||
|
||||
pub fn write_to_yaml_file<P: AsRef<Path>>(&self, filename: P) -> AnyhowResult<()> {
|
||||
let mut file = std::fs::File::create(filename)?;
|
||||
serde_yaml::to_writer(file, &self)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
use errors::AnyhowResult;
|
||||
use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
|
||||
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
|
||||
pub async fn load_scraped_result(url: &str, save_directory: &SaveDirectory) -> AnyhowResult<ScrapedWebArticle> {
|
||||
let scrape_yaml_filename = save_directory.scrape_summary_file_for_webpage_url(url)?;
|
||||
let mut file = std::fs::File::open(&scrape_yaml_filename)?;
|
||||
let scraping_result : ScrapedWebArticle = serde_yaml::from_reader(file)?;
|
||||
Ok(scraping_result)
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
pub mod hashed_directory_path;
|
||||
pub mod image_generation_debug_data;
|
||||
pub mod load_scraped_result;
|
||||
pub mod rendition_data;
|
||||
pub mod save_directory;
|
||||
pub mod speakable_monologue;
|
||||
@@ -1,31 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
/// For serializing OpenAI GPT requests.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct RenditionData {
|
||||
/// Location the article came from
|
||||
pub original_content_url: String,
|
||||
|
||||
/// The prompt we sent to OpenAI
|
||||
pub original_prompt: String,
|
||||
|
||||
/// Response from OpenAI
|
||||
pub response: String,
|
||||
}
|
||||
|
||||
impl RenditionData {
|
||||
|
||||
pub fn try_from_yaml_file<P: AsRef<Path>>(filename: P) -> AnyhowResult<Self> {
|
||||
let file = std::fs::File::open(filename)?;
|
||||
let object = serde_yaml::from_reader(file)?;
|
||||
Ok(object)
|
||||
}
|
||||
|
||||
pub fn write_to_yaml_file<P: AsRef<Path>>(&self, filename: P) -> AnyhowResult<()> {
|
||||
let mut file = std::fs::File::create(filename)?;
|
||||
serde_yaml::to_writer(file, &self)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use hashing::sha256::sha256_hash_string::sha256_hash_string;
|
||||
use log::info;
|
||||
use path_absolutize::Absolutize;
|
||||
use tokens::tokens::news_stories::NewsStoryToken;
|
||||
|
||||
use crate::persistence::hashed_directory_path::hashed_directory_path;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SaveDirectory {
|
||||
directory: PathBuf,
|
||||
}
|
||||
|
||||
impl SaveDirectory {
|
||||
|
||||
pub fn new<P: AsRef<Path>>(directory: P) -> Self {
|
||||
Self {
|
||||
directory: directory.as_ref().to_path_buf()
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Move web pages for news under "news_stories/{hashed_url_path}/"
|
||||
pub fn directory_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
let url_hash = sha256_hash_string(url)?;
|
||||
let directory_path= hashed_directory_path(&url_hash);
|
||||
|
||||
let directory = self.directory.clone()
|
||||
.join("webpages/")
|
||||
.join(directory_path)
|
||||
.join(url_hash);
|
||||
|
||||
Ok(directory)
|
||||
}
|
||||
|
||||
// TODO: Move web pages for news under "news_stories/{hashed_url_path}/audio/"
|
||||
// This requires associating the job with the URL.
|
||||
pub fn directory_for_audio(&self, news_story_token: &NewsStoryToken) -> AnyhowResult<PathBuf> {
|
||||
let entropy_part = news_story_token.entropy_suffix();
|
||||
let directory_path= hashed_directory_path(&entropy_part);
|
||||
|
||||
let directory = self.directory.clone()
|
||||
.join("audio/")
|
||||
.join(directory_path)
|
||||
.join(news_story_token.as_str());
|
||||
|
||||
Ok(directory)
|
||||
}
|
||||
|
||||
pub fn html_file_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("webpage.html"))
|
||||
}
|
||||
|
||||
pub fn scrape_summary_file_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("scrape_summary.yaml"))
|
||||
}
|
||||
|
||||
pub fn rendition_file_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("gpt_rendition.yaml"))
|
||||
}
|
||||
|
||||
pub fn speakable_monologue_file_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("speakable_monologue.yaml"))
|
||||
}
|
||||
|
||||
pub fn generated_headline_image_for_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("generated_headline_image.png"))
|
||||
}
|
||||
|
||||
pub fn image_generation_prompt_debug_file_webpage_url(&self, url: &str) -> AnyhowResult<PathBuf> {
|
||||
Ok(self.directory_for_webpage_url(url)?.join("image_generation_prompts.yaml"))
|
||||
}
|
||||
|
||||
pub fn audio_wav_file_for_news_story(&self, news_story_token: &NewsStoryToken, sequence_order: i64) -> AnyhowResult<PathBuf> {
|
||||
let filename = format!("{}.wav", sequence_order);
|
||||
Ok(self.directory_for_audio(news_story_token)?.join(filename))
|
||||
}
|
||||
|
||||
/// This is just the first directory structure, which is sequential audio files.
|
||||
/// We'll be using a database and well-formed filesystem layout later.
|
||||
pub fn get_audio_files_dir_v1(&self) -> PathBuf {
|
||||
let result = self.directory.join("audio_files");
|
||||
let result = result.absolutize()
|
||||
.map(|file| file.to_path_buf())
|
||||
.unwrap_or(result);
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
/// For handing off to FakeYou for audio generation
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SpeakableMonologue {
|
||||
/// Paragraphs to speak
|
||||
pub paragraphs: Vec<String>,
|
||||
}
|
||||
|
||||
impl SpeakableMonologue {
|
||||
|
||||
pub fn try_from_yaml_file<P: AsRef<Path>>(filename: P) -> AnyhowResult<Self> {
|
||||
let file = std::fs::File::open(filename)?;
|
||||
let object = serde_yaml::from_reader(file)?;
|
||||
Ok(object)
|
||||
}
|
||||
|
||||
pub fn write_to_yaml_file<P: AsRef<Path>>(&self, filename: P) -> AnyhowResult<()> {
|
||||
let mut file = std::fs::File::create(filename)?;
|
||||
serde_yaml::to_writer(file, &self)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
|
||||
use crate::configs::fakeyou_voice_option::FakeYouVoiceOption;
|
||||
use crate::configs::level_option::LevelOption;
|
||||
|
||||
/// User-controlled parameters that determine how the app behaves at runtime.
|
||||
#[derive(Clone)]
|
||||
pub struct AppControlState {
|
||||
/// Whether the playback should be paused
|
||||
/// Unreal Engine will get this over HTTP and know whether to no-op.
|
||||
is_paused: Arc<RwLock<bool>>,
|
||||
|
||||
/// Whether unreal should pause playback.
|
||||
is_unreal_paused: RelaxedAtomicBool,
|
||||
|
||||
/// Whether the web scraping should be paused.
|
||||
is_scraping_paused: RelaxedAtomicBool,
|
||||
|
||||
/// Whether calls to OpenAI should be paused.
|
||||
is_openai_paused: RelaxedAtomicBool,
|
||||
|
||||
/// Whether calls to FakeYou should be paused.
|
||||
is_fakeyou_paused: RelaxedAtomicBool,
|
||||
|
||||
/// Which level to present.
|
||||
current_level: Arc<RwLock<LevelOption>>,
|
||||
|
||||
/// Option of which voice to use for TTS.
|
||||
fakeyou_voice: Arc<RwLock<FakeYouVoiceOption>>,
|
||||
}
|
||||
|
||||
impl AppControlState {
|
||||
|
||||
pub fn new() -> Self {
|
||||
AppControlState {
|
||||
is_paused: Arc::new(RwLock::new(false)),
|
||||
is_unreal_paused: RelaxedAtomicBool::new(false),
|
||||
is_scraping_paused: RelaxedAtomicBool::new(false),
|
||||
is_openai_paused: RelaxedAtomicBool::new(false),
|
||||
is_fakeyou_paused: RelaxedAtomicBool::new(false),
|
||||
current_level: Arc::new(RwLock::new(LevelOption::default())),
|
||||
fakeyou_voice: Arc::new(RwLock::new(FakeYouVoiceOption::default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_paused(&self) -> AnyhowResult<bool> {
|
||||
match self.is_paused.read() {
|
||||
Ok(value) => Ok(*value),
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_is_paused(&self, new_value: bool) -> AnyhowResult<()> {
|
||||
match self.is_paused.write() {
|
||||
Ok(mut value) => {
|
||||
*value = new_value;
|
||||
Ok(())
|
||||
},
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_unreal_paused(&self) -> bool {
|
||||
self.is_unreal_paused.get()
|
||||
}
|
||||
|
||||
pub fn set_is_unreal_paused(&self, new_value: bool) {
|
||||
self.is_unreal_paused.set(new_value)
|
||||
}
|
||||
|
||||
pub fn is_scraping_paused(&self) -> bool {
|
||||
self.is_scraping_paused.get()
|
||||
}
|
||||
|
||||
pub fn set_is_scraping_paused(&self, new_value: bool) {
|
||||
self.is_scraping_paused.set(new_value)
|
||||
}
|
||||
|
||||
pub fn is_openai_paused(&self) -> bool {
|
||||
self.is_openai_paused.get()
|
||||
}
|
||||
|
||||
pub fn set_is_openai_paused(&self, new_value: bool) {
|
||||
self.is_openai_paused.set(new_value)
|
||||
}
|
||||
|
||||
pub fn is_fakeyou_paused(&self) -> bool {
|
||||
self.is_fakeyou_paused.get()
|
||||
}
|
||||
|
||||
pub fn set_is_fakeyou_paused(&self, new_value: bool) {
|
||||
self.is_fakeyou_paused.set(new_value)
|
||||
}
|
||||
|
||||
pub fn get_level(&self) -> AnyhowResult<LevelOption> {
|
||||
match self.current_level.read() {
|
||||
Ok(value) => Ok(*value),
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_level(&self, new_value: LevelOption) -> AnyhowResult<()> {
|
||||
match self.current_level.write() {
|
||||
Ok(mut value) => {
|
||||
*value = new_value;
|
||||
Ok(())
|
||||
},
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fakeyou_voice(&self) -> AnyhowResult<FakeYouVoiceOption> {
|
||||
match self.fakeyou_voice.read() {
|
||||
Ok(value) => Ok(*value),
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_fakeyou_voice(&self, new_value: FakeYouVoiceOption) -> AnyhowResult<()> {
|
||||
match self.fakeyou_voice.write() {
|
||||
Ok(mut value) => {
|
||||
*value = new_value;
|
||||
Ok(())
|
||||
},
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_openai::Client;
|
||||
use fakeyou_client::fakeyou_api_client::FakeYouApiClient;
|
||||
use sqlx::{Pool, Sqlite};
|
||||
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct JobState {
|
||||
pub openai_client: Arc<Client>,
|
||||
pub fakeyou_client: Arc<FakeYouApiClient>,
|
||||
|
||||
pub save_directory: SaveDirectory,
|
||||
pub sqlite_pool: Pool<Sqlite>,
|
||||
|
||||
pub app_control_state: AppControlState,
|
||||
}
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod app_control_state;
|
||||
pub mod job_state;
|
||||
@@ -1,83 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{App, Arg, ArgMatches};
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use log::info;
|
||||
use path_absolutize::Absolutize;
|
||||
|
||||
const DEFAULT_SAVE_DIRECTORY : &str = "runtime_data";
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct StartupArgs {
|
||||
/// Where the program stores its state and database
|
||||
pub save_directory: String,
|
||||
|
||||
pub openai_secret_key: String,
|
||||
pub fakeyou_api_token: String,
|
||||
pub fakeyou_session_cookie_payload: String,
|
||||
}
|
||||
|
||||
/// Read in environment and CLI args and output the consolidated startup state
|
||||
pub fn get_startup_args() -> AnyhowResult<StartupArgs> {
|
||||
let matches = App::new("aichatbot-sidecar")
|
||||
.arg(Arg::with_name("file")
|
||||
.short("d")
|
||||
.long("directory")
|
||||
.value_name("DIRECTORY")
|
||||
.help("The save directory")
|
||||
.takes_value(true)
|
||||
.required(false))
|
||||
.get_matches();
|
||||
|
||||
let openai_secret_key = easyenv::get_env_string_required("OPENAI_SECRET_KEY")?;
|
||||
|
||||
let fakeyou_api_token = easyenv::get_env_string_required("FAKEYOU_API_TOKEN")?;
|
||||
|
||||
let fakeyou_session_cookie_payload = easyenv::get_env_string_required("FAKEYOU_SESSION_COOKIE_PAYLOAD")?;
|
||||
|
||||
let save_directory = get_save_directory(&matches)?;
|
||||
|
||||
Ok(StartupArgs {
|
||||
save_directory,
|
||||
openai_secret_key,
|
||||
fakeyou_api_token,
|
||||
fakeyou_session_cookie_payload,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
fn get_save_directory(matches: &ArgMatches) -> AnyhowResult<String> {
|
||||
let mut maybe_save_directory = matches.value_of("directory")
|
||||
.map(|val| val.trim().to_string())
|
||||
.map(|val| PathBuf::from(val));
|
||||
|
||||
info!("SA dir(1): {:?}", maybe_save_directory);
|
||||
|
||||
if maybe_save_directory.is_none() {
|
||||
// Use the current directory
|
||||
maybe_save_directory = std::env::current_dir()
|
||||
.ok()
|
||||
.map(|pathbuf| pathbuf.join(DEFAULT_SAVE_DIRECTORY));
|
||||
//.map(|pathbuf| std::fs::canonicalize(pathbuf).ok())
|
||||
//.flatten();
|
||||
|
||||
info!("SA dir(2): {:?}", maybe_save_directory);
|
||||
}
|
||||
|
||||
let save_directory = maybe_save_directory
|
||||
.unwrap_or_else(|| PathBuf::from(DEFAULT_SAVE_DIRECTORY));
|
||||
|
||||
info!("SA dir(3): {:?}", save_directory);
|
||||
|
||||
let save_directory = save_directory
|
||||
.absolutize()
|
||||
.ok()
|
||||
.map(|inner| inner.to_path_buf())
|
||||
.map(|pathbuf| pathbuf.to_str().map(|s| s.to_string()))
|
||||
.flatten()
|
||||
.ok_or(anyhow!("could not construct path"))?;
|
||||
|
||||
info!("SA dir(4): {:?}", save_directory);
|
||||
|
||||
Ok(save_directory)
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
//!
|
||||
//! This is for testing new functionality quickly without running the whole program.
|
||||
//!
|
||||
|
||||
use async_openai::Client;
|
||||
use async_openai::types::{CreateImageRequest, ImageSize, ResponseFormat};
|
||||
use enums::common::sqlite::web_content_type::WebContentType;
|
||||
use errors::AnyhowResult;
|
||||
use sqlx::sqlite::SqlitePoolOptions;
|
||||
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::startup_args::get_startup_args;
|
||||
use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save;
|
||||
|
||||
pub async fn test() -> AnyhowResult<()> {
|
||||
let database_url = easyenv::get_env_string_required("DATABASE_URL")?;
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(5)
|
||||
.connect(&database_url).await?;
|
||||
|
||||
let _ = dotenv::from_filename(".env-aichatbot-secrets").ok();
|
||||
let startup_args = get_startup_args()?;
|
||||
let save_directory = SaveDirectory::new(&startup_args.save_directory);
|
||||
|
||||
//let url = "https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/";
|
||||
let url = "https://www.cnn.com/2023/02/04/business/automakers-problems-catching-up-with-tesla/index.html";
|
||||
ingest_url_scrape_and_save(url, WebContentType::CnnArticle, &save_directory).await?;
|
||||
|
||||
let openai_client = Client::new()
|
||||
.with_api_key(startup_args.openai_secret_key.clone());
|
||||
|
||||
let create_request = CreateImageRequest {
|
||||
n: Some(1),
|
||||
//prompt: "A news headline image; headline: After nearly one year of war, how Ukraine defied the odds — and may still defeat Russia".to_string(),
|
||||
//prompt: "A news headline image; headline: 'Heartbreaking': Visitor accidentally shatters Jeff Koons 'balloon dog' sculpture at Art Wynwood".to_string(),
|
||||
//prompt: "A news headline image; headline: Mayorkas goes on the offensive as GOP scrutiny builds, says it’s up to Congress to fix immigration system ".to_string(),
|
||||
//prompt: "Brave or lucky? See the moment a dog took on a hammerhead shark".to_string(),
|
||||
//prompt: "Headline: Jimmy Carter to begin receiving home hospice care".to_string(),
|
||||
//prompt: "Nine children hurt, shooting; Tragic, frightful, chaotic, violent, frightening.".to_string(),
|
||||
//prompt: "9 kids hurt in Georgia; Disturbing, tragic, frightening, violent, chaotic; Art Style/Director: Wes Anderson/Surrealist.; remove text; no text; no English text".to_string(),
|
||||
//prompt: "Summary: 9 kids hurt in Georgia; Adjectives: Disturbing, tragic, frightening, violent, chaotic; Art Style/Director: Wes Anderson/Surrealist.".to_string(),
|
||||
//prompt: "Reviving cheetahs in India. Adventurous, ambitious, necessary, daring, wild. Wes Anderson, Runnings fields with cheetahs racing around in the distance, trees swaying in the wind, and the sun setting in the background.".to_string(),
|
||||
prompt: "Closure found after WWII; moving, emotional, joyous, triumphant, proud.
|
||||
Art style: Impressionism.
|
||||
Setting: A coastal beach with a few people looking out to sea, with a large American flag waving in the background. Objects: a few boats in the distant, a lighthouse, and some driftwood on the shore. A photo or painting of this might be a peaceful beachscape with vibrant colors of the sky, sea, and flag.".to_string(),
|
||||
//prompt: "LGBTQ love, vibrant, colorful, playful, revolutionary. Art style: Bollywood musicals. Setting: A city street in the evening, lit up with vibrant neon signs and bustling crowds. There are couples in traditional Indian clothing, dancing and singing around colorful street vendors selling snacks and trinkets.".to_string(),
|
||||
//prompt: "Kinkade painting of Releasing twelve manatees: miraculous, astounding, incredible, remarkable, amazing. Setting: A peaceful beach, with gentle sand and a clear blue sky. Objects: A crowd of people lined up along the shore, a boat full of manatees, and the vibrant colors of the ocean.".to_string(),
|
||||
//prompt: "Photo of Releasing twelve manatees: miraculous, astounding, incredible, remarkable, amazing. Art style: Tim Burton.".to_string(),
|
||||
size: Some(ImageSize::S1024x1024),
|
||||
response_format: Some(ResponseFormat::Url),
|
||||
user: Some("test".to_string()),
|
||||
};
|
||||
|
||||
let response = openai_client
|
||||
.images()
|
||||
.create(create_request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// let create_edit_request = CreateImageEditRequest {
|
||||
// image: ImageInput { path: PathBuf::from("runtime_data/scaled_test.png") },
|
||||
// mask: ImageInput { path: PathBuf::from("runtime_data/scaled_mask.png") },
|
||||
// prompt: "Closure found after WWII; moving, emotional, joyous, triumphant, proud.
|
||||
//Art style: Impressionism.
|
||||
//Setting: A coastal beach with a few people looking out to sea, with a large American flag waving in the background. Objects: a few boats in the distant, a lighthouse, and some driftwood on the shore. A photo or painting of this might be a peaceful beachscape with vibrant colors of the sky, sea, and flag.".to_string(),
|
||||
// n: Some(1),
|
||||
// size: Some(ImageSize::S1024x1024),
|
||||
// response_format: Some(ResponseFormat::Url),
|
||||
// user: Some("test".to_string()),
|
||||
// };
|
||||
//
|
||||
// let response = openai_client
|
||||
// .images()
|
||||
// .create_edit(create_edit_request)
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
|
||||
println!("Response: {:?}", response);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::error;
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GetCurrentLevelResponse {
|
||||
pub success: bool,
|
||||
pub current_level: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum GetCurrentLevelError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for GetCurrentLevelError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetCurrentLevelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for GetCurrentLevelError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_current_level_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>
|
||||
) -> Result<HttpResponse, GetCurrentLevelError> {
|
||||
|
||||
let current_level = control_state.get_level()
|
||||
.map_err(|err| {
|
||||
error!("Error: {:?}", err);
|
||||
GetCurrentLevelError::ServerError
|
||||
})?;
|
||||
|
||||
let response = GetCurrentLevelResponse {
|
||||
success: true,
|
||||
current_level: current_level.tts_model_token_str().to_string(),
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| GetCurrentLevelError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
pub mod get_current_level;
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use actix_web::web::Query;
|
||||
use filesys::file_exists::file_exists;
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::{error, info};
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::web_server::server_state::ServerState;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GetNextAudioQuery {
|
||||
pub cursor: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GetNextAudioFileResponse {
|
||||
pub success: bool,
|
||||
pub is_paused: bool,
|
||||
pub audio_filename_platform: Option<String>,
|
||||
/// The audio filename, in a Unix style (forward slashes).
|
||||
/// Note, this might have a windows drive letter.
|
||||
pub audio_filename_unixy: Option<String>,
|
||||
pub next_cursor: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum GetNextAudioFileError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for GetNextAudioFileError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetNextAudioFileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for GetNextAudioFileError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_next_audio_file_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>,
|
||||
server_state: web::Data<Arc<ServerState>>,
|
||||
request: Query<GetNextAudioQuery>,
|
||||
) -> Result<HttpResponse, GetNextAudioFileError> {
|
||||
|
||||
info!("Requested cursor: {}", request.cursor);
|
||||
|
||||
let is_paused = control_state.is_paused()
|
||||
.map_err(|err| {
|
||||
error!("Error: {:?}", err);
|
||||
GetNextAudioFileError::ServerError
|
||||
})?;
|
||||
|
||||
|
||||
let audio_file_dir = server_state.save_directory.get_audio_files_dir_v1();
|
||||
|
||||
let maybe_audio_filename = format!("{}.wav", request.cursor);
|
||||
let maybe_audio_filename = audio_file_dir.join(maybe_audio_filename);
|
||||
|
||||
info!("Hypothetical audio file: {:?}", maybe_audio_filename);
|
||||
|
||||
let mut next_cursor;
|
||||
let mut audio_filename : Option<String> = None;
|
||||
let mut audio_filename_unixy : Option<String> = None;
|
||||
|
||||
if file_exists(&maybe_audio_filename) {
|
||||
next_cursor = request.cursor + 1;
|
||||
audio_filename = maybe_audio_filename.to_str()
|
||||
.map(|s| s.to_string());
|
||||
audio_filename_unixy = audio_filename.as_deref()
|
||||
.map(|s| s.to_string())
|
||||
.map(|s| s.replace("\\", "/"));
|
||||
} else {
|
||||
next_cursor = 0;
|
||||
audio_filename = None;
|
||||
};
|
||||
|
||||
let maybe_next_audio_filename = format!("{}.wav", next_cursor);
|
||||
let maybe_next_audio_filename = audio_file_dir.join(maybe_next_audio_filename);
|
||||
|
||||
if !file_exists(&maybe_next_audio_filename) {
|
||||
next_cursor = 0;
|
||||
}
|
||||
|
||||
let response = GetNextAudioFileResponse {
|
||||
success: true,
|
||||
is_paused,
|
||||
next_cursor,
|
||||
audio_filename_platform: audio_filename,
|
||||
audio_filename_unixy,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| GetNextAudioFileError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
Whitespace-only changes.
@@ -1,4 +0,0 @@
|
||||
pub mod get_next_audio_file_handler;
|
||||
pub mod index_handler;
|
||||
pub mod next_audio_file_handler;
|
||||
pub mod openai_inference_handler;
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::error;
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct NextAudioFileResponse {
|
||||
pub success: bool,
|
||||
pub is_paused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum NextAudioFileError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for NextAudioFileError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
NextAudioFileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for NextAudioFileError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn next_audio_file_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>
|
||||
) -> Result<HttpResponse, NextAudioFileError> {
|
||||
|
||||
let is_paused = control_state.is_paused()
|
||||
.map_err(|err| {
|
||||
error!("Error: {:?}", err);
|
||||
NextAudioFileError::ServerError
|
||||
})?;
|
||||
|
||||
let response = NextAudioFileResponse {
|
||||
success: true,
|
||||
is_paused,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| NextAudioFileError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use async_openai::Client;
|
||||
use async_openai::types::CreateCompletionRequestArgs;
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::{error, info};
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct OpenAiInferenceResponse {
|
||||
pub success: bool,
|
||||
pub openai_results: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum OpenAiInferenceError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for OpenAiInferenceError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
OpenAiInferenceError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for OpenAiInferenceError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn openai_inference_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>,
|
||||
openai_client: web::Data<Arc<Client>>,
|
||||
) -> Result<HttpResponse, OpenAiInferenceError> {
|
||||
|
||||
let article = r#"
|
||||
The U.S. military has been monitoring a suspected Chinese surveillance balloon that has been hovering over the northern U.S. for the past few days, and military and defense leaders have discussed shooting it out of the sky, according to two U.S. officials and a senior defense official.
|
||||
|
||||
“The United States government has detected and is tracking a high-altitude surveillance balloon that is over the continental United States right now,” Pentagon spokesperson Brig. Gen. Pat Ryder told NBC News. “We continue to track and monitor it closely.”
|
||||
|
||||
“Once the balloon was detected, the U.S. government acted immediately to protect against the collection of sensitive information,” Ryder said.
|
||||
"#.to_string();
|
||||
|
||||
let prompt = format!(r#"The following is a news article:
|
||||
|
||||
{}
|
||||
|
||||
Please reword this article as if it was spoken by a reporter.
|
||||
"#, article);
|
||||
|
||||
|
||||
let request = CreateCompletionRequestArgs::default()
|
||||
.model("text-davinci-003")
|
||||
.prompt(prompt)
|
||||
.max_tokens(1200_u16)
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
error!("Could not create OpenAI request: {:?}", err);
|
||||
OpenAiInferenceError::ServerError
|
||||
})?;
|
||||
|
||||
let response = openai_client
|
||||
.completions()
|
||||
.create(request)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Error calling OpenAI: {:?}", err);
|
||||
OpenAiInferenceError::ServerError
|
||||
})?;
|
||||
|
||||
info!("Open AI response: {:?}", response);
|
||||
|
||||
let text = response.choices.get(0)
|
||||
.map(|option| option.text.clone())
|
||||
.unwrap_or("".to_string());
|
||||
|
||||
let response = OpenAiInferenceResponse {
|
||||
success: true,
|
||||
openai_results: text,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| OpenAiInferenceError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
pub mod level;
|
||||
pub mod misc;
|
||||
pub mod news_stories;
|
||||
-188
@@ -1,188 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use actix_web::web::Query;
|
||||
use filesys::file_exists::file_exists;
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::{error, info, warn};
|
||||
use rand::seq::SliceRandom;
|
||||
use sqlite_queries::queries::by_table::news_stories::get_news_story_by_token::get_news_story_by_token;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_for_story_token::list_tts_render_tasks_for_story_token;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::tts_render_task::TtsRenderTask;
|
||||
use tokens::tokens::news_stories::NewsStoryToken;
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::web_server::handlers::misc::get_next_audio_file_handler::GetNextAudioFileError;
|
||||
use crate::web_server::server_state::ServerState;
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GetNextNewsStoryAudioFileQuery {
|
||||
/// News Story Token. REQUIRED.
|
||||
pub news_story_token: Option<NewsStoryToken>,
|
||||
|
||||
/// Audio cursor, 0-indexed.
|
||||
pub audio_cursor: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
pub struct GetNextNewsStoryAudioFileResponse {
|
||||
pub success: bool,
|
||||
pub news_story_token: NewsStoryToken,
|
||||
|
||||
//pub original_news_canonical_url: String,
|
||||
//pub original_news_title: String,
|
||||
|
||||
/// The number of audio files in the current sequence.
|
||||
pub sequence_audio_file_count: u64,
|
||||
|
||||
/// Current audio cursor (1-indexed). This is the one to play.
|
||||
pub current_audio_index: u64,
|
||||
|
||||
/// The local filesystem path of the audio file.
|
||||
pub audio_file_path: PathBuf,
|
||||
|
||||
/// The local headline image file path.
|
||||
pub headline_image_file_path: PathBuf,
|
||||
|
||||
/// Whether there's a next cursor in the current sequence.
|
||||
pub has_next_in_sequence: bool,
|
||||
|
||||
/// If there's a next cursor in the current sequence, this is it.
|
||||
/// If no such cursor, the field will be set to JSON NULL.
|
||||
pub maybe_next_cursor: Option<u64>,
|
||||
|
||||
/// If there's a next cursor in the current sequence, this is it.
|
||||
/// If no such cursor, the field will be set to the sigil "-1".
|
||||
/// This is for Unreal Engine's sake, which is sloppy with handling JSON.
|
||||
pub maybe_next_cursor_else_sigil_value: i64,
|
||||
|
||||
//pub audio_total_duration_seconds: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum GetNextNewsStoryAudioFileError {
|
||||
ServerError,
|
||||
BadRequest,
|
||||
NotFound,
|
||||
}
|
||||
|
||||
impl ResponseError for GetNextNewsStoryAudioFileError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetNextNewsStoryAudioFileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
GetNextNewsStoryAudioFileError::BadRequest => StatusCode::BAD_REQUEST,
|
||||
GetNextNewsStoryAudioFileError::NotFound => StatusCode::NOT_FOUND,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for GetNextNewsStoryAudioFileError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_next_news_story_audio_file_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>,
|
||||
server_state: web::Data<Arc<ServerState>>,
|
||||
query: Query<GetNextNewsStoryAudioFileQuery>,
|
||||
) -> Result<HttpResponse, GetNextNewsStoryAudioFileError> {
|
||||
|
||||
warn!("get_next_news_story_audio_file_handler() : {:?}", &query);
|
||||
|
||||
let news_story_token = query.news_story_token
|
||||
.clone()
|
||||
.ok_or(GetNextNewsStoryAudioFileError::BadRequest)?;
|
||||
|
||||
let query_result = get_news_story_by_token(
|
||||
&server_state.sqlite_pool,
|
||||
&news_story_token
|
||||
).await;
|
||||
|
||||
let news_story = match query_result {
|
||||
Ok(Some(news_story)) => news_story,
|
||||
Ok(None) => return Err(GetNextNewsStoryAudioFileError::NotFound),
|
||||
Err(err) => {
|
||||
error!("Database error: {:?}", err);
|
||||
return Err(GetNextNewsStoryAudioFileError::NotFound)
|
||||
},
|
||||
};
|
||||
|
||||
let story_type = "news_story"; // TODO: Enumlike.
|
||||
let story_token = news_story_token.as_str();
|
||||
|
||||
let tts_render_tasks = list_tts_render_tasks_for_story_token(
|
||||
story_type,
|
||||
story_token,
|
||||
&server_state.sqlite_pool)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Database error: {:?}", err);
|
||||
GetNextNewsStoryAudioFileError::ServerError
|
||||
})?;
|
||||
|
||||
let audio_cursor = query.audio_cursor.unwrap_or(0) as usize;
|
||||
|
||||
let tts_render_task : &TtsRenderTask = match tts_render_tasks.get(audio_cursor) {
|
||||
Some(task) => task,
|
||||
None => {
|
||||
warn!("TTS task not found for NewsStoryToken {:?} cursor {}", news_story_token, audio_cursor);
|
||||
return Err(GetNextNewsStoryAudioFileError::NotFound);
|
||||
}
|
||||
};
|
||||
|
||||
let audio_filename = server_state.save_directory
|
||||
.audio_wav_file_for_news_story(&news_story_token, tts_render_task.sequence_order)
|
||||
.map_err(|err| {
|
||||
error!("Path error: {:?}", err);
|
||||
GetNextNewsStoryAudioFileError::ServerError
|
||||
})?;
|
||||
|
||||
let headline_image_filename = server_state.save_directory
|
||||
.generated_headline_image_for_webpage_url(&news_story.original_news_canonical_url)
|
||||
.map_err(|err| {
|
||||
error!("Path error: {:?}", err);
|
||||
GetNextNewsStoryAudioFileError::ServerError
|
||||
})?;
|
||||
|
||||
let next_audio_cursor = audio_cursor + 1;
|
||||
|
||||
let mut has_next_in_sequence = false;
|
||||
let mut maybe_next_cursor = None;
|
||||
|
||||
if tts_render_tasks.get(next_audio_cursor).is_some() {
|
||||
has_next_in_sequence = true;
|
||||
maybe_next_cursor = Some(next_audio_cursor as u64);
|
||||
};
|
||||
|
||||
let response = GetNextNewsStoryAudioFileResponse {
|
||||
success: true,
|
||||
news_story_token: news_story_token.clone(),
|
||||
sequence_audio_file_count: news_story.audio_file_count as u64,
|
||||
current_audio_index: audio_cursor as u64,
|
||||
audio_file_path: audio_filename,
|
||||
headline_image_file_path: headline_image_filename,
|
||||
has_next_in_sequence,
|
||||
maybe_next_cursor: maybe_next_cursor.clone(),
|
||||
maybe_next_cursor_else_sigil_value: maybe_next_cursor
|
||||
.map(|val| val as i64)
|
||||
.unwrap_or(-1),
|
||||
};
|
||||
|
||||
warn!("Response: {:?}", &response);
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| GetNextNewsStoryAudioFileError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_http::StatusCode;
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use actix_web::web::Query;
|
||||
use filesys::file_exists::file_exists;
|
||||
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
|
||||
use log::{error, info, warn};
|
||||
use rand::seq::SliceRandom;
|
||||
use sqlite_queries::queries::by_table::news_stories::list::list_news_stories_all::list_news_stories_all;
|
||||
use sqlite_queries::queries::by_table::news_stories::list::list_news_stories_replayable::list_news_stories_replayable;
|
||||
use tokens::tokens::news_stories::NewsStoryToken;
|
||||
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::web_server::server_state::ServerState;
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GetNextNewsStoryQuery {
|
||||
pub news_story_token: Option<NewsStoryToken>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
pub struct GetNextNewsStoryFileResponse {
|
||||
pub success: bool,
|
||||
pub news_story_token: NewsStoryToken,
|
||||
pub original_news_canonical_url: String,
|
||||
pub original_news_title: String,
|
||||
pub summary_news_title: String,
|
||||
pub audio_file_count: u64,
|
||||
pub audio_total_duration_seconds: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum GetNextNewsStoryFileError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for GetNextNewsStoryFileError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetNextNewsStoryFileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
serialize_as_json_error(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl std::fmt::Display for GetNextNewsStoryFileError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_next_news_story_handler(
|
||||
_http_request: HttpRequest,
|
||||
control_state: web::Data<Arc<AppControlState>>,
|
||||
server_state: web::Data<Arc<ServerState>>,
|
||||
query: Query<GetNextNewsStoryQuery>,
|
||||
) -> Result<HttpResponse, GetNextNewsStoryFileError> {
|
||||
|
||||
warn!("get_next_news_story_handler() : {:?}", &query);
|
||||
|
||||
let mut stories = list_news_stories_replayable(&server_state.sqlite_pool)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Error querying: {:?}", err);
|
||||
GetNextNewsStoryFileError::ServerError
|
||||
})?;
|
||||
|
||||
// TODO: Probably shouldn't fall back on old stories! Maybe flag this.
|
||||
if stories.is_empty() {
|
||||
stories = list_news_stories_all(&server_state.sqlite_pool)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Error querying: {:?}", err);
|
||||
GetNextNewsStoryFileError::ServerError
|
||||
})?;
|
||||
}
|
||||
|
||||
// TODO: Use history information to skip recently played.
|
||||
|
||||
let story = stories.choose(&mut rand::thread_rng())
|
||||
.ok_or(GetNextNewsStoryFileError::ServerError)?;
|
||||
|
||||
let response = GetNextNewsStoryFileResponse {
|
||||
success: true,
|
||||
news_story_token: story.news_story_token.clone(),
|
||||
original_news_canonical_url: story.original_news_canonical_url.clone(),
|
||||
original_news_title: story.original_news_title.clone(),
|
||||
summary_news_title: story.summary_news_title.clone(),
|
||||
audio_file_count: story.audio_file_count as u64,
|
||||
audio_total_duration_seconds: story.audio_total_duration_seconds as u64,
|
||||
};
|
||||
|
||||
warn!("Response: {:?}", &response);
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| GetNextNewsStoryFileError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod get_next_news_story_audio_file_handler;
|
||||
pub mod get_next_news_story_handler;
|
||||
@@ -1,73 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_helpers::route_builder::RouteBuilder;
|
||||
use actix_web::{App, HttpResponse, HttpServer, web};
|
||||
use async_openai::Client;
|
||||
use errors::AnyhowResult;
|
||||
use http_server_common::endpoints::root_index::get_root_index;
|
||||
use sqlx::{Pool, Sqlite};
|
||||
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
use crate::web_server::handlers::level::get_current_level::get_current_level_handler;
|
||||
use crate::web_server::handlers::misc::get_next_audio_file_handler::get_next_audio_file_handler;
|
||||
use crate::web_server::handlers::misc::next_audio_file_handler::next_audio_file_handler;
|
||||
use crate::web_server::handlers::misc::openai_inference_handler::openai_inference_handler;
|
||||
use crate::web_server::handlers::news_stories::get_next_news_story_audio_file_handler::get_next_news_story_audio_file_handler;
|
||||
use crate::web_server::handlers::news_stories::get_next_news_story_handler::get_next_news_story_handler;
|
||||
use crate::web_server::server_state::ServerState;
|
||||
|
||||
pub struct LaunchWebServerArgs {
|
||||
pub app_control_state: Arc<AppControlState>,
|
||||
pub openai_client: Arc<Client>,
|
||||
pub save_directory: SaveDirectory,
|
||||
pub sqlite_pool: Pool<Sqlite>,
|
||||
}
|
||||
|
||||
pub async fn launch_web_server(args: LaunchWebServerArgs) -> AnyhowResult<()> {
|
||||
|
||||
let server_state = Arc::new(ServerState {
|
||||
app_control_state: args.app_control_state.clone(),
|
||||
openai_client: args.openai_client.clone(),
|
||||
save_directory: args.save_directory,
|
||||
sqlite_pool: args.sqlite_pool,
|
||||
});
|
||||
|
||||
HttpServer::new(move || {
|
||||
let app = App::new()
|
||||
.app_data(web::Data::new(args.app_control_state.clone()))
|
||||
.app_data(web::Data::new(args.openai_client.clone()))
|
||||
.app_data(web::Data::new(server_state.clone()));
|
||||
|
||||
let mut route_builder = RouteBuilder::from_app(app);
|
||||
|
||||
let app = route_builder
|
||||
.add_get("/", get_root_index)
|
||||
.add_get("/get_next_audio", get_next_audio_file_handler)
|
||||
.add_post("/next_audio", next_audio_file_handler)
|
||||
.add_get("/openai", openai_inference_handler)
|
||||
.add_get("/news_story/next_story", get_next_news_story_handler)
|
||||
.add_get("/news_story/audio", get_next_news_story_audio_file_handler)
|
||||
.add_get("/current_level", get_current_level_handler)
|
||||
.into_app();
|
||||
|
||||
/*app.service(
|
||||
web::resource("/next")
|
||||
.route(web::get().to(get_next_audio_file_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok()))
|
||||
)
|
||||
.service(
|
||||
web::resource("/next")
|
||||
.route(web::get().to(get_next_audio_file_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok()))
|
||||
)*/
|
||||
|
||||
app
|
||||
})
|
||||
.bind("localhost:23333")?
|
||||
.workers(8)
|
||||
.run()
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
pub mod handlers;
|
||||
pub mod launch_web_server;
|
||||
pub mod server_state;
|
||||
@@ -1,15 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_openai::Client;
|
||||
use sqlx::{Pool, Sqlite};
|
||||
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::app_control_state::AppControlState;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ServerState {
|
||||
pub app_control_state: Arc<AppControlState>,
|
||||
pub openai_client: Arc<Client>,
|
||||
pub save_directory: SaveDirectory,
|
||||
pub sqlite_pool: Pool<Sqlite>,
|
||||
}
|
||||
-102
@@ -1,102 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use enums::by_table::tts_render_tasks::tts_render_status::TtsRenderStatus;
|
||||
use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
|
||||
use errors::AnyhowResult;
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_awaiting_render::list_tts_render_tasks_awaiting_render;
|
||||
use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
|
||||
use sqlx::sqlite::SqlitePoolOptions;
|
||||
use strum::IntoEnumIterator;
|
||||
use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
|
||||
use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
|
||||
use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
|
||||
use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::audio::fakeyou_audio_create::process_single_record::process_single_record;
|
||||
|
||||
/// Get audio jobs and kick off FakeYou requests
|
||||
pub async fn fakeyou_audio_create_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("fakeyou_audio_create main loop");
|
||||
|
||||
while job_state.app_control_state.is_fakeyou_paused() {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
|
||||
single_job_loop_iteration(&job_state).await;
|
||||
|
||||
info!("fakeyou_audio_create loop finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(60));
|
||||
}
|
||||
}
|
||||
|
||||
// NB: No failures at this level, because we don't want to prevent progress on a stuck feed.
|
||||
async fn single_job_loop_iteration(job_state: &Arc<JobState>) {
|
||||
let statuses = vec![TtsRenderStatus::New, TtsRenderStatus::Failed];
|
||||
for status in statuses {
|
||||
scrape_jobs_of_status(status, job_state).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn scrape_jobs_of_status(status: TtsRenderStatus, job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("fakeyou_audio_create querying {:?} targets from id > {} ...", &status, last_id);
|
||||
|
||||
let query_result = list_tts_render_tasks_awaiting_render(
|
||||
status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("Render audio: {}/{}", target.story_type, target.story_token);
|
||||
|
||||
while job_state.app_control_state.is_fakeyou_paused() {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
|
||||
let result = process_single_record(&target, job_state).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_single_record;
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use fakeyou_client::api::tts_inference::CreateTtsInferenceRequest;
|
||||
use idempotency::uuid::generate_random_uuid;
|
||||
use log::{error, info};
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::tts_render_task::TtsRenderTask;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_successfully_submitted::Args as SuccessArgs;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_successfully_submitted::update_tts_render_task_successfully_submitted;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_unsuccessfully_submitted::Args as UnsuccessfulArgs;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_unsuccessfully_submitted::update_tts_render_task_unsuccessfully_submitted;
|
||||
use tokens::tokens::tts_models::TtsModelToken;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
|
||||
pub async fn process_single_record(target: &TtsRenderTask, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
|
||||
info!("Posting FakeYou create TTS request...");
|
||||
|
||||
let tts_model_token = TtsModelToken::new_from_str(&target.tts_voice_identifier);
|
||||
let uuid_idempotency_token = generate_random_uuid();
|
||||
let tts_render_attempts = target.tts_render_attempts + 1;
|
||||
|
||||
let result = job_state
|
||||
.fakeyou_client
|
||||
.create_tts_inference_with_backoff(CreateTtsInferenceRequest {
|
||||
uuid_idempotency_token: &uuid_idempotency_token,
|
||||
tts_model_token: &tts_model_token,
|
||||
inference_text: &target.full_text,
|
||||
}).await;
|
||||
|
||||
match result {
|
||||
Err(err) => {
|
||||
error!("problem submitting to FakeYou: {:}", err);
|
||||
|
||||
update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs {
|
||||
tts_render_task_token: &target.token,
|
||||
tts_render_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?;
|
||||
},
|
||||
Ok(res) => {
|
||||
if !res.success {
|
||||
error!("unknown problem in submitting TTS request to FakeYou");
|
||||
|
||||
update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs {
|
||||
tts_render_task_token: &target.token,
|
||||
tts_render_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?;
|
||||
} else {
|
||||
update_tts_render_task_successfully_submitted(SuccessArgs {
|
||||
tts_render_task_token: &target.token,
|
||||
tts_inference_job_token: &res.inference_job_token.ok_or(anyhow!("no token"))?,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?;
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use enums::by_table::tts_render_tasks::tts_render_status::TtsRenderStatus;
|
||||
use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus;
|
||||
use errors::AnyhowResult;
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_awaiting_download::list_tts_render_tasks_awaiting_download;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::list_tts_render_tasks_awaiting_render::list_tts_render_tasks_awaiting_render;
|
||||
use sqlite_queries::queries::by_table::web_scraping_targets::insert_web_scraping_target::{Args, insert_web_scraping_target};
|
||||
use sqlx::sqlite::SqlitePoolOptions;
|
||||
use strum::IntoEnumIterator;
|
||||
use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
|
||||
use web_scrapers::payloads::web_scraping_target::WebScrapingTarget;
|
||||
use web_scrapers::sites::cnn::cnn_indexer::{cnn_indexer, CnnFeed};
|
||||
use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed};
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::audio::fakeyou_audio_download::process_single_record::process_single_record;
|
||||
|
||||
/// See if jobs are complete and download the audio.
|
||||
pub async fn fakeyou_audio_download_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("fakeyou_audio_download main loop");
|
||||
|
||||
while job_state.app_control_state.is_fakeyou_paused() {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
|
||||
query_and_process_jobs(&job_state).await;
|
||||
|
||||
info!("fakeyou_audio_download loop finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(60));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async fn query_and_process_jobs(job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("fakeyou_audio_create querying targets from id > {} ...", last_id);
|
||||
|
||||
let query_result = list_tts_render_tasks_awaiting_download(
|
||||
last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("Download audio: {}/{}", target.story_type, target.story_token);
|
||||
|
||||
while job_state.app_control_state.is_fakeyou_paused() {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
|
||||
let result = process_single_record(&target, job_state).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_single_record;
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use fakeyou_client::api::tts_inference::{CreateTtsInferenceRequest, TtsInferenceJobStatusStatePayload};
|
||||
use fakeyou_client::get_audio_url::get_audio_url;
|
||||
use log::{error, info};
|
||||
use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::list::tts_render_task::TtsRenderTask;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_successfully_downloaded::Args;
|
||||
use sqlite_queries::queries::by_table::tts_render_tasks::update::update_tts_render_task_successfully_downloaded::update_tts_render_task_successfully_downloaded;
|
||||
use tokens::tokens::news_stories::NewsStoryToken;
|
||||
use tokens::tokens::tts_models::TtsModelToken;
|
||||
use tokens::tokens::tts_render_tasks::TtsRenderTaskToken;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
|
||||
pub async fn process_single_record(target: &TtsRenderTask, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
|
||||
info!("Attempting to download audio...");
|
||||
|
||||
let inference_job_token = match &target.maybe_inference_job_token {
|
||||
None => return Err(anyhow!("tts result has no inference job token")),
|
||||
Some(token) => token.to_string(),
|
||||
};
|
||||
|
||||
let result = job_state
|
||||
.fakeyou_client
|
||||
.get_tts_inference_job_status(&inference_job_token).await;
|
||||
|
||||
match result {
|
||||
Err(err) => {
|
||||
error!("problem submitting to FakeYou: {:}", err);
|
||||
|
||||
// TODO: Errors
|
||||
//update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs {
|
||||
// tts_render_task_token: &target.token,
|
||||
// tts_render_attempts,
|
||||
// sqlite_pool: &job_state.sqlite_pool,
|
||||
//}).await?;
|
||||
},
|
||||
Ok(res) => {
|
||||
match res.state {
|
||||
None => {
|
||||
error!("unknown problem in checking FakeYou TTS inference job status");
|
||||
// TODO: Errors
|
||||
//update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs {
|
||||
// tts_render_task_token: &target.token,
|
||||
// tts_render_attempts,
|
||||
// sqlite_pool: &job_state.sqlite_pool,
|
||||
//}).await?;
|
||||
}
|
||||
Some(payload) => {
|
||||
process_download(&target, &payload, job_state).await?;
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_download(target: &TtsRenderTask, payload: &TtsInferenceJobStatusStatePayload, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
let tts_result_token = match &payload.maybe_result_token {
|
||||
None => return Ok(()), // TODO: Wait
|
||||
Some(result_token) => result_token.to_string(),
|
||||
};
|
||||
|
||||
let bucket_path = match &payload.maybe_public_bucket_wav_audio_path {
|
||||
None => return Ok(()), // TODO: Wait
|
||||
Some(path) => path.to_string(),
|
||||
};
|
||||
|
||||
let audio_url = get_audio_url(&bucket_path, true);
|
||||
let news_story_token = NewsStoryToken::new_from_str(&target.story_token);
|
||||
|
||||
{
|
||||
let directory = job_state.save_directory.directory_for_audio(&news_story_token)?;
|
||||
std::fs::create_dir_all(&directory)?;
|
||||
}
|
||||
|
||||
let download_filename = job_state.save_directory
|
||||
.audio_wav_file_for_news_story(&news_story_token, target.sequence_order)?;
|
||||
|
||||
{
|
||||
let response = reqwest::get(&audio_url).await?;
|
||||
let mut file = std::fs::File::create(&download_filename)?;
|
||||
let mut content = Cursor::new(response.bytes().await?);
|
||||
std::io::copy(&mut content, &mut file)?;
|
||||
}
|
||||
|
||||
let audio_info = {
|
||||
let bytes = std::fs::read(&download_filename)?;
|
||||
decode_basic_audio_bytes_info(
|
||||
&bytes,
|
||||
Some("audio/wav"),
|
||||
Some(".wav"))?
|
||||
};
|
||||
|
||||
let audio_duration_millis = audio_info.duration_millis.unwrap_or(0) as i64;
|
||||
|
||||
update_tts_render_task_successfully_downloaded(Args {
|
||||
tts_render_task_token: &target.token,
|
||||
tts_result_token: &tts_result_token,
|
||||
result_url: &audio_url,
|
||||
audio_duration_millis,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod fakeyou_audio_create;
|
||||
pub mod fakeyou_audio_download;
|
||||
@@ -1,3 +0,0 @@
|
||||
pub mod audio;
|
||||
pub mod news_stories;
|
||||
pub mod web;
|
||||
@@ -1,5 +0,0 @@
|
||||
pub mod phase_1;
|
||||
pub mod phase_2;
|
||||
pub mod phase_3;
|
||||
pub mod phase_4;
|
||||
pub mod phase_5;
|
||||
@@ -1 +0,0 @@
|
||||
pub mod news_story_greenlighting;
|
||||
-75
@@ -1,75 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_greenlight::list_news_story_productions_awaiting_greenlight;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::news_stories::phase_1::news_story_greenlighting::process_target_record::process_target_record;
|
||||
|
||||
/// Decide which potential news stories to greenlight
|
||||
pub async fn news_story_greenlighting_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("news_story_greenlighting main loop iteration start");
|
||||
|
||||
query_and_process_all_batches(&job_state).await;
|
||||
|
||||
debug!("news_story_greenlighting loop finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
}
|
||||
|
||||
async fn query_and_process_all_batches(job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("news_story_greenlighting querying targets from id > {} ...", last_id);
|
||||
|
||||
let query_result = list_news_story_productions_awaiting_greenlight(
|
||||
last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("process greenlighting for {}", target.news_story_token);
|
||||
let result = process_target_record(&target, job_state).await;
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
-3
@@ -1,3 +0,0 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_target_record;
|
||||
pub mod site_handlers;
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use enums::common::sqlite::skip_reason::SkipReason;
|
||||
use enums::common::sqlite::web_content_type::WebContentType;
|
||||
use errors::AnyhowResult;
|
||||
use log::info;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_production_greenlit_status::{Args, update_news_story_production_greenlit_status};
|
||||
|
||||
use crate::persistence::load_scraped_result::load_scraped_result;
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::news_stories::phase_1::news_story_greenlighting::site_handlers::cnn_greenlighting::cnn_greenlighting;
|
||||
|
||||
pub async fn process_target_record(target: &NewsStoryProductionItem, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
|
||||
// TODO: Decide which articles to skip with heuristics (and soon AI)!
|
||||
|
||||
let mut maybe_skip_reason = None;
|
||||
|
||||
{
|
||||
let scraping_result = load_scraped_result(
|
||||
&target.original_news_canonical_url,
|
||||
&job_state.save_directory).await?;
|
||||
|
||||
let body_text = scraping_result.body_text.trim();
|
||||
|
||||
if body_text.is_empty() || body_text.len() < 50 {
|
||||
maybe_skip_reason = Some(SkipReason::EmptyContent);
|
||||
}
|
||||
}
|
||||
|
||||
if maybe_skip_reason.is_none() {
|
||||
maybe_skip_reason = match target.web_content_type {
|
||||
WebContentType::CnnArticle => cnn_greenlighting(target),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
let is_greenlit = maybe_skip_reason.is_none();
|
||||
|
||||
info!("news story {} greenlit: {}", &target.news_story_token, is_greenlit);
|
||||
|
||||
update_news_story_production_greenlit_status(Args {
|
||||
news_story_token: &target.news_story_token,
|
||||
maybe_skip_reason,
|
||||
is_greenlit,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
use enums::common::sqlite::skip_reason::SkipReason;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
|
||||
|
||||
pub fn cnn_greenlighting(target: &NewsStoryProductionItem) -> Option<SkipReason> {
|
||||
|
||||
if target.original_news_canonical_url.contains("/deals/")
|
||||
|| target.original_news_canonical_url.contains("cnn-underscored") {
|
||||
// We got a 3-minute long ad for Levi's with the "/deals/" URL.
|
||||
// CNN underscored is nothing but ads: https://www.cnn.com/cnn-underscored
|
||||
return Some(SkipReason::Advertisement);
|
||||
}
|
||||
|
||||
if target.original_news_canonical_url.contains("/videos/") {
|
||||
// Video content tends to have little text
|
||||
return Some(SkipReason::VideoContent);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
-1
@@ -1 +0,0 @@
|
||||
pub mod cnn_greenlighting;
|
||||
@@ -1,4 +0,0 @@
|
||||
pub mod news_story_image_generation;
|
||||
pub mod news_story_llm_category_summary;
|
||||
pub mod news_story_llm_rendition;
|
||||
pub mod news_story_llm_title_summary;
|
||||
-88
@@ -1,88 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
|
||||
use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus;
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_image_generation::list_news_story_productions_awaiting_image_generation;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::news_stories::phase_2::news_story_image_generation::process_single_item::process_single_item;
|
||||
|
||||
// TODO: Stop copy/pasting this. Come up with a generalizable task runner with dependencies, etc.
|
||||
|
||||
/// Generate images for the news story
|
||||
pub async fn news_story_image_generation_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("image generation main loop");
|
||||
|
||||
single_job_loop_iteration(&job_state).await;
|
||||
|
||||
debug!("image generation finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
// NB: No failures at this level, because we don't want to prevent progress on a stuck feed.
|
||||
async fn single_job_loop_iteration(job_state: &Arc<JobState>) {
|
||||
let statuses = vec![AwaitableJobStatus::ReadyWaiting, AwaitableJobStatus::RetryablyFailed];
|
||||
for status in statuses {
|
||||
rendition_jobs_of_status(status, job_state).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn rendition_jobs_of_status(status: AwaitableJobStatus, job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("image generation querying {:?} targets from id > {} ...", &status, last_id);
|
||||
|
||||
let query_result = list_news_story_productions_awaiting_image_generation(
|
||||
status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("Rendition for target: {:?}", target.original_news_canonical_url);
|
||||
|
||||
let result = process_single_item(&target, job_state).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
|
||||
thread::sleep(Duration::from_secs(5)); // NB: We probably (?) need to throttle API calls to OpenAI
|
||||
}
|
||||
}
|
||||
}
|
||||
-2
@@ -1,2 +0,0 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_single_item;
|
||||
-130
@@ -1,130 +0,0 @@
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_openai::Client;
|
||||
use async_openai::error::OpenAIError;
|
||||
use async_openai::types::{CreateCompletionRequestArgs, CreateImageRequest, ImageData, ImageSize, ResponseFormat};
|
||||
use base64::{Engine as _, engine::general_purpose};
|
||||
use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use log::{debug, error, info, warn};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_image_generation_status::Args;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_image_generation_status::update_news_story_image_generation_status;
|
||||
use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
|
||||
|
||||
use crate::persistence::image_generation_debug_data::ImageGenerationDebugData;
|
||||
use crate::persistence::load_scraped_result::load_scraped_result;
|
||||
use crate::persistence::rendition_data::RenditionData;
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::job_state::JobState;
|
||||
|
||||
pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
|
||||
let scraping_result = load_scraped_result(
|
||||
&target.original_news_canonical_url,
|
||||
&job_state.save_directory).await?;
|
||||
|
||||
let title = scraping_result.maybe_title
|
||||
.unwrap_or("kitsune walking in a forest".to_string());
|
||||
|
||||
let dalle_prompt = generate_dalle_prompt_with_gpt(&title, &job_state.openai_client).await?;
|
||||
|
||||
let create_request = CreateImageRequest {
|
||||
n: Some(1),
|
||||
prompt: dalle_prompt.clone(),
|
||||
size: Some(ImageSize::S1024x1024),
|
||||
response_format: Some(ResponseFormat::B64Json),
|
||||
user: Some("article".to_string()),
|
||||
};
|
||||
|
||||
// TODO: Save both prompts for debugging.
|
||||
|
||||
let response = job_state.openai_client
|
||||
.images()
|
||||
.create(create_request)
|
||||
.await?;
|
||||
|
||||
let maybe_image = response.data.get(0)
|
||||
.map(|image| image.as_ref());
|
||||
|
||||
match maybe_image {
|
||||
Some(ImageData::B64Json(image_data)) => {
|
||||
let decoded_bytes = general_purpose::STANDARD.decode(image_data.as_bytes())?;
|
||||
|
||||
let image_filename = job_state.save_directory
|
||||
.generated_headline_image_for_webpage_url(&target.original_news_canonical_url)?;
|
||||
|
||||
let mut file = std::fs::File::create(image_filename)?;
|
||||
file.write_all(&decoded_bytes)?;
|
||||
}
|
||||
_ => {},
|
||||
}
|
||||
|
||||
let debug_data = ImageGenerationDebugData {
|
||||
image_generation_prompt: dalle_prompt.to_string(),
|
||||
maybe_preconditioning_prompt: None, // TODO: Plumb this through.
|
||||
};
|
||||
|
||||
let yaml_filename = job_state.save_directory
|
||||
.image_generation_prompt_debug_file_webpage_url(&target.original_news_canonical_url)?;
|
||||
|
||||
let _r = debug_data.write_to_yaml_file(yaml_filename)?;
|
||||
|
||||
let image_generation_attempts = target.image_generation_attempts + 1;
|
||||
|
||||
update_news_story_image_generation_status(Args {
|
||||
news_story_token: &target.news_story_token,
|
||||
is_successful: true,
|
||||
image_generation_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?; // NB: If these queries fail, we could get stuck in retry hell.
|
||||
|
||||
// {
|
||||
// let yaml_filename = job_state.save_directory
|
||||
// .rendition_file_for_webpage_url(&target.original_news_canonical_url)?;
|
||||
// rendition_data.write_to_yaml_file(yaml_filename)?;
|
||||
// }
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn generate_dalle_prompt_with_gpt(headline: &str, openai_client: &Arc<Client>) -> AnyhowResult<String> {
|
||||
info!("Building OpenAI Request (GPT for Dall-E prompt)...");
|
||||
|
||||
let mut prompt = r#"
|
||||
Turn this headline into a five word or less summary, then add five descriptive adjectives,
|
||||
then add the most fitting art style or director. Then describe a setting and potential objects
|
||||
in this scene. Describe what a photo or painting of this might look like:
|
||||
"#.to_string();
|
||||
|
||||
prompt.push_str("\n\n");
|
||||
prompt.push_str(headline);
|
||||
|
||||
let request = CreateCompletionRequestArgs::default()
|
||||
.model("text-davinci-003")
|
||||
.prompt(prompt)
|
||||
.max_tokens(2500_u16)
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
error!("Error building request: {:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
debug!("Making OpenAI Request (GPT for Dall-E prompt)...");
|
||||
|
||||
let response = openai_client
|
||||
.completions()
|
||||
.create(request)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("OpenAI Error: {:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
let response_text = response.choices.get(0)
|
||||
.map(|option| option.text.clone())
|
||||
.unwrap_or(headline.to_string());
|
||||
|
||||
Ok(response_text)
|
||||
}
|
||||
-86
@@ -1,86 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
|
||||
use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus;
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_llm_rendition::list_news_story_productions_awaiting_llm_rendition;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::news_stories::phase_2::news_story_llm_category_summary::process_single_item::process_single_item;
|
||||
|
||||
/// Project scraped, summarize the categories.
|
||||
pub async fn news_story_llm_category_summary_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("news_story_llm_category_summary_main_loop main loop");
|
||||
|
||||
single_job_loop_iteration(&job_state).await;
|
||||
|
||||
debug!("llm_category_summary loop finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
// NB: No failures at this level, because we don't want to prevent progress on a stuck feed.
|
||||
async fn single_job_loop_iteration(job_state: &Arc<JobState>) {
|
||||
let statuses = vec![AwaitableJobStatus::ReadyWaiting, AwaitableJobStatus::RetryablyFailed];
|
||||
for status in statuses {
|
||||
rendition_jobs_of_status(status, job_state).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn rendition_jobs_of_status(status: AwaitableJobStatus, job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("web_rendition querying {:?} targets from id > {} ...", &status, last_id);
|
||||
|
||||
let query_result = list_news_story_productions_awaiting_llm_rendition(
|
||||
status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("Rendition for target: {:?}", target.original_news_canonical_url);
|
||||
|
||||
let result = process_single_item(&target, job_state).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
|
||||
thread::sleep(Duration::from_secs(5)); // NB: We probably (?) need to throttle API calls to OpenAI
|
||||
}
|
||||
}
|
||||
}
|
||||
-2
@@ -1,2 +0,0 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_single_item;
|
||||
-148
@@ -1,148 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_openai::Client;
|
||||
use async_openai::error::OpenAIError;
|
||||
use async_openai::types::CreateCompletionRequestArgs;
|
||||
use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use log::{error, info, warn};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::news_story_production_item::NewsStoryProductionItem;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_llm_categorization_status::Args;
|
||||
use sqlite_queries::queries::by_table::news_story_productions::update::update_news_story_llm_categorization_status::update_news_story_llm_categorization_status;
|
||||
use web_scrapers::payloads::web_scraping_result::ScrapedWebArticle;
|
||||
|
||||
use crate::gpt_prompts::categorization_gpt_prompt::CategorizationGptPrompt;
|
||||
use crate::persistence::load_scraped_result::load_scraped_result;
|
||||
use crate::persistence::rendition_data::RenditionData;
|
||||
use crate::persistence::save_directory::SaveDirectory;
|
||||
use crate::shared_state::job_state::JobState;
|
||||
|
||||
pub async fn process_single_item(target: &NewsStoryProductionItem, job_state: &Arc<JobState>) -> AnyhowResult<()> {
|
||||
|
||||
let scraping_result = load_scraped_result(
|
||||
&target.original_news_canonical_url,
|
||||
&job_state.save_directory).await?;
|
||||
|
||||
let prompt = CategorizationGptPrompt::new(&scraping_result.paragraphs);
|
||||
let prompt = prompt.make_prompt();
|
||||
|
||||
let rendition_result = call_openai_gpt(
|
||||
&target.original_news_canonical_url,
|
||||
&prompt,
|
||||
&job_state.openai_client
|
||||
).await;
|
||||
|
||||
let llm_categorization_attempts = target.llm_categorization_attempts + 1;
|
||||
|
||||
let rendition_data = match rendition_result {
|
||||
Err(err) => {
|
||||
error!("Error using GPT: {:?}", err);
|
||||
|
||||
match &err {
|
||||
WrappedOpenAiError::Other(_) => {}
|
||||
WrappedOpenAiError::OpenAiError(inner) => {
|
||||
match inner {
|
||||
OpenAIError::Reqwest(e) => { error!("Request error: {:?}", e) }
|
||||
OpenAIError::ApiError(e) => { error!("API error: {:?}", e) }
|
||||
OpenAIError::JSONDeserialize(e) => { error!("JSON error: {:?}", e) }
|
||||
OpenAIError::FileSaveError(e) => { error!("File save error: {:?}", e) }
|
||||
OpenAIError::FileReadError(e) => { error!("File read error: {:?}", e) }
|
||||
OpenAIError::StreamError(e) => { error!("Stream error: {:?}", e) }
|
||||
OpenAIError::InvalidArgument(e) => { error!("Invalid args error: {:?}", e) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
update_news_story_llm_categorization_status(Args {
|
||||
news_story_token: &target.news_story_token,
|
||||
is_successful: false,
|
||||
maybe_categorization: None,
|
||||
llm_categorization_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?; // NB: If these queries fail, we could get stuck in retry hell.
|
||||
|
||||
return Err(anyhow!("OpenAI Rendition Error for {}: {:?}", target.news_story_token, err));
|
||||
}
|
||||
Ok(rendition_data) => rendition_data,
|
||||
};
|
||||
|
||||
let categorization = rendition_data.response.trim();
|
||||
|
||||
// NB: GPT often completes the prompt we used itself, so we grab the last line (presumably output)
|
||||
let categorization = categorization.split("\n")
|
||||
.map(|line| line.trim().to_string())
|
||||
.filter(|line| line.len() > 5)
|
||||
.last()
|
||||
.unwrap_or("".to_string());
|
||||
|
||||
if categorization.len() < 5 {
|
||||
update_news_story_llm_categorization_status(Args {
|
||||
news_story_token: &target.news_story_token,
|
||||
is_successful: false,
|
||||
maybe_categorization: None,
|
||||
llm_categorization_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?; // NB: If these queries fail, we could get stuck in retry hell.
|
||||
|
||||
return Err(anyhow!("too short"));
|
||||
}
|
||||
|
||||
update_news_story_llm_categorization_status(Args {
|
||||
news_story_token: &target.news_story_token,
|
||||
is_successful: true,
|
||||
maybe_categorization: Some(categorization.to_string()),
|
||||
llm_categorization_attempts,
|
||||
sqlite_pool: &job_state.sqlite_pool,
|
||||
}).await?; // NB: If these queries fail, we could get stuck in retry hell.
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum WrappedOpenAiError {
|
||||
OpenAiError(OpenAIError),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
async fn call_openai_gpt(url: &str, prompt: &str, openai_client: &Arc<Client>) -> Result<RenditionData, WrappedOpenAiError> {
|
||||
warn!("Building OpenAI Request...");
|
||||
let request = CreateCompletionRequestArgs::default()
|
||||
.model("text-davinci-003")
|
||||
.prompt(prompt)
|
||||
.max_tokens(2500_u16)
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
error!("Error building request: {:?}", err);
|
||||
WrappedOpenAiError::Other(format!("error: {:?}", err))
|
||||
})?;
|
||||
|
||||
warn!("Making OpenAI Request...");
|
||||
let response = openai_client
|
||||
.completions()
|
||||
.create(request)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("OpenAI Error: {:?}", err);
|
||||
WrappedOpenAiError::OpenAiError(err)
|
||||
})?;
|
||||
|
||||
info!("Open AI response: {:?}", response);
|
||||
|
||||
let rendition_text = response.choices.get(0)
|
||||
.map(|option| option.text.clone())
|
||||
.unwrap_or("".to_string());
|
||||
|
||||
Ok(RenditionData {
|
||||
original_content_url: url.to_string(),
|
||||
original_prompt: prompt.to_string(),
|
||||
response: rendition_text,
|
||||
})
|
||||
}
|
||||
|
||||
//fn next_status(rendition_attempts: i64) -> RenditionStatus {
|
||||
// if rendition_attempts >= 2 {
|
||||
// RenditionStatus::PermanentlyFailed
|
||||
// } else {
|
||||
// RenditionStatus::Failed
|
||||
// }
|
||||
//}
|
||||
-86
@@ -1,86 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use enums::by_table::web_rendition_targets::rendition_status::RenditionStatus;
|
||||
use enums::common::sqlite::awaitable_job_status::AwaitableJobStatus;
|
||||
use log::{debug, error, info};
|
||||
use sqlite_queries::queries::by_table::news_story_productions::list::list_news_story_productions_awaiting_llm_rendition::list_news_story_productions_awaiting_llm_rendition;
|
||||
|
||||
use crate::shared_state::job_state::JobState;
|
||||
use crate::workers::news_stories::phase_2::news_story_llm_rendition::process_single_item::process_single_item;
|
||||
|
||||
/// Project scraped, etc. content into the desired shape with GPT.
|
||||
pub async fn news_story_llm_rendition_main_loop(job_state: Arc<JobState>) {
|
||||
loop {
|
||||
debug!("news_story_llm_rendition_main_loop main loop");
|
||||
|
||||
single_job_loop_iteration(&job_state).await;
|
||||
|
||||
debug!("gpt_rendition loop finished; waiting...");
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
// NB: No failures at this level, because we don't want to prevent progress on a stuck feed.
|
||||
async fn single_job_loop_iteration(job_state: &Arc<JobState>) {
|
||||
let statuses = vec![AwaitableJobStatus::ReadyWaiting, AwaitableJobStatus::RetryablyFailed];
|
||||
for status in statuses {
|
||||
rendition_jobs_of_status(status, job_state).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn rendition_jobs_of_status(status: AwaitableJobStatus, job_state: &Arc<JobState>) {
|
||||
const BATCH_SIZE : i64 = 10;
|
||||
|
||||
let mut last_id = 0;
|
||||
let mut failure_count = 0;
|
||||
|
||||
loop {
|
||||
// NB: Protect sqlite from contention.
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
|
||||
debug!("web_rendition querying {:?} targets from id > {} ...", &status, last_id);
|
||||
|
||||
let query_result = list_news_story_productions_awaiting_llm_rendition(
|
||||
status, last_id, BATCH_SIZE, &job_state.sqlite_pool).await;
|
||||
|
||||
let targets = match query_result {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
error!("failure querying batch: {:?}", err);
|
||||
|
||||
failure_count += 1;
|
||||
|
||||
// NB: Don't starve progress.
|
||||
if failure_count > 2 {
|
||||
failure_count = 0;
|
||||
last_id += 1;
|
||||
} else if failure_count > 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
failure_count = 0;
|
||||
|
||||
if targets.is_empty() {
|
||||
return; // Done with batches.
|
||||
}
|
||||
|
||||
for target in targets {
|
||||
debug!("Rendition for target: {:?}", target.original_news_canonical_url);
|
||||
|
||||
let result = process_single_item(&target, job_state).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error processing target: {:?}", err);
|
||||
}
|
||||
|
||||
last_id = target.id;
|
||||
|
||||
thread::sleep(Duration::from_secs(5)); // NB: We probably (?) need to throttle API calls to OpenAI
|
||||
}
|
||||
}
|
||||
}
|
||||
Loaded 100 of 154 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user