diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index fb5a4a0248..33b1e49b49 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -20,10 +20,12 @@ use crate::shared_state::app_control_state::AppControlState; use crate::shared_state::job_state::JobState; use crate::startup_args::get_startup_args; use crate::web_server::launch_web_server::{launch_web_server, LaunchWebServerArgs}; +use crate::workers::news_stories::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop; use crate::workers::news_stories::news_story_greenlighting::main_loop::news_story_greenlighting_main_loop; use crate::workers::news_stories::news_story_llm_rendition::main_loop::news_story_llm_rendition_main_loop; -use crate::workers::web_content_scraping::main_loop::web_content_scraping_main_loop; -use crate::workers::web_index_ingestion::main_loop::web_index_ingestion_main_loop; +use crate::workers::web::web_content_scraping::main_loop::web_content_scraping_main_loop; +use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; +use crate::workers::web::web_index_ingestion::main_loop::web_index_ingestion_main_loop; use enums::by_table::web_scraping_targets::web_content_type::WebContentType; use errors::AnyhowResult; use log::info; @@ -36,8 +38,6 @@ use tokio::runtime::{Builder, Runtime}; use web_scrapers::sites::cnn::cnn_article_scraper::cnn_article_scraper; use web_scrapers::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; use web_scrapers::sites::theguardian::theguardian_scraper::theguardian_scraper_test; -use workers::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; -use crate::workers::news_stories::news_story_audio_preprocessing::main_loop::news_story_audio_preprocessing_main_loop; //#[tokio::main] //pub async fn main() -> AnyhowResult<()> { diff --git a/crates/programs/aichatbot-sidecar/src/workers/fakeyou_audio_generation/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/fakeyou_audio_generation/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/programs/aichatbot-sidecar/src/workers/audio/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/mod.rs new file mode 100644 index 0000000000..7dd1ee36ce --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/audio/mod.rs @@ -0,0 +1,2 @@ +pub mod fakeyou_audio_create; +pub mod fakeyou_audio_download; diff --git a/crates/programs/aichatbot-sidecar/src/workers/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/mod.rs index 8187348141..a73d738a31 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/mod.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/mod.rs @@ -1,4 +1,3 @@ -pub mod fakeyou_audio_generation; +pub mod audio; pub mod news_stories; -pub mod web_content_scraping; -pub mod web_index_ingestion; +pub mod web; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web/mod.rs new file mode 100644 index 0000000000..2f14cbbc78 --- /dev/null +++ b/crates/programs/aichatbot-sidecar/src/workers/web/mod.rs @@ -0,0 +1,2 @@ +pub mod web_content_scraping; +pub mod web_index_ingestion; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/main_loop.rs similarity index 96% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/main_loop.rs index 752db4542a..33865f8a14 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/main_loop.rs @@ -1,5 +1,5 @@ use crate::shared_state::job_state::JobState; -use crate::workers::web_content_scraping::single_target::process_target_record::process_target_record; +use crate::workers::web::web_content_scraping::single_target::process_target_record::process_target_record; use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; use errors::AnyhowResult; use log::{error, info}; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/filter_scraped_result_heuristics.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/filter_scraped_result_heuristics.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/filter_scraped_result_heuristics.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/ingest_url_scrape_and_save.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/ingest_url_scrape_and_save.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/mod.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/process_target_record.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs similarity index 93% rename from crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/process_target_record.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs index 03644f0cff..9a383ca8d9 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/single_target/process_target_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web/web_content_scraping/single_target/process_target_record.rs @@ -1,6 +1,6 @@ use crate::shared_state::job_state::JobState; -use crate::workers::web_content_scraping::single_target::filter_scraped_result_heuristics::filter_scraped_result_heuristics; -use crate::workers::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; +use crate::workers::web::web_content_scraping::single_target::filter_scraped_result_heuristics::filter_scraped_result_heuristics; +use crate::workers::web::web_content_scraping::single_target::ingest_url_scrape_and_save::ingest_url_scrape_and_save; use enums::by_table::web_scraping_targets::scraping_status::ScrapingStatus; use errors::AnyhowResult; use log::error; diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/main_loop.rs diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/mod.rs similarity index 100% rename from crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/mod.rs rename to crates/programs/aichatbot-sidecar/src/workers/web/web_index_ingestion/mod.rs