diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs index 814172feb0..9a3acb388e 100644 --- a/crates/programs/aichatbot-sidecar/src/main.rs +++ b/crates/programs/aichatbot-sidecar/src/main.rs @@ -44,33 +44,6 @@ pub async fn main2() -> AnyhowResult<()> { .max_connections(5) .connect(&database_url).await?; - /* - let mut targets = Vec::new(); - - //let targets = techcrunch_scraper_test().await?; - //let targets = theguardian_scraper_test().await?; - - targets.extend(cnn_scraper_test().await?); - targets.extend(techcrunch_scraper_test().await?); - targets.extend(theguardian_scraper_test().await?); - - for target in targets.iter() { - //println!("\n\nTarget: {:?}", target); - - let _r = insert_web_scraping_target(Args { - canonical_url: &target.canonical_url, - web_content_type: target.web_content_type, - maybe_title: target.maybe_title.as_deref(), - maybe_article_full_image_url: target.maybe_full_image_url.as_deref(), - maybe_article_thumbnail_image_url: target.maybe_thumbnail_image_url.as_deref(), - sqlite_pool: &pool, - }).await; - } - */ - - //let _r = cnn_article_scraper("https://www.cnn.com/2023/02/02/tech/first-generation-iphone-auction/index.html").await?; - //let result = techcrunch_article_scraper("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/").await?; - let _ = dotenv::from_filename(".env-aichatbot-secrets").ok(); let startup_args = get_startup_args()?; let save_directory = SaveDirectory::new(&startup_args.save_directory); @@ -110,7 +83,7 @@ pub async fn main() -> AnyhowResult<()> { save_directory: save_directory.clone(), }); - info!("Starting web server..."); + info!("Starting worker threads and web server..."); let app_control_state2 = app_control_state.clone(); let openai_client2 = openai_client.clone(); @@ -118,7 +91,7 @@ pub async fn main() -> AnyhowResult<()> { // NB: both egui and imgui (which we aren't using) complain about launching on a non-main thread. // They even complain that this is impossible on Windows (and our program aims to be multiplatform) - // Thus, we launch everything else into its own thread. (TODO: Jobs + Server in one thread) + // Thus, we launch everything else into its own thread. thread::spawn(move || { let server_future = launch_web_server(LaunchWebServerArgs { app_control_state: app_control_state2, @@ -129,26 +102,35 @@ pub async fn main() -> AnyhowResult<()> { let tokio_runtime = Runtime::new()?; tokio_runtime.spawn(async { - let _r = web_index_ingestion_main_loop(job_state2).await; + let _r = web_index_ingestion_main_loop(job_state2.clone()).await; }); + // TODO: Scraping thread (needs Sqlite queries) + //tokio_runtime.spawn(async { + // // TODO... + // let _r = web_index_ingestion_main_loop(job_state2.clone()).await; + //}); + + // TODO: GPT Transformation thread + //tokio_runtime.spawn(async { + // // TODO... + //}); + + // TODO: FakeYou enrichment thread + //tokio_runtime.spawn(async { + // // TODO... + //}); + + // TODO: Final scheduling thread + //tokio_runtime.spawn(async { + // // TODO... + //}); + let runtime = actix_web::rt::System::new(); runtime.block_on(server_future) }); -// info!("Starting async processes..."); -// -// thread::spawn(move|| { -// -// tokio_runtime.block_on(async || { -// loop { -// thread::sleep(Duration::from_secs(60)) -// std::future:: -// } -// }.await) -// }); - info!("Starting GUI ..."); let _r = launch_gui(startup_args.clone(), app_control_state.clone()); diff --git a/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs b/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs index 83913d8953..fdb33ca479 100644 --- a/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs +++ b/crates/programs/aichatbot-sidecar/src/shared_state/job_state.rs @@ -1,6 +1,7 @@ use crate::persistence::save_directory::SaveDirectory; use sqlx::{Pool, Sqlite}; +#[derive(Clone)] pub struct JobState { pub sqlite_pool: Pool, pub save_directory: SaveDirectory, diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/main_loop.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs index 218637c518..6b1b262316 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web_content_scraping/mod.rs @@ -1 +1,2 @@ pub mod ingest_url_scrape_and_save; +pub mod main_loop;