From 5a031b1b803964eb2202bc0934679a135698c8f2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 6 Feb 2023 04:49:28 -0500 Subject: [PATCH] indexer flow --- .../src/workers/web_index_ingestion/main_loop.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs index e4ee785107..eadb3eebb6 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/web_index_ingestion/main_loop.rs @@ -13,14 +13,16 @@ use web_scrapers::payloads::web_scraping_target::WebScrapingTarget; use web_scrapers::sites::techcrunch::techcrunch_indexer::{techcrunch_indexer, TechcrunchFeed}; pub async fn web_index_ingestion_main_loop(job_state: Arc) { - loop { info!("web_index_ingestion main loop"); match single_iteration(&job_state).await { - Ok(_) => thread::sleep(Duration::from_secs(300)), + Ok(_) => { + info!("web_index_ingestion loop finished; waiting..."); + thread::sleep(Duration::from_secs(10)) + }, Err(err) => { - error!("Error with indexing: {:?}", err); + error!("web_index_ingestion - error with indexing: {:?}", err); thread::sleep(Duration::from_secs(60)); } } @@ -32,9 +34,9 @@ async fn single_iteration(job_state: &Arc) -> AnyhowResult<()> { for variant in TechcrunchFeed::iter() { let index_targets = techcrunch_indexer(variant).await?; insert_targets(job_state, &index_targets).await?; + thread::sleep(Duration::from_secs(2)); } - //let index_targets = techcrunch_indexer(TechcrunchFeed::Main).await?; //insert_targets(&index_targets).await?;