From a31ecc01abb1997d50deebe438b0d2c3c0906e97 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 17 Jun 2023 11:43:25 -0700 Subject: [PATCH] configurable health threshold --- .../src/http_server/endpoints/health_check_handler.rs | 6 +++--- .../src/http_server/http_server_shared_state.rs | 1 + .../job/inference_job/src/http_server/run_http_server.rs | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/service/job/inference_job/src/http_server/endpoints/health_check_handler.rs b/crates/service/job/inference_job/src/http_server/endpoints/health_check_handler.rs index 21027ebd8a..e0bc9f2434 100644 --- a/crates/service/job/inference_job/src/http_server/endpoints/health_check_handler.rs +++ b/crates/service/job/inference_job/src/http_server/endpoints/health_check_handler.rs @@ -63,9 +63,6 @@ pub async fn get_health_check_handler( HealthCheckError::ServerError })?; - // TODO: Configurability - let is_healthy = job_stats.consecutive_failure_count < 10; - let total_tries = job_stats.total_failure_count.saturating_add(job_stats.total_success_count); let total_success_ratio = if total_tries > 0 { @@ -80,6 +77,9 @@ pub async fn get_health_check_handler( 0.0 }; + let is_healthy = + job_stats.consecutive_failure_count < server_state.consecutive_failure_unhealthy_threshold; + let response = HealthCheckResponse { success: true, is_healthy, diff --git a/crates/service/job/inference_job/src/http_server/http_server_shared_state.rs b/crates/service/job/inference_job/src/http_server/http_server_shared_state.rs index 7ab40d5e39..534526580f 100644 --- a/crates/service/job/inference_job/src/http_server/http_server_shared_state.rs +++ b/crates/service/job/inference_job/src/http_server/http_server_shared_state.rs @@ -3,4 +3,5 @@ use jobs_common::job_stats::JobStats; #[derive(Clone)] pub struct HttpServerSharedState { pub job_stats: JobStats, + pub consecutive_failure_unhealthy_threshold: u64, } diff --git a/crates/service/job/inference_job/src/http_server/run_http_server.rs b/crates/service/job/inference_job/src/http_server/run_http_server.rs index b3f744d18f..3a3e79e2ec 100644 --- a/crates/service/job/inference_job/src/http_server/run_http_server.rs +++ b/crates/service/job/inference_job/src/http_server/run_http_server.rs @@ -27,6 +27,8 @@ pub fn run_http_server(args: CreateServerArgs) -> AnyhowResult let server_state = HttpServerSharedState { job_stats: args.job_stats.clone(), + consecutive_failure_unhealthy_threshold: + easyenv::get_env_num("CONSECUTIVE_FAILURE_UNHEALTHY_THRESHOLD", 3)?, }; let server_state_arc = web::Data::new(Arc::new(server_state));