From e428f0ae9d0041657eea9575fcc159b1920ffe05 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 29 Feb 2024 19:34:29 -0500 Subject: [PATCH] logging for debug (why is r2d2 breaking?) --- crates/service/job/inference_job/src/main.rs | 10 ++++++++-- .../job/inference_job/src/util/instrumentation.rs | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 0b0abc000a..223030abc7 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -139,13 +139,15 @@ async fn main() -> AnyhowResult<()> { "MYSQL_URL", DEFAULT_MYSQL_CONNECTION_STRING); - info!("Connecting to database..."); + info!("Connecting to MySQL database..."); let mysql_pool = MySqlPoolOptions::new() .max_connections(2) .connect(&db_connection_string) .await?; + info!("Connected to MySQL."); + let common_env = CommonEnv::read_from_env()?; let sidecar_max_synthesizer_models = easyenv::get_env_num( @@ -191,8 +193,12 @@ async fn main() -> AnyhowResult<()> { let maybe_keepalive_redis_pool = match easyenv::get_env_string_optional("REDIS_FOR_KEEPALIVE_URL") { - None => None, + None => { + warn!("Redis for job keepalive is DISABLED! This might break some jobs."); + None + }, Some(redis_url) => { + info!("Connecting to Redis for keepalive signals... {}", redis_url); let redis_manager = RedisConnectionManager::new(redis_url)?; let redis_pool = r2d2::Pool::builder().build(redis_manager)?; Some(redis_pool) diff --git a/crates/service/job/inference_job/src/util/instrumentation.rs b/crates/service/job/inference_job/src/util/instrumentation.rs index a3c806ce06..d269a4d696 100644 --- a/crates/service/job/inference_job/src/util/instrumentation.rs +++ b/crates/service/job/inference_job/src/util/instrumentation.rs @@ -1,5 +1,5 @@ use std::time::Duration; -use log::warn; +use log::{info, warn}; use opentelemetry::KeyValue; use opentelemetry::metrics::{Counter, Histogram, Meter, Unit}; use opentelemetry_otlp::WithExportConfig; @@ -19,6 +19,8 @@ pub struct JobInstrumentLabels { pub fn init_otel_metrics_pipeline( job_instrument_labels: JobInstrumentLabels, ) -> Result<(), opentelemetry::metrics::MetricsError> { + info!("Setting up otel metrics pipeline..."); + let provider = opentelemetry_otlp::new_pipeline() .metrics(opentelemetry_sdk::runtime::Tokio) // TODO: 1. read host from env 2. Single pod of otel-collector is probably not good enough, run daemonset?