diff --git a/Cargo.lock b/Cargo.lock index 80adf83ae0..b6903f8ee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.18" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -3145,6 +3145,20 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kube-pod-cleanup" +version = "0.0.1" +dependencies = [ + "config", + "easyenv", + "enums", + "errors", + "hashing", + "log", + "once_cell", + "regex", +] + [[package]] name = "language-tags" version = "0.2.2" @@ -3284,12 +3298,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru-cache" @@ -3401,9 +3412,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -4586,9 +4597,21 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.4" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", @@ -4597,9 +4620,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" [[package]] name = "remove_dir_all" diff --git a/Cargo.toml b/Cargo.toml index f1f6318592..7a0f90553a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/cli/database_backup", "crates/cli/database_cleanup", "crates/cli/dev_database_seed", + "crates/cli/kube_pod_cleanup", "crates/cli/scrape_test", # Shared libraries diff --git a/crates/cli/kube_pod_cleanup/Cargo.toml b/crates/cli/kube_pod_cleanup/Cargo.toml new file mode 100644 index 0000000000..bbad464c89 --- /dev/null +++ b/crates/cli/kube_pod_cleanup/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "kube-pod-cleanup" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "kube-pod-cleanup" +path = "src/main.rs" + +[dependencies] +# Internal +config = { path = "../../lib/deprecated/config" } +easyenv = { path = "../../lib/easyenv" } +enums = { path = "../../_types/enums" } +errors = { path = "../../lib/errors" } +hashing = { path = "../../lib/files/hashing" } + +# External +log = "0.4.20" +once_cell = "1.18.0" +regex = "1.10.0" + +[dev-dependencies] +# None yet diff --git a/crates/cli/kube_pod_cleanup/src/delete_pods.rs b/crates/cli/kube_pod_cleanup/src/delete_pods.rs new file mode 100644 index 0000000000..8a01df84bc --- /dev/null +++ b/crates/cli/kube_pod_cleanup/src/delete_pods.rs @@ -0,0 +1,23 @@ +use std::process::Command; + +use log::info; + +use errors::AnyhowResult; + +pub fn delete_pods(pod_names: Vec) -> AnyhowResult<()> { + //let pod_names = pod_names.join(" "); + + let mut args = Vec::from(["delete".to_string(), "pods".to_string()]); + + args.extend(pod_names); + + let output = Command::new("kubectl") + .args(args) + .output()?; + + let stdout = String::from_utf8(output.stdout)?; + + info!("Output: {}", stdout); + + Ok(()) +} diff --git a/crates/cli/kube_pod_cleanup/src/list_pods.rs b/crates/cli/kube_pod_cleanup/src/list_pods.rs new file mode 100644 index 0000000000..98ea17169f --- /dev/null +++ b/crates/cli/kube_pod_cleanup/src/list_pods.rs @@ -0,0 +1,147 @@ +use std::ops::Index; +use std::process::Command; + +use log::info; +use once_cell::sync::Lazy; +use regex::Regex; + +use errors::AnyhowResult; + +/// Information about pods in the cluster +/// Obtained by `kubectl get pods`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PodInfo { + pub name: String, + pub ready: String, + pub status: String, + pub restarts: String, + pub age: String, +} + +//#[derive(Clone, Debug)] +//pub enum PodStatus { +// Running, +// +// Completed, +// ContainerStatusUnknown, +// CrashLoopBackOff, +// Error, +// Evicted, +// /// eg. Init:ContainerStatusUnknown +// InitContainerStatusUnknown, +// /// eg. Init:ErrImagePull +// InitErrImagePull, +// InitError, +// InitImagePullBackoff, +// /// eg. Init:2/3 +// InitPartial, +// NodeAffinity, +// NodeShutdown, +// OOMKilled, +// /// eg. OutOfnvidia.com/gpu +// OutOfGpu, +// Pending, +// PodInitializing, +// Terminated, +// Terminating, +// UnexpectedAdmissionError, +// +// OtherStatus(String), +//} + +pub fn list_pods() -> AnyhowResult> { + info!("kube-pod-cleanup"); + + // Returns, eg. + // NAME READY STATUS RESTARTS AGE + // analytics-job-b8cbf88bd-c5ghc 1/1 Running 0 90d + // storyteller-obs-7d6df4b678-xprsq 1/1 Running 0 90d + // storyteller-web-8566f9bdc5-247d4 0/1 Error 0 6d11h + // storyteller-web-8566f9bdc5-25lw6 0/1 Evicted 0 2d15h + let output = Command::new("kubectl") + .args(["get", "pods"]) + .output()?; + + let stdout = String::from_utf8(output.stdout)?; + + parse_pod_status_output(&stdout) +} + +fn parse_pod_status_output(stdout: &str) -> AnyhowResult> { + // Parse `kubectl get pods` lines + const LIST_POD_REGEX : Lazy = Lazy::new(|| { + Regex::new(r"(\S+)\s+(\S+)\s+(\S+)\s+((\d+\s\(\S+ ago\))|(\S+))\s+(\S+).*") + .expect("regex should parse") + }); + + let lines = stdout.split("\n") + .into_iter() + .map(|line| line.trim().to_string()) + .collect::>(); + + // NB: We skip the first line as that's the header. + Ok(lines.iter() + .filter_map(|line| { + LIST_POD_REGEX.captures(line) + }) + .skip(1) + .map(|captures| { + let name = captures.index(1); + let ready = captures.index(2); + let status = captures.index(3); + let restarts = captures.index(4); + let age = captures.index(7); + PodInfo { + name: name.to_string(), + ready: ready.to_string(), + status: status.to_string(), + restarts: restarts.to_string(), + age: age.to_string(), + } + }) + .collect::>()) +} + +#[cfg(test)] +mod tests { + use crate::list_pods::{parse_pod_status_output, PodInfo}; + + #[test] + fn test_parse() { + let example_output_lines = r#" +NAME READY STATUS RESTARTS AGE +analytics-job-b8cbf88bd-c5ghc 1/1 Running 0 90d +storyteller-web-8566f9bdc5-2kv6v 1/1 Running 4 (2d12h ago) 4d23h +storyteller-web-8566f9bdc5-2fzlt 0/1 ContainerStatusUnknown 1 6d12h +storyteller-web-8566f9bdc5-247d4 0/1 Error 0 6d12h +storyteller-web-8566f9bdc5-25lw6 0/1 Evicted 0 2d16h + "#; + + let pods = parse_pod_status_output(example_output_lines) + .expect("should parse"); + + assert_eq!(pods.get(0).unwrap().clone(), PodInfo { + name: "analytics-job-b8cbf88bd-c5ghc".to_string(), + ready: "1/1".to_string(), + status: "Running".to_string(), + restarts: "0".to_string(), + age: "90d".to_string(), + }); + + assert_eq!(pods.get(1).unwrap().clone(), PodInfo { + name: "storyteller-web-8566f9bdc5-2kv6v".to_string(), + ready: "1/1".to_string(), + status: "Running".to_string(), + restarts: "4 (2d12h ago)".to_string(), + age: "4d23h".to_string(), + }); + + assert_eq!(pods.get(2).unwrap().clone(), PodInfo { + name: "storyteller-web-8566f9bdc5-2fzlt".to_string(), + ready: "0/1".to_string(), + status: "ContainerStatusUnknown".to_string(), + restarts: "1".to_string(), + age: "6d12h".to_string(), + }); + } +} diff --git a/crates/cli/kube_pod_cleanup/src/main.rs b/crates/cli/kube_pod_cleanup/src/main.rs new file mode 100644 index 0000000000..82ca6b3ea5 --- /dev/null +++ b/crates/cli/kube_pod_cleanup/src/main.rs @@ -0,0 +1,49 @@ +use std::ops::Index; + +use log::info; + +use config::shared_constants::DEFAULT_RUST_LOG; +use errors::AnyhowResult; +use crate::delete_pods::delete_pods; + +use crate::list_pods::list_pods; + +pub mod list_pods; +pub mod delete_pods; + +pub fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + info!("kube-pod-cleanup"); + + const POD_STATUSES_TO_KEEP : [&str; 1] = [ + "Running", + ]; + + const POD_STATUSES_TO_CLEAR : [&str; 3] = [ + "Error", + "Evicted", + "ContainerStatusUnknown", + ]; + + let all_pods = list_pods()?; + + let pods_to_clear = all_pods.iter() + .filter(|pod| !POD_STATUSES_TO_KEEP.contains(&pod.status.as_str())) + .filter(|pod| POD_STATUSES_TO_CLEAR.contains(&pod.status.as_str())) + .collect::>(); + + info!("Clearing {} pods...", pods_to_clear.len()); + + let pod_names_to_clear = pods_to_clear.iter() + .map(|pod| pod.name.to_string()) + .collect::>(); + //.join(" "); + + info!("Clearing pods: {:?}", &pod_names_to_clear); + + delete_pods(pod_names_to_clear)?; + + info!("Done!"); + Ok(()) +} diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index bbe5086562..0f7cbc26a6 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -255,22 +255,6 @@ async fn main() -> AnyhowResult<()> { job_stats: job_stats.clone(), }; - //// TODO/REMOVE - testing download of the dev weights - //let records = [ - // ("Alice", "qtqaprnd5shtybve4fqpvcfp50yjw238fbgj92z1521c50xqdxy1akkhkw7tesj0"), - // ("Biden", "n945w0xsq15xrh16hc147a5mc1a91gwh886e14qqzte1gr9z9q3yawjfvanp4fmg"), - // ("Goku", "cnnv05yjst2m737dpmxazgfpksjf4y7cxxern2ph7gddgnkh2bw1ephg5mhjbz14"), - // ("Hilary", "7wav68ba2yy86491jk36cgk36tkmzesr452dgfs28wchkrr03bd0h3e6c1bbz9eg"), - // ("Obama", "z3gy4v56sgtfrxfrpvaj7v74sqc67rcqs89jb884b00zfdm9vmkf1w2fsnta0gwp"), - // ("Trump", "qcy7pv3rph0ntkqnpz5cfg9ksyh7kkz53v1wbr2ckvt8znvhxqn7ca5mz7wzm3q5"), - //]; - //let paths = BucketPathUnifier::default_paths(); - //for (name, hash) in records { - // let object_path = paths.zero_shot_tts_speaker_encoding(hash, 0); - // let filesystem_path = format!("{}_weights.npz", name); - // private_bucket_client.download_file_to_disk(object_path, filesystem_path).await?; - //} - let job_dependencies = JobDependencies { scoped_execution: ScopedExecution::new_from_env()?, fs: FileSystemDetails {