mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-02 16:38:14 +00:00
kubectl cli to cleanup dead pods
This commit is contained in:
Brandon Thomas committed
1 parent
a0f464e454
commit
ccc67d43cc
7 files changed
+285
-29
No files matched your search
Generated
+36
-13
@@ -627,9 +627,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.18"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
|
||||
checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -3145,6 +3145,20 @@ dependencies = [
|
||||
"winapi-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kube-pod-cleanup"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"config",
|
||||
"easyenv",
|
||||
"enums",
|
||||
"errors",
|
||||
"hashing",
|
||||
"log",
|
||||
"once_cell",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "language-tags"
|
||||
version = "0.2.2"
|
||||
@@ -3284,12 +3298,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
|
||||
|
||||
[[package]]
|
||||
name = "lru-cache"
|
||||
@@ -3401,9 +3412,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
version = "2.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
@@ -4586,9 +4597,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.5.4"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
|
||||
checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -4597,9 +4620,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.25"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
|
||||
checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
|
||||
@@ -12,6 +12,7 @@ members = [
|
||||
"crates/cli/database_backup",
|
||||
"crates/cli/database_cleanup",
|
||||
"crates/cli/dev_database_seed",
|
||||
"crates/cli/kube_pod_cleanup",
|
||||
"crates/cli/scrape_test",
|
||||
|
||||
# Shared libraries
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "kube-pod-cleanup"
|
||||
edition = "2018"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "kube-pod-cleanup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
config = { path = "../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../lib/easyenv" }
|
||||
enums = { path = "../../_types/enums" }
|
||||
errors = { path = "../../lib/errors" }
|
||||
hashing = { path = "../../lib/files/hashing" }
|
||||
|
||||
# External
|
||||
log = "0.4.20"
|
||||
once_cell = "1.18.0"
|
||||
regex = "1.10.0"
|
||||
|
||||
[dev-dependencies]
|
||||
# None yet
|
||||
@@ -0,0 +1,23 @@
|
||||
use std::process::Command;
|
||||
|
||||
use log::info;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
pub fn delete_pods(pod_names: Vec<String>) -> AnyhowResult<()> {
|
||||
//let pod_names = pod_names.join(" ");
|
||||
|
||||
let mut args = Vec::from(["delete".to_string(), "pods".to_string()]);
|
||||
|
||||
args.extend(pod_names);
|
||||
|
||||
let output = Command::new("kubectl")
|
||||
.args(args)
|
||||
.output()?;
|
||||
|
||||
let stdout = String::from_utf8(output.stdout)?;
|
||||
|
||||
info!("Output: {}", stdout);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
use std::ops::Index;
|
||||
use std::process::Command;
|
||||
|
||||
use log::info;
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
/// Information about pods in the cluster
|
||||
/// Obtained by `kubectl get pods`.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct PodInfo {
|
||||
pub name: String,
|
||||
pub ready: String,
|
||||
pub status: String,
|
||||
pub restarts: String,
|
||||
pub age: String,
|
||||
}
|
||||
|
||||
//#[derive(Clone, Debug)]
|
||||
//pub enum PodStatus {
|
||||
// Running,
|
||||
//
|
||||
// Completed,
|
||||
// ContainerStatusUnknown,
|
||||
// CrashLoopBackOff,
|
||||
// Error,
|
||||
// Evicted,
|
||||
// /// eg. Init:ContainerStatusUnknown
|
||||
// InitContainerStatusUnknown,
|
||||
// /// eg. Init:ErrImagePull
|
||||
// InitErrImagePull,
|
||||
// InitError,
|
||||
// InitImagePullBackoff,
|
||||
// /// eg. Init:2/3
|
||||
// InitPartial,
|
||||
// NodeAffinity,
|
||||
// NodeShutdown,
|
||||
// OOMKilled,
|
||||
// /// eg. OutOfnvidia.com/gpu
|
||||
// OutOfGpu,
|
||||
// Pending,
|
||||
// PodInitializing,
|
||||
// Terminated,
|
||||
// Terminating,
|
||||
// UnexpectedAdmissionError,
|
||||
//
|
||||
// OtherStatus(String),
|
||||
//}
|
||||
|
||||
pub fn list_pods() -> AnyhowResult<Vec<PodInfo>> {
|
||||
info!("kube-pod-cleanup");
|
||||
|
||||
// Returns, eg.
|
||||
// NAME READY STATUS RESTARTS AGE
|
||||
// analytics-job-b8cbf88bd-c5ghc 1/1 Running 0 90d
|
||||
// storyteller-obs-7d6df4b678-xprsq 1/1 Running 0 90d
|
||||
// storyteller-web-8566f9bdc5-247d4 0/1 Error 0 6d11h
|
||||
// storyteller-web-8566f9bdc5-25lw6 0/1 Evicted 0 2d15h
|
||||
let output = Command::new("kubectl")
|
||||
.args(["get", "pods"])
|
||||
.output()?;
|
||||
|
||||
let stdout = String::from_utf8(output.stdout)?;
|
||||
|
||||
parse_pod_status_output(&stdout)
|
||||
}
|
||||
|
||||
fn parse_pod_status_output(stdout: &str) -> AnyhowResult<Vec<PodInfo>> {
|
||||
// Parse `kubectl get pods` lines
|
||||
const LIST_POD_REGEX : Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(r"(\S+)\s+(\S+)\s+(\S+)\s+((\d+\s\(\S+ ago\))|(\S+))\s+(\S+).*")
|
||||
.expect("regex should parse")
|
||||
});
|
||||
|
||||
let lines = stdout.split("\n")
|
||||
.into_iter()
|
||||
.map(|line| line.trim().to_string())
|
||||
.collect::<Vec<String>>();
|
||||
|
||||
// NB: We skip the first line as that's the header.
|
||||
Ok(lines.iter()
|
||||
.filter_map(|line| {
|
||||
LIST_POD_REGEX.captures(line)
|
||||
})
|
||||
.skip(1)
|
||||
.map(|captures| {
|
||||
let name = captures.index(1);
|
||||
let ready = captures.index(2);
|
||||
let status = captures.index(3);
|
||||
let restarts = captures.index(4);
|
||||
let age = captures.index(7);
|
||||
PodInfo {
|
||||
name: name.to_string(),
|
||||
ready: ready.to_string(),
|
||||
status: status.to_string(),
|
||||
restarts: restarts.to_string(),
|
||||
age: age.to_string(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::list_pods::{parse_pod_status_output, PodInfo};
|
||||
|
||||
#[test]
|
||||
fn test_parse() {
|
||||
let example_output_lines = r#"
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
analytics-job-b8cbf88bd-c5ghc 1/1 Running 0 90d
|
||||
storyteller-web-8566f9bdc5-2kv6v 1/1 Running 4 (2d12h ago) 4d23h
|
||||
storyteller-web-8566f9bdc5-2fzlt 0/1 ContainerStatusUnknown 1 6d12h
|
||||
storyteller-web-8566f9bdc5-247d4 0/1 Error 0 6d12h
|
||||
storyteller-web-8566f9bdc5-25lw6 0/1 Evicted 0 2d16h
|
||||
"#;
|
||||
|
||||
let pods = parse_pod_status_output(example_output_lines)
|
||||
.expect("should parse");
|
||||
|
||||
assert_eq!(pods.get(0).unwrap().clone(), PodInfo {
|
||||
name: "analytics-job-b8cbf88bd-c5ghc".to_string(),
|
||||
ready: "1/1".to_string(),
|
||||
status: "Running".to_string(),
|
||||
restarts: "0".to_string(),
|
||||
age: "90d".to_string(),
|
||||
});
|
||||
|
||||
assert_eq!(pods.get(1).unwrap().clone(), PodInfo {
|
||||
name: "storyteller-web-8566f9bdc5-2kv6v".to_string(),
|
||||
ready: "1/1".to_string(),
|
||||
status: "Running".to_string(),
|
||||
restarts: "4 (2d12h ago)".to_string(),
|
||||
age: "4d23h".to_string(),
|
||||
});
|
||||
|
||||
assert_eq!(pods.get(2).unwrap().clone(), PodInfo {
|
||||
name: "storyteller-web-8566f9bdc5-2fzlt".to_string(),
|
||||
ready: "0/1".to_string(),
|
||||
status: "ContainerStatusUnknown".to_string(),
|
||||
restarts: "1".to_string(),
|
||||
age: "6d12h".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use std::ops::Index;
|
||||
|
||||
use log::info;
|
||||
|
||||
use config::shared_constants::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
use crate::delete_pods::delete_pods;
|
||||
|
||||
use crate::list_pods::list_pods;
|
||||
|
||||
pub mod list_pods;
|
||||
pub mod delete_pods;
|
||||
|
||||
pub fn main() -> AnyhowResult<()> {
|
||||
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
|
||||
|
||||
info!("kube-pod-cleanup");
|
||||
|
||||
const POD_STATUSES_TO_KEEP : [&str; 1] = [
|
||||
"Running",
|
||||
];
|
||||
|
||||
const POD_STATUSES_TO_CLEAR : [&str; 3] = [
|
||||
"Error",
|
||||
"Evicted",
|
||||
"ContainerStatusUnknown",
|
||||
];
|
||||
|
||||
let all_pods = list_pods()?;
|
||||
|
||||
let pods_to_clear = all_pods.iter()
|
||||
.filter(|pod| !POD_STATUSES_TO_KEEP.contains(&pod.status.as_str()))
|
||||
.filter(|pod| POD_STATUSES_TO_CLEAR.contains(&pod.status.as_str()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
info!("Clearing {} pods...", pods_to_clear.len());
|
||||
|
||||
let pod_names_to_clear = pods_to_clear.iter()
|
||||
.map(|pod| pod.name.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
//.join(" ");
|
||||
|
||||
info!("Clearing pods: {:?}", &pod_names_to_clear);
|
||||
|
||||
delete_pods(pod_names_to_clear)?;
|
||||
|
||||
info!("Done!");
|
||||
Ok(())
|
||||
}
|
||||
@@ -255,22 +255,6 @@ async fn main() -> AnyhowResult<()> {
|
||||
job_stats: job_stats.clone(),
|
||||
};
|
||||
|
||||
//// TODO/REMOVE - testing download of the dev weights
|
||||
//let records = [
|
||||
// ("Alice", "qtqaprnd5shtybve4fqpvcfp50yjw238fbgj92z1521c50xqdxy1akkhkw7tesj0"),
|
||||
// ("Biden", "n945w0xsq15xrh16hc147a5mc1a91gwh886e14qqzte1gr9z9q3yawjfvanp4fmg"),
|
||||
// ("Goku", "cnnv05yjst2m737dpmxazgfpksjf4y7cxxern2ph7gddgnkh2bw1ephg5mhjbz14"),
|
||||
// ("Hilary", "7wav68ba2yy86491jk36cgk36tkmzesr452dgfs28wchkrr03bd0h3e6c1bbz9eg"),
|
||||
// ("Obama", "z3gy4v56sgtfrxfrpvaj7v74sqc67rcqs89jb884b00zfdm9vmkf1w2fsnta0gwp"),
|
||||
// ("Trump", "qcy7pv3rph0ntkqnpz5cfg9ksyh7kkz53v1wbr2ckvt8znvhxqn7ca5mz7wzm3q5"),
|
||||
//];
|
||||
//let paths = BucketPathUnifier::default_paths();
|
||||
//for (name, hash) in records {
|
||||
// let object_path = paths.zero_shot_tts_speaker_encoding(hash, 0);
|
||||
// let filesystem_path = format!("{}_weights.npz", name);
|
||||
// private_bucket_client.download_file_to_disk(object_path, filesystem_path).await?;
|
||||
//}
|
||||
|
||||
let job_dependencies = JobDependencies {
|
||||
scoped_execution: ScopedExecution::new_from_env()?,
|
||||
fs: FileSystemDetails {
|
||||
|
||||
Reference in new issue
Block a user