From 9cd7e566517c910310994c85c2aca0bda8884b9b Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Wed, 27 May 2020 18:48:53 -0400
Subject: [PATCH] Pod cleanup script (ugh, this is gross)

---
 pod_cleanup.sh | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100755 pod_cleanup.sh

diff --git a/pod_cleanup.sh b/pod_cleanup.sh
new file mode 100755
index 0000000000..cc968cd485
--- /dev/null
+++ b/pod_cleanup.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# For some reason, pods aren't going away after they're marked for deletion. 
+# They're getting stuck as "Terminating" for days at a time, leading to an 
+# accumulation of limitless stale pods.
+#
+# I'm not sure what the cause is, but there are lots of reports of this on
+# Google. 
+#
+# One thing I uncovered (in kubernetes events) that might be related:
+#
+# 22m         Normal    TaintManagerEviction     pod/{name}    Marking for deletion Pod {name}
+#
+# But I honestly don't know what is causing this. Perhaps the fuse filesytem?
+# Perhaps DigitalOcean weirdness?
+#
+# This stop-gap non-solution is taken from https://stackoverflow.com/q/50336665
+#
+pods=$( kubectl -n mumble get pods | grep -v Running | tail -n +2 | awk -F " " '{print $1}' )
+for pod in $pods;
+do
+  echo "Deleting pod ${pod}"
+  kubectl -n mumble delete pod $pod --grace-period=0 --force
+done
+