From d1d208d714587ac9fd2a8da34217efdf532f9130 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Mon, 1 May 2023 03:59:05 -0700
Subject: [PATCH] add more timing stats to the jobs tables

---
 .../generic_inference/generic_inference_jobs.sql      |  6 +++++-
 .../down.sql                                          |  6 ++++++
 .../up.sql                                            | 11 +++++++++++
 _sql/migrations_squashed/generic_download_jobs.sql    |  4 ++++
 _sql/migrations_squashed/generic_inference_jobs.sql   |  4 ++++
 5 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 _sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/down.sql
 create mode 100644 _sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/up.sql

diff --git a/_sql/analytics/generic_inference/generic_inference_jobs.sql b/_sql/analytics/generic_inference/generic_inference_jobs.sql
index 09e55c1ab7..23e3c1086b 100644
--- a/_sql/analytics/generic_inference/generic_inference_jobs.sql
+++ b/_sql/analytics/generic_inference/generic_inference_jobs.sql
@@ -15,7 +15,10 @@ and success_execution_millis IS NOT NULL
 order by id desc
 limit 50;
 
--- Same, but with usernames
+
+-- Detailed report on most recent jobs, ordered by worst performing.
+-- TODO: Determine if the problem is in downloading models. Make sure the job timer doesn't include
+--   sections where models / files get downloaded.
 select *
 from (
     select
@@ -42,6 +45,7 @@ from (
 ) as t
 order by execution_minutes desc;
 
+
 -- Get pending so-vits-svc jobs
 select count(*)
 from generic_inference_jobs
diff --git a/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/down.sql b/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/down.sql
new file mode 100644
index 0000000000..68912503ff
--- /dev/null
+++ b/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/down.sql
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_download_jobs DROP COLUMN success_inference_execution_millis;
+ALTER TABLE generic_inference_jobs DROP COLUMN success_inference_execution_millis;
diff --git a/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/up.sql b/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/up.sql
new file mode 100644
index 0000000000..7d3d704c7c
--- /dev/null
+++ b/_sql/migrations/2023-05-01-105648_alter_jobs_add_inference_duration_millis/up.sql
@@ -0,0 +1,11 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_download_jobs
+    ADD COLUMN success_inference_execution_millis INT(10) UNSIGNED DEFAULT NULL
+    AFTER success_execution_millis;
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN success_inference_execution_millis INT(10) UNSIGNED DEFAULT NULL
+    AFTER success_execution_millis;
diff --git a/_sql/migrations_squashed/generic_download_jobs.sql b/_sql/migrations_squashed/generic_download_jobs.sql
index 38a0886f01..e57c1a6b4b 100644
--- a/_sql/migrations_squashed/generic_download_jobs.sql
+++ b/_sql/migrations_squashed/generic_download_jobs.sql
@@ -96,6 +96,10 @@ CREATE TABLE generic_download_jobs (
   -- derived execution time analytics with it.
   success_execution_millis INT(10) UNSIGNED DEFAULT NULL,
 
+  -- This is only the inference portion and does not include downloads, uploads,
+  -- or any subsequent processing.
+  success_inference_execution_millis INT(10) UNSIGNED DEFAULT NULL,
+
   -- Worker hostname (linux hostname, k8s pod name)
   -- Assigned when a worker picks up the job
   -- Reassigned if the job fails and gets picked up again
diff --git a/_sql/migrations_squashed/generic_inference_jobs.sql b/_sql/migrations_squashed/generic_inference_jobs.sql
index 835a9293b4..d579e202d9 100644
--- a/_sql/migrations_squashed/generic_inference_jobs.sql
+++ b/_sql/migrations_squashed/generic_inference_jobs.sql
@@ -153,6 +153,10 @@ CREATE TABLE generic_inference_jobs (
   -- derived execution time analytics with it.
   success_execution_millis INT(10) UNSIGNED DEFAULT NULL,
 
+  -- This is only the inference portion and does not include downloads, uploads,
+  -- or any subsequent processing.
+  success_inference_execution_millis INT(10) UNSIGNED DEFAULT NULL,
+
   -- TODO: Remove once `assigned_worker` column is added
   --   and existing queries for this column are removed.
   -- The last worker (hostname or pod name) to touch the job, either in the case of success or failure.