From 4a1f1441bfa92a962a944663d8ef2eac6e6f25be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 Apr 2023 06:08:34 -0700 Subject: [PATCH] split analytics and operational queries --- _sql/actions/tts_inference_jobs.sql | 24 +++++++++++++++++++ _sql/actions/tts_results.sql | 11 +++++++++ _sql/actions/users.sql | 20 ++++++++++++++++ _sql/actions/w2l_inference_jobs.sql | 16 +++++++++++++ .../generic_inference_jobs.sql | 16 +++++++++++++ _sql/analytics/tts/tts_inference_jobs.sql | 19 --------------- _sql/analytics/tts/tts_results.sql | 8 ------- .../analytics/users/user_top_contributors.sql | 17 ------------- .../w2l/w2l_template_upload_jobs.sql | 12 ---------- 9 files changed, 87 insertions(+), 56 deletions(-) create mode 100644 _sql/actions/tts_inference_jobs.sql create mode 100644 _sql/actions/tts_results.sql create mode 100644 _sql/actions/users.sql create mode 100644 _sql/actions/w2l_inference_jobs.sql create mode 100644 _sql/analytics/generic_inference/generic_inference_jobs.sql diff --git a/_sql/actions/tts_inference_jobs.sql b/_sql/actions/tts_inference_jobs.sql new file mode 100644 index 0000000000..04f54ff109 --- /dev/null +++ b/_sql/actions/tts_inference_jobs.sql @@ -0,0 +1,24 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Kill pending jobs +update tts_inference_jobs set status = 'dead' where status = 'pending'; + +-- Kill all low priority pending, waiting, and in-progress jobs +update tts_inference_jobs +set status = 'dead' +where status IN ('pending', 'started', 'attempt_failed') + and priority_level IN (0, 1); + +-- Kill all pending jobs +update tts_inference_jobs +set status = 'dead' +where status = 'pending'; + +-- Kill all waiting and in-progress jobs +update tts_inference_jobs +set status = 'dead' +where status IN ('pending', 'started', 'attempt_failed'); + + diff --git a/_sql/actions/tts_results.sql b/_sql/actions/tts_results.sql new file mode 100644 index 0000000000..17327cd461 --- /dev/null +++ b/_sql/actions/tts_results.sql @@ -0,0 +1,11 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Delete TTS results for a single model +-- Do this in short batches so a lock isn't held for prohibitively long. +update tts_results +set mod_deleted_at = NOW() +where model_token = 'TM:ztt5s1be5tq6' + and mod_deleted_at IS NULL + limit 5000; diff --git a/_sql/actions/users.sql b/_sql/actions/users.sql new file mode 100644 index 0000000000..5e143c066b --- /dev/null +++ b/_sql/actions/users.sql @@ -0,0 +1,20 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Upgrade top TTS model uploaders to a loyalty plan +update users +set maybe_loyalty_program_key = 'fakeyou_contributor' +where token IN ( + select creator_user_token from + ( + select count(*) as upload_count, + creator_user_token + from tts_models + group by creator_user_token + having upload_count >= 10 + order by upload_count desc + ) as t +); + + diff --git a/_sql/actions/w2l_inference_jobs.sql b/_sql/actions/w2l_inference_jobs.sql new file mode 100644 index 0000000000..8f634c8c21 --- /dev/null +++ b/_sql/actions/w2l_inference_jobs.sql @@ -0,0 +1,16 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Unstick old jobs that might be stuck as "started" +update w2l_template_upload_jobs +set status='dead' +where status='started' + and created_at < ( CURDATE() - INTERVAL 1 DAY ); + +-- Kill pending jobs with tiktok URLs: +-- These need to be blocked in the server and job directly. +update w2l_template_upload_jobs +set status='dead' +where status='pending' + and download_url LIKE '%tiktok%'; diff --git a/_sql/analytics/generic_inference/generic_inference_jobs.sql b/_sql/analytics/generic_inference/generic_inference_jobs.sql new file mode 100644 index 0000000000..18445da119 --- /dev/null +++ b/_sql/analytics/generic_inference/generic_inference_jobs.sql @@ -0,0 +1,16 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- See how long recent jobs are taking +select + id, + maybe_input_source_token, + assigned_worker, + assigned_cluster, + success_execution_millis / 1000 / 60 as minutes +from generic_inference_jobs +where status != 'pending' +and success_execution_millis IS NOT NULL +order by id desc +limit 50; diff --git a/_sql/analytics/tts/tts_inference_jobs.sql b/_sql/analytics/tts/tts_inference_jobs.sql index 823b59669c..19e713dfd0 100644 --- a/_sql/analytics/tts/tts_inference_jobs.sql +++ b/_sql/analytics/tts/tts_inference_jobs.sql @@ -2,9 +2,6 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile --- Kill pending jobs -update tts_inference_jobs set status = 'dead' where status = 'pending'; - -- Total pending jobs select count(*) from tts_inference_jobs @@ -32,22 +29,6 @@ group by model_token order by attempts desc limit 50; --- Kill all low priority pending, waiting, and in-progress jobs -update tts_inference_jobs -set status = 'dead' -where status IN ('pending', 'started', 'attempt_failed') - and priority_level IN (0, 1); - --- Kill all pending jobs -update tts_inference_jobs -set status = 'dead' -where status = 'pending'; - --- Kill all waiting and in-progress jobs -update tts_inference_jobs -set status = 'dead' -where status IN ('pending', 'started', 'attempt_failed'); - -- Sample the pending inference text select creator_ip_address, model_token, diff --git a/_sql/analytics/tts/tts_results.sql b/_sql/analytics/tts/tts_results.sql index 66472ad3c8..347ff3df02 100644 --- a/_sql/analytics/tts/tts_results.sql +++ b/_sql/analytics/tts/tts_results.sql @@ -64,14 +64,6 @@ ORDER BY created_at DESC --- Delete TTS results for a single model --- Do this in short batches so a lock isn't held for prohibitively long. -update tts_results -set mod_deleted_at = NOW() -where model_token = 'TM:ztt5s1be5tq6' -and mod_deleted_at IS NULL -limit 5000; - --- Search for particular text by a particular model SELECT raw_inference_text, diff --git a/_sql/analytics/users/user_top_contributors.sql b/_sql/analytics/users/user_top_contributors.sql index f8781c0920..08ce76864e 100644 --- a/_sql/analytics/users/user_top_contributors.sql +++ b/_sql/analytics/users/user_top_contributors.sql @@ -25,20 +25,3 @@ on u.token = m.creator_user_token where m.upload_count >= 5; - --- Upgrade top TTS model uploaders to a loyalty plan -update users -set maybe_loyalty_program_key = 'fakeyou_contributor' -where token IN ( - select creator_user_token from - ( - select count(*) as upload_count, - creator_user_token - from tts_models - group by creator_user_token - having upload_count >= 10 - order by upload_count desc - ) as t -); - - diff --git a/_sql/analytics/w2l/w2l_template_upload_jobs.sql b/_sql/analytics/w2l/w2l_template_upload_jobs.sql index ceafb1a5e0..82afe63587 100644 --- a/_sql/analytics/w2l/w2l_template_upload_jobs.sql +++ b/_sql/analytics/w2l/w2l_template_upload_jobs.sql @@ -14,15 +14,3 @@ from w2l_template_upload_jobs where status='started' and created_at < ( CURDATE() - INTERVAL 1 DAY ); --- Unstick old jobs that might be stuck as "started" -update w2l_template_upload_jobs -set status='dead' -where status='started' - and created_at < ( CURDATE() - INTERVAL 1 DAY ); - --- Kill pending jobs with tiktok URLs: --- These need to be blocked in the server and job directly. -update w2l_template_upload_jobs -set status='dead' -where status='pending' - and download_url LIKE '%tiktok%';