From 2e50cfaa67733e503b72d98b3008c5b4d62ca02b Mon Sep 17 00:00:00 2001 From: Hanashi Date: Fri, 14 Aug 2026 15:17:55 -0400 Subject: [PATCH] schema for kinovi dashboard --- .../down.sql | 5 + .../up.sql | 61 ++++++++++ .../down.sql | 5 + .../up.sql | 51 ++++++++ .../down.sql | 5 + .../up.sql | 83 +++++++++++++ .../down.sql | 5 + .../up.sql | 109 +++++++++++++++++ .../kinovi_credits_purchase_rollups.sql | 54 +++++++++ .../kinovi_spend/kinovi_credits_purchases.sql | 64 ++++++++++ .../kinovi_model_spend_rollups.sql | 112 ++++++++++++++++++ .../kinovi_spend/kinovi_order_rollups.sql | 86 ++++++++++++++ 12 files changed, 640 insertions(+) create mode 100644 _database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/down.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/up.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/down.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/up.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/down.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/up.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/down.sql create mode 100644 _database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/up.sql create mode 100644 _database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchase_rollups.sql create mode 100644 _database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchases.sql create mode 100644 _database/sql/migrations_squashed/kinovi_spend/kinovi_model_spend_rollups.sql create mode 100644 _database/sql/migrations_squashed/kinovi_spend/kinovi_order_rollups.sql diff --git a/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/down.sql b/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/down.sql new file mode 100644 index 0000000000..b89bd60993 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE IF EXISTS kinovi_credits_purchases; diff --git a/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/up.sql b/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/up.sql new file mode 100644 index 0000000000..f97d335e12 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0000_kinovi_credits_purchases/up.sql @@ -0,0 +1,61 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Raw mirror of the Kinovi billing payments history (`billing.getPayments`), +-- one row per credit-pack purchase event on a Kinovi account. This is the +-- "what Kinovi says we bought" side of the spend dashboard; the actual cash +-- leaves later (we are invoiced in arrears, typically T30 — irrelevant to +-- this table, which records the purchase events themselves). +-- +-- Populated by kinovi_reports_job every ~15 minutes: pages newest-first from +-- offset 0 and stops at the first page whose rows all already exist (by +-- uq_account_payment). The first run naturally walks the entire history +-- (~1k rows); no separate sync-state bookkeeping is needed. +CREATE TABLE kinovi_credits_purchases ( + -- Replication only. Rows are addressed by (kinovi_account, + -- kinovi_payment_id); individual rows are never looked up publicly, so + -- there is no token column. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Which of our Kinovi accounts made this purchase. KinoviAccount enum: + -- 'volcengine' / 'byteplus' / 'byteplus_ultra' (future accounts extend the + -- enum; readers must tolerate unknown values). + kinovi_account VARCHAR(32) NOT NULL, + + -- Kinovi's row id for this payment (their `id`, e.g. "31722"). Their API + -- returns it as a string; kept verbatim. Dedup anchor with kinovi_account. + kinovi_payment_id VARCHAR(64) NOT NULL, + + -- ========== MONEY / CREDITS ========== + + -- Purchase price in USD cents. Kinovi's API returns dollars (2159, 99.99); + -- converted to cents on ingest (round to the nearest cent) so we never + -- store floats. The standard bulk pack is 525,000 credits for 215,909 + -- cents ($2,159.09, ~243.16 credits/$1). + amount_usd_cents BIGINT UNSIGNED NOT NULL, + + -- Kinovi credits granted by this purchase (e.g. 525,000). + credits_earned BIGINT UNSIGNED NOT NULL, + + -- ========== TIMESTAMPS ========== + + -- When Kinovi recorded the purchase (their createdAt, UTC). Bin/aggregate + -- on THIS, not created_at (which is when our job ingested the row). + kinovi_created_at TIMESTAMP NOT NULL, + + -- Their updatedAt. Falls back to createdAt on ingest when missing or + -- unparseable. + kinovi_updated_at TIMESTAMP NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + -- Idempotency: one row per Kinovi payment per account. + UNIQUE KEY uq_account_payment (kinovi_account, kinovi_payment_id), + KEY idx_kinovi_created (kinovi_created_at), + KEY idx_account_kinovi_created (kinovi_account, kinovi_created_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/down.sql b/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/down.sql new file mode 100644 index 0000000000..e07f7c480f --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE IF EXISTS kinovi_credits_purchase_rollups; diff --git a/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/up.sql b/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/up.sql new file mode 100644 index 0000000000..e0187ee9e8 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0001_kinovi_credits_purchase_rollups/up.sql @@ -0,0 +1,51 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Pre-aggregated rollups of kinovi_credits_purchases at every graphing +-- resolution, per Kinovi account. One row per (account, period_type, +-- period_start); the dashboard sums across accounts for the combined series +-- or plots them separately. Sparsely populated — no row for empty periods. +-- Derived from (and fully rebuildable from) kinovi_credits_purchases; +-- maintained by kinovi_reports_job after each ingest sweep. +CREATE TABLE kinovi_credits_purchase_rollups ( + -- Replication only. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC. All period types normalize into this one column + -- (a day row is midnight UTC, a month row is the 1st at midnight, etc.), + -- so one query shape serves every graph resolution. + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES (over kinovi_created_at within the bucket) ========== + + -- Number of pack purchases in the bucket. + purchase_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Total purchase price, USD cents (what we owe Kinovi for these packs). + amount_usd_cents BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- Total Kinovi credits granted by the purchases in the bucket. + credits_earned BIGINT UNSIGNED NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_account_period (kinovi_account, period_type, period_start), + -- "All accounts, one resolution, date range" — the dashboard's main query. + KEY idx_period (period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/down.sql b/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/down.sql new file mode 100644 index 0000000000..1bedc3e5a6 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE IF EXISTS kinovi_order_rollups; diff --git a/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/up.sql b/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/up.sql new file mode 100644 index 0000000000..6b7b7ce0f5 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0002_kinovi_order_rollups/up.sql @@ -0,0 +1,83 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Rollups of Kinovi order costs (credits consumed by our generations, and +-- credits refunded back) at every graphing resolution, per Kinovi account. +-- One row per (account, period_type, period_start); sparsely populated. +-- +-- THERE IS NO RAW TABLE UNDERNEATH THIS — quarter-hour buckets are computed +-- directly from the Kinovi credits ledger API (`credits.getCreditHistory`) +-- by kinovi_reports_job, and the coarser resolutions are derived by summing +-- quarter-hours. The job's sweep (every ~15 minutes): +-- +-- 1. Page the ledger newest-first back to max(2h lookback, the newest +-- existing quarter-hour bucket) — the second bound self-heals outages +-- of any length. +-- 2. Bucket entries by their OWN timestamps into quarter hours, counting +-- only consumption and refund entries (grants are the purchases +-- table's business). +-- 3. RECOMPUTE-AND-REPLACE each covered bucket (never increment), which +-- makes overlapping sweeps idempotent without storing raw entries. +-- 4. Re-derive the covering hour/day/week/month buckets from quarter +-- hours. +-- +-- Refunds land in the bucket where the REFUND was issued, not where the +-- original order was billed (the ledger is append-only, so a late refund is +-- always a *new* entry inside the sweep window — no deep re-consumption is +-- ever needed). net_credits_consumed can therefore go negative in a +-- refund-heavy bucket. +-- +-- All figures are in KINOVI CREDITS. USD conversion happens at read time +-- (credits ÷ the negotiated bulk rate, currently 243.16 credits/$1) so a +-- future rate change never requires rewriting history; the purchases rollup +-- carries the actual dollars. +CREATE TABLE kinovi_order_rollups ( + -- Replication only. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC. All period types normalize into this one column + -- (a day row is midnight UTC, a month row is the 1st at midnight, etc.), + -- so one query shape serves every graph resolution. + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES ========== + + -- # of consumption entries (~generation orders billed) in the bucket. + order_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- # of refund entries in the bucket. + refund_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Credits consumed by generations (magnitude of negative consumption + -- deltas). + credits_consumed BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- Credits refunded back (magnitude of positive refund deltas). + credits_refunded BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- consumed − refunded (SIGNED — refunds often net against consumption + -- from an EARLIER bucket). This is the cost line the dashboard graphs. + net_credits_consumed BIGINT NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_account_period (kinovi_account, period_type, period_start), + -- "All accounts, one resolution, date range" — the dashboard's main query. + KEY idx_period (period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/down.sql b/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/down.sql new file mode 100644 index 0000000000..6c6796c433 --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE IF EXISTS kinovi_model_spend_rollups; diff --git a/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/up.sql b/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/up.sql new file mode 100644 index 0000000000..d393501abd --- /dev/null +++ b/_database/sql/migrations/2026-08-14-110000-0003_kinovi_model_spend_rollups/up.sql @@ -0,0 +1,109 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Per-model / per-variant rollups of Kinovi order costs at every graphing +-- resolution, sourced ENTIRELY FROM KINOVI'S SIDE (`userOrder.getOrders`) — +-- deliberately NOT from our own generic_inference_jobs, so this remains an +-- independent check on our recorded costs. One row per (account, model, +-- resolution, has_video_reference, period_type, period_start); sparsely +-- populated. +-- +-- THERE IS NO RAW ORDERS TABLE UNDERNEATH THIS — like kinovi_order_rollups, +-- buckets are recomputed directly from the API by kinovi_reports_job: +-- +-- 1. Cursor-page getOrders newest-first back to max(2h lookback, the +-- newest existing bucket) — orders settle (PENDING/PROCESSING -> +-- COMPLETED/FAILED) within minutes, so the lookback re-sweep captures +-- late settlement; the second bound self-heals outages. +-- 2. Bucket orders by THEIR createdAt into quarter hours, keyed by the +-- dimensions below (taken from the order's own `params`). +-- 3. RECOMPUTE-AND-REPLACE covered buckets (never increment) for +-- idempotency, then re-derive hour/day/week/month. +-- +-- Costs count COMPLETED orders' totalCredits (failed orders are refunded by +-- Kinovi ~in full). Rare late policy refunds do NOT appear in the orders +-- API (the order stays COMPLETED), so this table can slightly overstate net +-- cost; kinovi_order_rollups (ledger-based, refund-complete) remains the +-- authoritative TOTAL — the two cross-check each other. +-- +-- All figures are in KINOVI CREDITS; USD conversion happens at read time. +CREATE TABLE kinovi_model_spend_rollups ( + -- Replication only. Rows are addressed by the composite unique key below; + -- individual rows are never looked up publicly, so there is no token. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- ========== MODEL / VARIANT DIMENSIONS (from the order's params) ========== + + -- Kinovi's own model identifier, verbatim (their `params.model`, e.g. + -- 'seedance-20', 'seedance-25', 'midjourney-v8'; falls back to the + -- order's toolId, else 'unknown'). Kinovi controls these values — mapped + -- in code with an Unknown passthrough, never rewritten at ingest. + kinovi_model VARCHAR(64) NOT NULL, + + -- Canonical encoding of every BILLING-AFFECTING dimension of the order + -- within its model — which cell of the model's rate card it billed. + -- Constructed by ONE canonical builder in code (deterministic, so equal + -- configurations always collide into the same rollup row): lowercase + -- 'key=value' / bare-flag segments, sorted, comma-joined, defaults + -- omitted. Empty string = the model's base rate. Examples: + -- '' -- base rate (e.g. audio model, no modifiers) + -- 'res=720p' -- 720p, no surcharges + -- 'res=720p,vidref' -- 720p with a reference video attached + -- 'quality=2' -- Midjourney quality tier + -- Future rate-affecting factors (audio generation, bitrate tiers, ...) + -- become new segments — no schema change. Dimensions that DON'T affect + -- the model's billing are deliberately excluded per model, so rows stay + -- coarse where price is uniform. + rate_card_key VARCHAR(128) NOT NULL, + + -- Denormalized convenience copy of the resolution segment of + -- rate_card_key ('480p'/'720p'/'1080p'/'4k'; WxH forms like '1280x720' + -- normalize to their p-label). NULL for models where resolution does not + -- apply (audio) or the order carries none. NOT part of the unique key — + -- rate_card_key is the identity; this exists for the common + -- filter/group-by-resolution queries. + maybe_resolution VARCHAR(32) DEFAULT NULL, + + -- ========== PERIOD ========== + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC (same normalization as the other rollup tables). + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES (over order createdAt within the bucket) ========== + + -- All orders created in the bucket, settled or not. In-flight orders + -- (PENDING/PROCESSING) count here and migrate into completed_count / + -- failed_count as the lookback re-sweep observes them settling. + order_count INT UNSIGNED NOT NULL DEFAULT 0, + + completed_count INT UNSIGNED NOT NULL DEFAULT 0, + failed_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Sum of totalCredits over COMPLETED orders — the cost line the dashboard + -- graphs per model/variant. + credits_charged BIGINT UNSIGNED NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_cell_period (kinovi_account, kinovi_model, rate_card_key, period_type, period_start), + -- "All models, date range" — the dashboard's main breakdown query + -- (filter by period, group by model / rate_card_key / resolution). + KEY idx_period (period_type, period_start), + KEY idx_model_period (kinovi_model, period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchase_rollups.sql b/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchase_rollups.sql new file mode 100644 index 0000000000..1cca4109eb --- /dev/null +++ b/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchase_rollups.sql @@ -0,0 +1,54 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- Pre-aggregated rollups of kinovi_credits_purchases at every graphing +-- resolution, per Kinovi account. One row per (account, period_type, +-- period_start); the dashboard sums across accounts for the combined series +-- or plots them separately. Sparsely populated — no row for empty periods. +-- Derived from (and fully rebuildable from) kinovi_credits_purchases; +-- maintained by kinovi_reports_job after each ingest sweep. +CREATE TABLE kinovi_credits_purchase_rollups ( + -- Replication only. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC. All period types normalize into this one column + -- (a day row is midnight UTC, a month row is the 1st at midnight, etc.), + -- so one query shape serves every graph resolution. + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES (over kinovi_created_at within the bucket) ========== + + -- Number of pack purchases in the bucket. + purchase_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Total purchase price, USD cents (what we owe Kinovi for these packs). + amount_usd_cents BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- Total Kinovi credits granted by the purchases in the bucket. + credits_earned BIGINT UNSIGNED NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_account_period (kinovi_account, period_type, period_start), + -- "All accounts, one resolution, date range" — the dashboard's main query. + KEY idx_period (period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchases.sql b/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchases.sql new file mode 100644 index 0000000000..2b9e14df4a --- /dev/null +++ b/_database/sql/migrations_squashed/kinovi_spend/kinovi_credits_purchases.sql @@ -0,0 +1,64 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- Raw mirror of the Kinovi billing payments history (`billing.getPayments`), +-- one row per credit-pack purchase event on a Kinovi account. This is the +-- "what Kinovi says we bought" side of the spend dashboard; the actual cash +-- leaves later (we are invoiced in arrears, typically T30 — irrelevant to +-- this table, which records the purchase events themselves). +-- +-- Populated by kinovi_reports_job every ~15 minutes: pages newest-first from +-- offset 0 and stops at the first page whose rows all already exist (by +-- uq_account_payment). The first run naturally walks the entire history +-- (~1k rows); no separate sync-state bookkeeping is needed. +CREATE TABLE kinovi_credits_purchases ( + -- Replication only. Rows are addressed by (kinovi_account, + -- kinovi_payment_id); individual rows are never looked up publicly, so + -- there is no token column. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Which of our Kinovi accounts made this purchase. KinoviAccount enum: + -- 'volcengine' / 'byteplus' / 'byteplus_ultra' (future accounts extend the + -- enum; readers must tolerate unknown values). + kinovi_account VARCHAR(32) NOT NULL, + + -- Kinovi's row id for this payment (their `id`, e.g. "31722"). Their API + -- returns it as a string; kept verbatim. Dedup anchor with kinovi_account. + kinovi_payment_id VARCHAR(64) NOT NULL, + + -- ========== MONEY / CREDITS ========== + + -- Purchase price in USD cents. Kinovi's API returns dollars (2159, 99.99); + -- converted to cents on ingest (round to the nearest cent) so we never + -- store floats. The standard bulk pack is 525,000 credits for 215,909 + -- cents ($2,159.09, ~243.16 credits/$1). + amount_usd_cents BIGINT UNSIGNED NOT NULL, + + -- Kinovi credits granted by this purchase (e.g. 525,000). + credits_earned BIGINT UNSIGNED NOT NULL, + + -- ========== TIMESTAMPS ========== + + -- When Kinovi recorded the purchase (their createdAt, UTC). Bin/aggregate + -- on THIS, not created_at (which is when our job ingested the row). + kinovi_created_at TIMESTAMP NOT NULL, + + -- Their updatedAt. Falls back to createdAt on ingest when missing or + -- unparseable. + kinovi_updated_at TIMESTAMP NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + -- Idempotency: one row per Kinovi payment per account. + UNIQUE KEY uq_account_payment (kinovi_account, kinovi_payment_id), + KEY idx_kinovi_created (kinovi_created_at), + KEY idx_account_kinovi_created (kinovi_account, kinovi_created_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations_squashed/kinovi_spend/kinovi_model_spend_rollups.sql b/_database/sql/migrations_squashed/kinovi_spend/kinovi_model_spend_rollups.sql new file mode 100644 index 0000000000..e866680f2a --- /dev/null +++ b/_database/sql/migrations_squashed/kinovi_spend/kinovi_model_spend_rollups.sql @@ -0,0 +1,112 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- Per-model / per-variant rollups of Kinovi order costs at every graphing +-- resolution, sourced ENTIRELY FROM KINOVI'S SIDE (`userOrder.getOrders`) — +-- deliberately NOT from our own generic_inference_jobs, so this remains an +-- independent check on our recorded costs. One row per (account, model, +-- resolution, has_video_reference, period_type, period_start); sparsely +-- populated. +-- +-- THERE IS NO RAW ORDERS TABLE UNDERNEATH THIS — like kinovi_order_rollups, +-- buckets are recomputed directly from the API by kinovi_reports_job: +-- +-- 1. Cursor-page getOrders newest-first back to max(2h lookback, the +-- newest existing bucket) — orders settle (PENDING/PROCESSING -> +-- COMPLETED/FAILED) within minutes, so the lookback re-sweep captures +-- late settlement; the second bound self-heals outages. +-- 2. Bucket orders by THEIR createdAt into quarter hours, keyed by the +-- dimensions below (taken from the order's own `params`). +-- 3. RECOMPUTE-AND-REPLACE covered buckets (never increment) for +-- idempotency, then re-derive hour/day/week/month. +-- +-- Costs count COMPLETED orders' totalCredits (failed orders are refunded by +-- Kinovi ~in full). Rare late policy refunds do NOT appear in the orders +-- API (the order stays COMPLETED), so this table can slightly overstate net +-- cost; kinovi_order_rollups (ledger-based, refund-complete) remains the +-- authoritative TOTAL — the two cross-check each other. +-- +-- All figures are in KINOVI CREDITS; USD conversion happens at read time. +CREATE TABLE kinovi_model_spend_rollups ( + -- Replication only. Rows are addressed by the composite unique key below; + -- individual rows are never looked up publicly, so there is no token. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- ========== MODEL / VARIANT DIMENSIONS (from the order's params) ========== + + -- Kinovi's own model identifier, verbatim (their `params.model`, e.g. + -- 'seedance-20', 'seedance-25', 'midjourney-v8'; falls back to the + -- order's toolId, else 'unknown'). Kinovi controls these values — mapped + -- in code with an Unknown passthrough, never rewritten at ingest. + kinovi_model VARCHAR(64) NOT NULL, + + -- Canonical encoding of every BILLING-AFFECTING dimension of the order + -- within its model — which cell of the model's rate card it billed. + -- Constructed by ONE canonical builder in code (deterministic, so equal + -- configurations always collide into the same rollup row): lowercase + -- 'key=value' / bare-flag segments, sorted, comma-joined, defaults + -- omitted. Empty string = the model's base rate. Examples: + -- '' -- base rate (e.g. audio model, no modifiers) + -- 'res=720p' -- 720p, no surcharges + -- 'res=720p,vidref' -- 720p with a reference video attached + -- 'quality=2' -- Midjourney quality tier + -- Future rate-affecting factors (audio generation, bitrate tiers, ...) + -- become new segments — no schema change. Dimensions that DON'T affect + -- the model's billing are deliberately excluded per model, so rows stay + -- coarse where price is uniform. + rate_card_key VARCHAR(128) NOT NULL, + + -- Denormalized convenience copy of the resolution segment of + -- rate_card_key ('480p'/'720p'/'1080p'/'4k'; WxH forms like '1280x720' + -- normalize to their p-label). NULL for models where resolution does not + -- apply (audio) or the order carries none. NOT part of the unique key — + -- rate_card_key is the identity; this exists for the common + -- filter/group-by-resolution queries. + maybe_resolution VARCHAR(32) DEFAULT NULL, + + -- ========== PERIOD ========== + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC (same normalization as the other rollup tables). + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES (over order createdAt within the bucket) ========== + + -- All orders created in the bucket, settled or not. In-flight orders + -- (PENDING/PROCESSING) count here and migrate into completed_count / + -- failed_count as the lookback re-sweep observes them settling. + order_count INT UNSIGNED NOT NULL DEFAULT 0, + + completed_count INT UNSIGNED NOT NULL DEFAULT 0, + failed_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Sum of totalCredits over COMPLETED orders — the cost line the dashboard + -- graphs per model/variant. + credits_charged BIGINT UNSIGNED NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_cell_period (kinovi_account, kinovi_model, rate_card_key, period_type, period_start), + -- "All models, date range" — the dashboard's main breakdown query + -- (filter by period, group by model / rate_card_key / resolution). + KEY idx_period (period_type, period_start), + KEY idx_model_period (kinovi_model, period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_database/sql/migrations_squashed/kinovi_spend/kinovi_order_rollups.sql b/_database/sql/migrations_squashed/kinovi_spend/kinovi_order_rollups.sql new file mode 100644 index 0000000000..055153845d --- /dev/null +++ b/_database/sql/migrations_squashed/kinovi_spend/kinovi_order_rollups.sql @@ -0,0 +1,86 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- Rollups of Kinovi order costs (credits consumed by our generations, and +-- credits refunded back) at every graphing resolution, per Kinovi account. +-- One row per (account, period_type, period_start); sparsely populated. +-- +-- THERE IS NO RAW TABLE UNDERNEATH THIS — quarter-hour buckets are computed +-- directly from the Kinovi credits ledger API (`credits.getCreditHistory`) +-- by kinovi_reports_job, and the coarser resolutions are derived by summing +-- quarter-hours. The job's sweep (every ~15 minutes): +-- +-- 1. Page the ledger newest-first back to max(2h lookback, the newest +-- existing quarter-hour bucket) — the second bound self-heals outages +-- of any length. +-- 2. Bucket entries by their OWN timestamps into quarter hours, counting +-- only consumption and refund entries (grants are the purchases +-- table's business). +-- 3. RECOMPUTE-AND-REPLACE each covered bucket (never increment), which +-- makes overlapping sweeps idempotent without storing raw entries. +-- 4. Re-derive the covering hour/day/week/month buckets from quarter +-- hours. +-- +-- Refunds land in the bucket where the REFUND was issued, not where the +-- original order was billed (the ledger is append-only, so a late refund is +-- always a *new* entry inside the sweep window — no deep re-consumption is +-- ever needed). net_credits_consumed can therefore go negative in a +-- refund-heavy bucket. +-- +-- All figures are in KINOVI CREDITS. USD conversion happens at read time +-- (credits ÷ the negotiated bulk rate, currently 243.16 credits/$1) so a +-- future rate change never requires rewriting history; the purchases rollup +-- carries the actual dollars. +CREATE TABLE kinovi_order_rollups ( + -- Replication only. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- KinoviAccount enum: 'volcengine' / 'byteplus' / 'byteplus_ultra'. + kinovi_account VARCHAR(32) NOT NULL, + + -- RollupPeriodType enum — the graphing resolution of this row: + -- 'quarter_hour' -- 15-minute buckets (:00, :15, :30, :45) + -- 'hour' + -- 'day' + -- 'week' -- ISO weeks, Monday 00:00 UTC + -- 'month' -- 1st of the month, 00:00 UTC + period_type VARCHAR(16) NOT NULL, + + -- Bucket start, UTC. All period types normalize into this one column + -- (a day row is midnight UTC, a month row is the 1st at midnight, etc.), + -- so one query shape serves every graph resolution. + period_start TIMESTAMP NOT NULL, + + -- ========== AGGREGATES ========== + + -- # of consumption entries (~generation orders billed) in the bucket. + order_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- # of refund entries in the bucket. + refund_count INT UNSIGNED NOT NULL DEFAULT 0, + + -- Credits consumed by generations (magnitude of negative consumption + -- deltas). + credits_consumed BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- Credits refunded back (magnitude of positive refund deltas). + credits_refunded BIGINT UNSIGNED NOT NULL DEFAULT 0, + + -- consumed − refunded (SIGNED — refunds often net against consumption + -- from an EARLIER bucket). This is the cost line the dashboard graphs. + net_credits_consumed BIGINT NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- ========== INDICES ========== + PRIMARY KEY (id), + UNIQUE KEY uq_account_period (kinovi_account, period_type, period_start), + -- "All accounts, one resolution, date range" — the dashboard's main query. + KEY idx_period (period_type, period_start) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;