Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9ee1bd225 | ||
|
|
68886b62fe | ||
|
|
293a41c7d4 | ||
|
|
35a859529f | ||
|
|
3a93e358cd | ||
|
|
1cb0b32898 | ||
|
|
c50bb78c32 | ||
|
|
55ab800542 | ||
|
|
2c93b4cf42 | ||
|
|
9169e4d662 | ||
|
|
4c7af51851 | ||
|
|
eadfb4f821 | ||
|
|
0c163d6c19 | ||
|
|
26484685ee | ||
|
|
5bceb33801 | ||
|
|
c6e2c4296a | ||
|
|
8551d58def | ||
|
|
807180bf22 | ||
|
|
f48d2beea7 | ||
|
|
dc72bbd5e6 | ||
|
|
b6a184bcb0 | ||
|
|
d0e1d113fe | ||
|
|
1facb312dc | ||
|
|
f2c8a0390f | ||
|
|
2e83d39d2b | ||
|
|
4f1c9134ee | ||
|
|
f0a877c988 | ||
|
|
1b2d2f49e5 | ||
|
|
3e6d759fe2 | ||
|
|
f77f09018a | ||
|
|
fc9d727ffc | ||
|
|
e2da58b3e5 | ||
|
|
681f32b230 | ||
|
|
edfa80dde3 |
No files matched your search
+102
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | PRIMARY_KEY | UNSIGNED | AUTO_INCREMENT",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "confirmation_code",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 32,
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status: UserLoginChallengeStatus",
|
||||
"ordinal": 2,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 64,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_deciding_user_token",
|
||||
"ordinal": 3,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_redeemed_user_session_id",
|
||||
"ordinal": 4,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "UNIQUE_KEY",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_failure_type: UserLoginChallengeFailureType",
|
||||
"ordinal": 5,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "BINARY",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ip_address_creation",
|
||||
"ordinal": 6,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 160,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "expires_at",
|
||||
"ordinal": 7,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
|
||||
"max_size": 19,
|
||||
"type": "Timestamp"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "06dc6f19f013fe24ffd32a313b66dd10a87498bf17c9a63b1ece87e338b0dc74",
|
||||
"query": "\nSELECT id, confirmation_code, status AS `status: UserLoginChallengeStatus`, maybe_deciding_user_token, maybe_redeemed_user_session_id,\n maybe_failure_type AS `maybe_failure_type: UserLoginChallengeFailureType`, ip_address_creation, expires_at\nFROM user_login_challenges WHERE device_token_sha256 = ? FOR UPDATE\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 5
|
||||
}
|
||||
},
|
||||
"hash": "1af20e0d9a5630302f987aea4847b65a34d6c412245f7fbf9c6fb7b513d7324b",
|
||||
"query": "\nUPDATE user_login_challenges SET status = ?, maybe_failure_type = ?, maybe_failed_at = expires_at\nWHERE id = ? AND status IN (?, ?) AND expires_at <= NOW()\n"
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "expired",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | BINARY",
|
||||
"max_size": 1,
|
||||
"type": "LongLong"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "1c618e9816d6f3f7f217770f988a132d9521693da6474e945628d1b3efbc4b80",
|
||||
"query": "\nSELECT expires_at <= NOW() AS expired FROM user_login_challenges WHERE id = ?\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 6
|
||||
}
|
||||
},
|
||||
"hash": "2ddd5589138c822385b39dde58fc0d44ffa003b09018f1799e243d8f09530981",
|
||||
"query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n maybe_impersonation_user_token,\n ip_address_creation,\n maybe_creation_type,\n expires_at\n)\nVALUES (?, ?, ?, ?, ?, ?)\n "
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "session_token",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user_token",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "459898b1b42a51097dcbc0f182bd0badce642b11bd33a7e1973a6cc25538ffbf",
|
||||
"query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\nAND expires_at > NOW()\n "
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 4
|
||||
}
|
||||
},
|
||||
"hash": "4d3666433b8100c9b5f351c2be0faa982127c5bb4431ef14e84ff11c5a2fd9ff",
|
||||
"query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n maybe_creation_type,\n expires_at\n)\nVALUES ( ?, ?, ?, ?, NOW() + interval 1 year )\n "
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | PRIMARY_KEY | AUTO_INCREMENT",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "token",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user_token",
|
||||
"ordinal": 2,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "username",
|
||||
"ordinal": 3,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 80,
|
||||
"type": "VarString"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 3
|
||||
}
|
||||
},
|
||||
"hash": "705c952102e32df11ba98ec39a3c7943bdf633947c32d4e8f7d77a861560b1bd",
|
||||
"query": "\nSELECT s.id, s.token, s.user_token, u.username FROM user_sessions s JOIN users u ON u.token = s.user_token\nWHERE s.id = ? AND s.user_token = ? AND s.maybe_creation_type = ?\n AND s.deleted_at IS NULL AND s.expires_at > NOW() AND s.maybe_impersonation_user_token IS NULL\n AND u.is_banned = FALSE AND u.user_deleted_at IS NULL AND u.mod_deleted_at IS NULL\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 5
|
||||
}
|
||||
},
|
||||
"hash": "7e9bc784634f436a8bf898223a5ddeea57886b7c248d82e782dd3289c23cbaec",
|
||||
"query": "\nUPDATE user_login_challenges SET status = ?, maybe_redeemed_user_session_id = ?,\n maybe_ip_address_redemption = ?, maybe_redeemed_at = NOW()\nWHERE id = ? AND status = ? AND expires_at > NOW() AND maybe_redeemed_user_session_id IS NULL\n"
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | PRIMARY_KEY | AUTO_INCREMENT",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "token",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user_token",
|
||||
"ordinal": 2,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "username",
|
||||
"ordinal": 3,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 80,
|
||||
"type": "VarString"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "8bfbb8bef3bec235f37c5f4f5c2f3a8b0833241a63da122b8abd97cd9b71208f",
|
||||
"query": "\nSELECT s.id, s.token, s.user_token, u.username FROM user_sessions s JOIN users u ON u.token = s.user_token\nWHERE s.token = ? AND s.deleted_at IS NULL AND s.expires_at > NOW()\n AND s.maybe_impersonation_user_token IS NULL AND u.is_banned = FALSE\n AND u.user_deleted_at IS NULL AND u.mod_deleted_at IS NULL\n"
|
||||
}
|
||||
+454
@@ -0,0 +1,454 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "user_token",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "username",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 80,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "display_name",
|
||||
"ordinal": 2,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 80,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "email_address",
|
||||
"ordinal": 3,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 1020,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "email_gravatar_hash",
|
||||
"ordinal": 4,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 128,
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "email_confirmed",
|
||||
"ordinal": 5,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "email_confirmed_by_google",
|
||||
"ordinal": 6,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "email_is_synthetic",
|
||||
"ordinal": 7,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "is_without_password",
|
||||
"ordinal": 8,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "username_is_not_customized",
|
||||
"ordinal": 9,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_stripe_customer_id",
|
||||
"ordinal": 10,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "BINARY",
|
||||
"max_size": 1020,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_loyalty_program_key",
|
||||
"ordinal": 11,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "BINARY",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "disable_gravatar",
|
||||
"ordinal": 12,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "auto_play_audio_preference",
|
||||
"ordinal": 13,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "auto_play_video_preference",
|
||||
"ordinal": 14,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "preferred_tts_result_visibility: enums::common::visibility::Visibility",
|
||||
"ordinal": 15,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | ENUM",
|
||||
"max_size": 28,
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "preferred_w2l_result_visibility: enums::common::visibility::Visibility",
|
||||
"ordinal": 16,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | ENUM",
|
||||
"max_size": 28,
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user_role_slug",
|
||||
"ordinal": 17,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 64,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "is_banned",
|
||||
"ordinal": 18,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_access_studio",
|
||||
"ordinal": 19,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_feature_flags",
|
||||
"ordinal": 20,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "BINARY",
|
||||
"max_size": 1020,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_use_tts",
|
||||
"ordinal": 21,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_use_w2l",
|
||||
"ordinal": 22,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_own_tts_results",
|
||||
"ordinal": 23,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_own_w2l_results",
|
||||
"ordinal": 24,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_own_account",
|
||||
"ordinal": 25,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_upload_tts_models",
|
||||
"ordinal": 26,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_upload_w2l_templates",
|
||||
"ordinal": 27,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_own_tts_models",
|
||||
"ordinal": 28,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_own_w2l_templates",
|
||||
"ordinal": 29,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_approve_w2l_templates",
|
||||
"ordinal": 30,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_edit_other_users_profiles",
|
||||
"ordinal": 31,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_edit_other_users_tts_models",
|
||||
"ordinal": 32,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_edit_other_users_w2l_templates",
|
||||
"ordinal": 33,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_other_users_tts_models",
|
||||
"ordinal": 34,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_other_users_tts_results",
|
||||
"ordinal": 35,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_other_users_w2l_templates",
|
||||
"ordinal": 36,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_other_users_w2l_results",
|
||||
"ordinal": 37,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_ban_users",
|
||||
"ordinal": 38,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "can_delete_users",
|
||||
"ordinal": 39,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "",
|
||||
"max_size": 1,
|
||||
"type": "Tiny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "934aad794dcb0057e39c471e7b25e5e191a395a2bd16553cb58a483b91c68437",
|
||||
"query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_gravatar_hash,\n \n users.email_confirmed,\n users.email_confirmed_by_google,\n users.email_is_synthetic,\n users.is_without_password,\n users.username_is_not_customized,\n\n users.maybe_stripe_customer_id,\n users.maybe_loyalty_program_key,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::common::visibility::Visibility`,\n users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: enums::common::visibility::Visibility`,\n\n users.user_role_slug,\n users.is_banned,\n\n users.can_access_studio,\n users.maybe_feature_flags,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND user_sessions.expires_at > NOW()\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n "
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 4
|
||||
}
|
||||
},
|
||||
"hash": "ab9e5c27ce0c460aa6d5d1b202829c2082fa0701773582d22aae05a9d376d4a6",
|
||||
"query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n maybe_creation_type,\n expires_at\n)\nVALUES ( ?, ?, ?, ?, NOW() + interval 1 year )\n "
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"ordinal": 0,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | PRIMARY_KEY | UNSIGNED | AUTO_INCREMENT",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "confirmation_code",
|
||||
"ordinal": 1,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 32,
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status: UserLoginChallengeStatus",
|
||||
"ordinal": 2,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 64,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_deciding_user_token",
|
||||
"ordinal": 3,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_redeemed_user_session_id",
|
||||
"ordinal": 4,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "UNIQUE_KEY",
|
||||
"max_size": 20,
|
||||
"type": "LongLong"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "maybe_failure_type: UserLoginChallengeFailureType",
|
||||
"ordinal": 5,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "BINARY",
|
||||
"max_size": 128,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ip_address_creation",
|
||||
"ordinal": 6,
|
||||
"type_info": {
|
||||
"char_set": 224,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"max_size": 160,
|
||||
"type": "VarString"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "expires_at",
|
||||
"ordinal": 7,
|
||||
"type_info": {
|
||||
"char_set": 63,
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
|
||||
"max_size": 19,
|
||||
"type": "Timestamp"
|
||||
}
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
}
|
||||
},
|
||||
"hash": "c0c85c8c55fb494d44efa6f1f2d37491c740d0ac897dcfb7a7e737bb01b824d6",
|
||||
"query": "\nSELECT id, confirmation_code, status AS `status: UserLoginChallengeStatus`, maybe_deciding_user_token, maybe_redeemed_user_session_id,\n maybe_failure_type AS `maybe_failure_type: UserLoginChallengeFailureType`, ip_address_creation, expires_at\nFROM user_login_challenges WHERE approval_token_sha256 = ? FOR UPDATE\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 6
|
||||
}
|
||||
},
|
||||
"hash": "cc29378fa77e7f09ca2ca0803127d111703a4edca24acfe3ccecf79c22fc1a80",
|
||||
"query": "\nINSERT INTO user_login_challenges\n (token, approval_token_sha256, device_token_sha256, confirmation_code, ip_address_creation, status, created_at, expires_at)\nVALUES (?, ?, ?, ?, ?, ?, NOW(), NOW() + INTERVAL 20 MINUTE)\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 4
|
||||
}
|
||||
},
|
||||
"hash": "d0067b5e1b3ef5dec5572eb2c0741d129ab2f706db4bb9f37a32bb89aa44fa4f",
|
||||
"query": "\nUPDATE user_login_challenges SET status = ?, maybe_failure_type = ?, maybe_failed_at = expires_at\nWHERE status IN (?, ?) AND expires_at <= NOW() ORDER BY expires_at LIMIT 1000\n"
|
||||
}
|
||||
+27
-5
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"query": "\nSELECT\n id,\n token as `token: MediaFileToken`,\n created_at as `created_at: DateTime<Utc>`,\n maybe_thumbnail_version as `maybe_thumbnail_version: u8`,\n public_bucket_directory_hash,\n maybe_public_bucket_prefix,\n maybe_public_bucket_extension\nFROM media_files\nWHERE\n id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)\n AND id < ?\n AND media_class = ?\n AND maybe_thumbnail_version IS NULL\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\nORDER BY id DESC\nLIMIT ?\n ",
|
||||
"query": "\nSELECT\n media.id,\n media.token as `token: MediaFileToken`,\n media.created_at as `created_at: DateTime<Utc>`,\n NOW(6) as `database_read_at!: DateTime<Utc>`,\n jobs.successfully_completed_at as `maybe_generation_completed_at?: DateTime<Utc>`,\n media.maybe_thumbnail_version as `maybe_thumbnail_version: u8`,\n media.public_bucket_directory_hash,\n media.maybe_public_bucket_prefix,\n media.maybe_public_bucket_extension\nFROM media_files AS media\nLEFT JOIN generic_inference_jobs AS jobs ON jobs.token = media.maybe_source_job_token\nWHERE\n media.id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)\n AND media.id < ?\n AND media.media_class = ?\n AND media.maybe_thumbnail_version IS NULL\n AND media.user_deleted_at IS NULL\n AND media.mod_deleted_at IS NULL\nORDER BY media.id DESC\nLIMIT ?\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -35,6 +35,26 @@
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "database_read_at!: DateTime<Utc>",
|
||||
"type_info": {
|
||||
"type": "Datetime",
|
||||
"flags": "NOT_NULL | BINARY",
|
||||
"char_set": 63,
|
||||
"max_size": 26
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "maybe_generation_completed_at?: DateTime<Utc>",
|
||||
"type_info": {
|
||||
"type": "Timestamp",
|
||||
"flags": "BINARY",
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "maybe_thumbnail_version: u8",
|
||||
"type_info": {
|
||||
"type": "Tiny",
|
||||
@@ -44,7 +64,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"ordinal": 6,
|
||||
"name": "public_bucket_directory_hash",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
@@ -54,7 +74,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"ordinal": 7,
|
||||
"name": "maybe_public_bucket_prefix",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
@@ -64,7 +84,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"ordinal": 8,
|
||||
"name": "maybe_public_bucket_extension",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
@@ -81,11 +101,13 @@
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "daed2dcd03b3c9fbdb9057b743df078f43e2af87a2c31f418234f98da5ff979f"
|
||||
"hash": "d14ad3a64d261e64fa75f97c93ca74af97fad8591d6ebe9887aeabc43bed0982"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 8
|
||||
}
|
||||
},
|
||||
"hash": "da64968638e564d28ed180270df407157f6c9b3f27c3443417f30c12c158ab89",
|
||||
"query": "\nUPDATE user_login_challenges SET status = ?, maybe_deciding_user_token = ?,\n maybe_ip_address_decision = ?, maybe_decided_at = NOW(), maybe_failure_type = ?,\n maybe_ip_address_failure = ?, maybe_failed_at = IF(?, NULL, NOW())\nWHERE id = ? AND status = ? AND expires_at > NOW()\n"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Right": 4
|
||||
}
|
||||
},
|
||||
"hash": "eb5fdd0e93978edfbe97c58238ea81f0c21dba047975beb53a39224894e9d41c",
|
||||
"query": "\nINSERT INTO user_sessions (token, user_token, ip_address_creation, maybe_creation_type, expires_at)\nSELECT ?, token, ?, ?, NOW() + INTERVAL 1 YEAR FROM users\nWHERE token = ? AND is_banned = FALSE AND user_deleted_at IS NULL AND mod_deleted_at IS NULL\n"
|
||||
}
|
||||
@@ -32,12 +32,45 @@ artcraft-services/
|
||||
│ ├── plugins/ # Shared billing and service components
|
||||
│ └── web/storyteller_web/ # Main Actix Web HTTP API
|
||||
├── frontend/
|
||||
│ ├── apps/artcraft-webapp/ # Browser application at app.getartcraft.com
|
||||
│ ├── apps/artcraft-website/ # Product website at getartcraft.com
|
||||
│ └── libs/ # Shared React components and TypeScript libraries
|
||||
│ ├── apps/artcraft-webapp/ # Browser application at app.getartcraft.com
|
||||
│ ├── apps/artcraft-website-next/ # Current public website at getartcraft.com (Next.js)
|
||||
│ ├── apps/artcraft-website/ # Legacy product website (Vite)
|
||||
│ └── libs/ # Shared React components and TypeScript libraries
|
||||
└── Cargo.toml # Rust workspace
|
||||
```
|
||||
|
||||
## Updating Desktop Download Links
|
||||
|
||||
Desktop release links are configured independently in **three places**. When
|
||||
updating the advertised ArtCraft release, update both `WINDOWS_VERSION` and
|
||||
`MAC_VERSION` in all three unless the request explicitly targets one platform or
|
||||
app:
|
||||
|
||||
1. **Current public website (Next.js):**
|
||||
[frontend/apps/artcraft-website-next/src/lib/download-links.ts](frontend/apps/artcraft-website-next/src/lib/download-links.ts).
|
||||
This controls the installer links and visible version at
|
||||
`https://getartcraft.com/download`. Add the release to `DOWNLOAD_HISTORY`
|
||||
(newest first), then update both version selectors.
|
||||
2. **Legacy website (Vite):**
|
||||
[frontend/apps/artcraft-website/src/config/github_download_links.ts](frontend/apps/artcraft-website/src/config/github_download_links.ts).
|
||||
Add the release to `DOWNLOAD_HISTORY` and update both version selectors.
|
||||
Updating only this file does **not** update the current public website.
|
||||
3. **Browser webapp:**
|
||||
[frontend/apps/artcraft-webapp/src/config/github_download_links.ts](frontend/apps/artcraft-webapp/src/config/github_download_links.ts).
|
||||
Update both version constants; these links are used by the welcome and
|
||||
checkout-success pages at `app.getartcraft.com`.
|
||||
|
||||
Keep previous releases for rollback. The Next.js download page reads its visible
|
||||
version from `DOWNLOAD_VERSIONS`, which uses the same selectors as the URLs; do
|
||||
not hardcode a separate version in the page component.
|
||||
|
||||
Check that all three configs produce the requested release tag and installer
|
||||
filenames for Windows (`ArtCraft_<version>_x64-setup.exe`) and macOS
|
||||
(`ArtCraft_<version>_universal.dmg`). Run `npm run typecheck` from
|
||||
`frontend/apps/artcraft-website-next` after changing its config. Repository edits
|
||||
require deployment of the affected apps before the public pages change; do not
|
||||
report a local update as a deployed update.
|
||||
|
||||
## Code Style
|
||||
|
||||
- Rust with no minimum supported version
|
||||
@@ -63,6 +96,18 @@ Organize for top-to-bottom reading. Important things first, details later.
|
||||
- Among helpers: meatier logic above leaf-level formatters
|
||||
- **In test modules**: constants first, then test cases (grouped into sub-modules when 2+), then helper functions last
|
||||
|
||||
## Public API Compatibility
|
||||
|
||||
- Public HTTP endpoints have deployed legacy clients. Preserve their request and
|
||||
response schemas, defaults, limits, and pagination behavior when optimizing them.
|
||||
- Put a changed contract behind a new endpoint/version and update new clients to
|
||||
call it explicitly. Do not add a new mode or change response fields on an existing
|
||||
public endpoint as part of a performance fix.
|
||||
- For an endpoint-local revision, version the operation path (for example,
|
||||
`/v1/media_files/list_v2/user/{username}`). Reserve a top-level `/v2` for broader
|
||||
API changes.
|
||||
- Add regression tests for the original contract alongside tests for the new route.
|
||||
|
||||
## Markdown
|
||||
|
||||
- **Tables must be space-padded so columns align in plain text.** Markdown
|
||||
|
||||
Generated
+4
@@ -8764,6 +8764,7 @@ dependencies = [
|
||||
"redis_common",
|
||||
"redis_schema",
|
||||
"regex",
|
||||
"reqwest 0.12.23",
|
||||
"resend-rs",
|
||||
"reusable_types",
|
||||
"rootly_client",
|
||||
@@ -8775,6 +8776,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"server_environment",
|
||||
"sha1_hash",
|
||||
"sha2 0.11.0-rc.5",
|
||||
"shared_env_var_config",
|
||||
"speculoos",
|
||||
"sqlx",
|
||||
@@ -10384,11 +10386,13 @@ dependencies = [
|
||||
"anyhow",
|
||||
"bootstrap",
|
||||
"bucket_paths",
|
||||
"chrono",
|
||||
"cloud_storage",
|
||||
"concurrency",
|
||||
"easyenv",
|
||||
"errors",
|
||||
"ffmpeg_utils",
|
||||
"futures 0.3.32",
|
||||
"jobs_common",
|
||||
"log",
|
||||
"mysql_queries",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
DROP TABLE IF EXISTS user_login_challenges;
|
||||
@@ -0,0 +1,93 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- One approval request for a new cookie-based user session on another device.
|
||||
-- See _database/sql/user_login_challenges.md for transitions and API requirements.
|
||||
-- This table does not authorize MCP sessions or API keys.
|
||||
CREATE TABLE user_login_challenges (
|
||||
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Non-secret identifier for logs/support. Never sufficient to approve or redeem.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- SHA-256 digests of TWO INDEPENDENT cryptographically random 256-bit tokens.
|
||||
-- Hash the canonical unpadded base64url token text; never store the raw tokens.
|
||||
-- The approval token goes in the browser/QR URL and only permits reviewing a
|
||||
-- request and submitting an authenticated user's explicit approve/decline choice.
|
||||
approval_token_sha256 BINARY(32) NOT NULL,
|
||||
|
||||
-- Only the initiating device receives this token. Required for polling/redeeming.
|
||||
-- Never include it in the approval URL, QR code, browser page, or application logs.
|
||||
device_token_sha256 BINARY(32) NOT NULL,
|
||||
|
||||
-- Eight random uppercase consonants, displayed as e.g. WDJB-MJHT on BOTH screens.
|
||||
-- Comparison aid only: not unique, not a lookup key, not an authentication factor.
|
||||
confirmation_code CHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
|
||||
-- UserLoginChallengeStatus: pending, approved, redeemed, failed.
|
||||
-- Only pending may be approved/declined. Only approved may mint a session.
|
||||
-- Expiry is checked independently of status on EVERY approval/redemption request.
|
||||
-- Application inserts explicitly bind UserLoginChallengeStatus::Pending.
|
||||
status VARCHAR(16) NOT NULL,
|
||||
|
||||
-- The authenticated web user who approved OR declined. NULL until a decision.
|
||||
-- On approval this becomes the immutable owner of the session to be minted.
|
||||
maybe_deciding_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Set atomically with status=redeemed and session insertion, under a row lock.
|
||||
-- Numeric reference avoids copying a session credential into this audit table.
|
||||
-- Intentionally signed to match the existing user_sessions.id column.
|
||||
-- At most one session per challenge; retries may only return that SAME live session.
|
||||
maybe_redeemed_user_session_id BIGINT(20) DEFAULT NULL,
|
||||
|
||||
-- UserLoginChallengeFailureType: user_declined, expired.
|
||||
-- Only failed rows have a failure type and maybe_failed_at.
|
||||
-- Transient errors / invalid credentials must not overwrite a valid challenge.
|
||||
maybe_failure_type VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== IP ADDRESSES ==========
|
||||
|
||||
-- Server-observed requesting device IP, displayed on the approval screen.
|
||||
-- Canonical IPv4/IPv6 strings; never accept an IP supplied in a request body.
|
||||
ip_address_creation VARCHAR(40) NOT NULL,
|
||||
|
||||
-- Browser/mobile IP for approval or denial; may differ from the device IP.
|
||||
maybe_ip_address_decision VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- Device IP at FIRST successful redemption. Retries do not overwrite it.
|
||||
maybe_ip_address_redemption VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- For user_declined this is the browser IP. For expiry there is no actor/IP.
|
||||
-- ip_address_creation is retained on ALL failures, including timeouts.
|
||||
maybe_ip_address_failure VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Immutable deadline: insert with created_at=NOW(), expires_at=NOW()+INTERVAL 20 MINUTE.
|
||||
-- The default deliberately fails closed if a writer forgets to set the deadline.
|
||||
-- Approval, polling, and retrying must NEVER extend it. NOW() >= expires_at is expired.
|
||||
expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
maybe_decided_at TIMESTAMP NULL DEFAULT NULL,
|
||||
maybe_redeemed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
|
||||
-- Decision time for user_declined; expires_at for expired (even if recorded later).
|
||||
maybe_failed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (approval_token_sha256),
|
||||
UNIQUE KEY (device_token_sha256),
|
||||
UNIQUE KEY (maybe_redeemed_user_session_id),
|
||||
KEY index_status_expires_at (status, expires_at),
|
||||
KEY index_expires_at (expires_at),
|
||||
KEY index_deciding_user_created_at (maybe_deciding_user_token, created_at),
|
||||
KEY index_creation_ip_created_at (ip_address_creation, created_at),
|
||||
KEY index_failure_ip_failed_at (maybe_ip_address_failure, maybe_failed_at)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
ALTER TABLE user_sessions
|
||||
DROP COLUMN maybe_creation_type;
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- UserWebSessionCreationType: direct_login, device_approval, impersonation.
|
||||
-- NULL means unknown / legacy writer, not direct_login. Do not backfill guesses.
|
||||
-- Additive metadata only: does not change validation, expiry, or impersonation rules.
|
||||
-- MCP sessions and API keys are separate tables and do not use this field.
|
||||
ALTER TABLE user_sessions
|
||||
ADD COLUMN maybe_creation_type VARCHAR(32) DEFAULT NULL;
|
||||
@@ -24,6 +24,11 @@ These are created with the `diesel` tool, and in particular:
|
||||
|
||||
- Primary key `id` columns must be `BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT`. Do not use signed IDs.
|
||||
- Always create tables with `ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`.
|
||||
- Application-owned state/status/type columns must not have schema defaults. Use
|
||||
`NOT NULL` without `DEFAULT` when required, and have application inserts explicitly
|
||||
supply the value. Document the corresponding Rust enum; application queries bind its
|
||||
variants instead of hardcoding enum strings in SQL. See
|
||||
`crates/schema/database/mysql_queries/AGENTS.md` for the query rules.
|
||||
- All SQL files should start with the standard IntelliJ noinspection comments:
|
||||
```sql
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
@@ -33,6 +38,7 @@ These are created with the `diesel` tool, and in particular:
|
||||
|
||||
## Rules
|
||||
|
||||
- Keep documentation in this directory as regular files, not symlinks. Diesel can treat a documentation symlink as a migration directory and reject the entire migration list.
|
||||
- Every `up.sql` must have a corresponding `down.sql` that cleanly reverses the migration.
|
||||
- For `CREATE TABLE` migrations, the `down.sql` is typically `DROP TABLE IF EXISTS table_name;`.
|
||||
- For `ALTER TABLE` migrations, the `down.sql` reverses the alteration (e.g. `DROP COLUMN`).
|
||||
@@ -1 +0,0 @@
|
||||
AGENTS.md
|
||||
@@ -0,0 +1,6 @@
|
||||
# SQL Migrations
|
||||
|
||||
Read and follow [AGENTS.md](AGENTS.md) in this directory.
|
||||
|
||||
This must remain a regular file. Diesel treats a symlink here as a migration
|
||||
directory and fails with `Invalid migration directory`.
|
||||
@@ -13,6 +13,9 @@ Update the corresponding squashed file whenever a migration changes a table's sc
|
||||
|
||||
- Primary key `id` columns must be `BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT`. Do not use signed IDs.
|
||||
- Always create tables with `ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`.
|
||||
- Application-owned state/status/type columns must not have schema defaults. Required
|
||||
values stay `NOT NULL` and are supplied explicitly by application code using the
|
||||
corresponding Rust enum. Keep these definitions consistent with the applied migrations.
|
||||
- All SQL files should start with the standard IntelliJ noinspection comments:
|
||||
```sql
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
-- One approval request for a new cookie-based user session on another device.
|
||||
-- See _database/sql/user_login_challenges.md for transitions and API requirements.
|
||||
-- This table does not authorize MCP sessions or API keys.
|
||||
CREATE TABLE user_login_challenges (
|
||||
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Non-secret identifier for logs/support. Never sufficient to approve or redeem.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- SHA-256 digests of TWO INDEPENDENT cryptographically random 256-bit tokens.
|
||||
-- Hash the canonical unpadded base64url token text; never store the raw tokens.
|
||||
-- The approval token goes in the browser/QR URL and only permits reviewing a
|
||||
-- request and submitting an authenticated user's explicit approve/decline choice.
|
||||
approval_token_sha256 BINARY(32) NOT NULL,
|
||||
|
||||
-- Only the initiating device receives this token. Required for polling/redeeming.
|
||||
-- Never include it in the approval URL, QR code, browser page, or application logs.
|
||||
device_token_sha256 BINARY(32) NOT NULL,
|
||||
|
||||
-- Eight random uppercase consonants, displayed as e.g. WDJB-MJHT on BOTH screens.
|
||||
-- Comparison aid only: not unique, not a lookup key, not an authentication factor.
|
||||
confirmation_code CHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
|
||||
-- UserLoginChallengeStatus: pending, approved, redeemed, failed.
|
||||
-- Only pending may be approved/declined. Only approved may mint a session.
|
||||
-- Expiry is checked independently of status on EVERY approval/redemption request.
|
||||
-- Application inserts explicitly bind UserLoginChallengeStatus::Pending.
|
||||
status VARCHAR(16) NOT NULL,
|
||||
|
||||
-- The authenticated web user who approved OR declined. NULL until a decision.
|
||||
-- On approval this becomes the immutable owner of the session to be minted.
|
||||
maybe_deciding_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Set atomically with status=redeemed and session insertion, under a row lock.
|
||||
-- Numeric reference avoids copying a session credential into this audit table.
|
||||
-- Intentionally signed to match the existing user_sessions.id column.
|
||||
-- At most one session per challenge; retries may only return that SAME live session.
|
||||
maybe_redeemed_user_session_id BIGINT(20) DEFAULT NULL,
|
||||
|
||||
-- UserLoginChallengeFailureType: user_declined, expired.
|
||||
-- Only failed rows have a failure type and maybe_failed_at.
|
||||
-- Transient errors / invalid credentials must not overwrite a valid challenge.
|
||||
maybe_failure_type VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== IP ADDRESSES ==========
|
||||
|
||||
-- Server-observed requesting device IP, displayed on the approval screen.
|
||||
-- Canonical IPv4/IPv6 strings; never accept an IP supplied in a request body.
|
||||
ip_address_creation VARCHAR(40) NOT NULL,
|
||||
|
||||
-- Browser/mobile IP for approval or denial; may differ from the device IP.
|
||||
maybe_ip_address_decision VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- Device IP at FIRST successful redemption. Retries do not overwrite it.
|
||||
maybe_ip_address_redemption VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- For user_declined this is the browser IP. For expiry there is no actor/IP.
|
||||
-- ip_address_creation is retained on ALL failures, including timeouts.
|
||||
maybe_ip_address_failure VARCHAR(40) DEFAULT NULL,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Immutable deadline: insert with created_at=NOW(), expires_at=NOW()+INTERVAL 20 MINUTE.
|
||||
-- The default deliberately fails closed if a writer forgets to set the deadline.
|
||||
-- Approval, polling, and retrying must NEVER extend it. NOW() >= expires_at is expired.
|
||||
expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
maybe_decided_at TIMESTAMP NULL DEFAULT NULL,
|
||||
maybe_redeemed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
|
||||
-- Decision time for user_declined; expires_at for expired (even if recorded later).
|
||||
maybe_failed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (approval_token_sha256),
|
||||
UNIQUE KEY (device_token_sha256),
|
||||
UNIQUE KEY (maybe_redeemed_user_session_id),
|
||||
KEY index_status_expires_at (status, expires_at),
|
||||
KEY index_expires_at (expires_at),
|
||||
KEY index_deciding_user_created_at (maybe_deciding_user_token, created_at),
|
||||
KEY index_creation_ip_created_at (ip_address_creation, created_at),
|
||||
KEY index_failure_ip_failed_at (maybe_ip_address_failure, maybe_failed_at)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -41,6 +41,12 @@ CREATE TABLE user_sessions (
|
||||
-- designed to be recoverable.
|
||||
deleted_at TIMESTAMP NULL,
|
||||
|
||||
-- UserWebSessionCreationType: direct_login, device_approval, impersonation.
|
||||
-- NULL means unknown / legacy writer, not direct_login. Do not backfill guesses.
|
||||
-- Additive metadata only: does not change validation, expiry, or impersonation rules.
|
||||
-- MCP sessions and API keys are separate tables and do not use this field.
|
||||
maybe_creation_type VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
# Website-approved device login
|
||||
|
||||
Implemented across storyteller-web, the browser approval page, and the Tauri
|
||||
login dialog. Apply the two migrations before deploying the API; deploy the web
|
||||
approval route before releasing the desktop buttons. No production migrations or
|
||||
deployments were performed while developing this change.
|
||||
|
||||
Call the flow **device approval**, and each attempt a **login challenge**. It works
|
||||
for a desktop browser link or a phone scanning a QR code, regardless of how the
|
||||
approving account signed in. It creates an ordinary cookie-based web session.
|
||||
|
||||
The design borrows separate device/browser credentials, polling, explicit consent,
|
||||
and code comparison from [OAuth device authorization (RFC 8628)](https://www.rfc-editor.org/rfc/rfc8628.html#section-3.3.1).
|
||||
It is an ArtCraft session bridge, not an OAuth token endpoint implementation.
|
||||
|
||||
## Two migrations
|
||||
|
||||
1. `2026-09-25-120000-0000_create_user_login_challenges` creates the challenge and
|
||||
its terminal outcome audit record together. A separate failure table is not
|
||||
needed for one immutable outcome per attempt. Status is `NOT NULL` without a
|
||||
default; inserts explicitly bind `UserLoginChallengeStatus::Pending`.
|
||||
2. `2026-09-25-120000-0001_alter_user_sessions_add_maybe_creation_type` adds nullable
|
||||
provenance to cookie sessions. Existing rows and older writers retain NULL.
|
||||
|
||||
Both have reverse migrations and matching materialized schemas under
|
||||
`migrations_squashed/user_tables/`. Deploy the schema before new writers. Roll
|
||||
back feature code before dropping the column/table; rollback removes bridge audit
|
||||
records but does not revoke sessions already issued.
|
||||
|
||||
`UserWebSessionCreationType` values:
|
||||
|
||||
| Rust variant | Stored value | Meaning |
|
||||
|------------------|-------------------|-------------------------------------------------------------------|
|
||||
| DirectLogin | direct_login | Password or SSO login/signup, including desktop password login. |
|
||||
| DeviceApproval | device_approval | A new session minted from an approved login challenge. |
|
||||
| Impersonation | impersonation | Existing consented staff impersonation flow. |
|
||||
| None | NULL | Unknown provenance, including existing rows and older writers. |
|
||||
|
||||
`DirectLogin` describes the creation mechanism more accurately than `Web`, since
|
||||
ordinary password login is also available in the desktop app. This field does not
|
||||
replace `maybe_impersonation_user_token` or influence authentication/authorization.
|
||||
MCP and API credentials continue using `mcp_sessions` and `api_keys` respectively.
|
||||
Existing password/SSO and impersonation session writers now record provenance.
|
||||
There is no historical backfill or default that would invent it.
|
||||
|
||||
## Credentials and confirmation
|
||||
|
||||
Generate two independent 32-byte CSPRNG values per challenge. Encode each as
|
||||
unpadded base64url (43 ASCII characters). Store only the SHA-256 digest of that
|
||||
canonical ASCII representation in the two uniquely indexed `BINARY(32)` columns.
|
||||
Reject noncanonical input instead of accepting multiple representations. These
|
||||
are random tokens, not passwords, so a fast cryptographic hash is appropriate.
|
||||
|
||||
| Value | Who receives it | What it permits |
|
||||
|---------------------|-------------------------------|--------------------------------------------------------|
|
||||
| token | Internal logs/support | Identifying a row; no authentication. |
|
||||
| approval token | Desktop, browser link, QR | Review and authenticated approval/decline only. |
|
||||
| device token | Initiating desktop only | Polling and collecting the newly authorized session. |
|
||||
| confirmation code | Desktop and approval screen | Visual comparison only; no lookup or authentication. |
|
||||
|
||||
Use eight uniformly random characters from `BCDFGHJKLMNPQRSTVWXZ` for the code.
|
||||
Store `WDJBMJHT`, display `WDJB-MJHT`, and ask the user to verify the match. It need
|
||||
not be unique because it is never used to locate or authorize a challenge. Manual
|
||||
short-code entry is outside this initial flow; both entry points carry the full
|
||||
approval token. The private device token must never be in the URL or QR code.
|
||||
|
||||
The website displays the requesting IP, the account being authorized, the matching
|
||||
code, and explicit **Approve desktop login** / **Decline** actions. An existing
|
||||
website session skips only the login screen, never consent. If login is needed,
|
||||
keep the pending approval context in first-party browser state and resume the
|
||||
confirmation page after password or Google SSO login. Recheck expiry after login.
|
||||
Do not send the bridge tokens to Google or another identity provider. Allow only
|
||||
the fixed first-party return route, not an arbitrary caller-supplied redirect URL.
|
||||
|
||||
The approval page is a read-only GET; link scanners and previews cannot decide
|
||||
the challenge. Decisions require a valid, non-impersonated user web session plus
|
||||
CSRF/origin protection. Never accept an API key or MCP credential as consent.
|
||||
Require the user to have initiated the request on a device in their possession;
|
||||
the visible code and IP help detect phishing but are not authentication factors.
|
||||
|
||||
The approval URL uses HTTPS and carries the browser credential in its fragment.
|
||||
The page captures and removes that fragment before referral collection or React
|
||||
providers initialize, skips analytics/referral tracking on direct entry, and sets
|
||||
a no-referrer policy. API responses send `Cache-Control: no-store` and
|
||||
`Referrer-Policy: no-referrer`; API credentials travel in JSON bodies, never query
|
||||
strings. Shared fonts and the Google sign-in provider still load. Keep request and
|
||||
response bodies out of gateway logs. Invalid credentials reveal no account or
|
||||
challenge details. The IP follows the existing trusted-gateway forwarding policy;
|
||||
the gateway must replace untrusted incoming forwarding headers.
|
||||
|
||||
## State and the 20-minute deadline
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> pending
|
||||
pending --> approved: authenticated approval
|
||||
pending --> failed: user_declined
|
||||
pending --> failed: expired
|
||||
approved --> failed: expired before redemption
|
||||
approved --> redeemed: atomically mint one session
|
||||
redeemed --> [*]
|
||||
failed --> [*]
|
||||
```
|
||||
|
||||
Set `created_at = NOW()` and `expires_at = NOW() + INTERVAL 20 MINUTE` in the same
|
||||
INSERT using the database's clock. The default expiry is immediate so a writer
|
||||
that omits it fails closed. Neither approval nor polling extends this deadline.
|
||||
All eligibility checks require `expires_at > NOW()`; equality means expired.
|
||||
The create query enforces this exact lifetime; these migrations do not add
|
||||
triggers or database CHECK constraints. Keep database connections in UTC.
|
||||
|
||||
The persisted status alone is insufficient to decide validity. A pending or
|
||||
approved row whose deadline passed is already expired even before a sweeper
|
||||
marks it `failed`. Approvals and first redemptions must check the deadline again
|
||||
under the row lock. Polls are normally read-only and do not touch `updated_at`.
|
||||
|
||||
| Status | Required accompanying data |
|
||||
|------------|-------------------------------------------------------------------------------------|
|
||||
| pending | No decision, failure, redemption, or session reference. |
|
||||
| approved | Deciding user, decision IP/time; no failure, redemption, or session reference. |
|
||||
| redeemed | Approval data plus session ID, first redemption IP/time; no failure. |
|
||||
| failed | Failure type/time; no redemption or session reference; retain any prior approval. |
|
||||
|
||||
`UserLoginChallengeFailureType` initially has two values:
|
||||
|
||||
- `user_declined`: record the deciding user, browser IP in both decision/failure
|
||||
IP fields, and the decision time in both decision/failure timestamps. Keep the
|
||||
original desktop IP. Polling with the valid device token reports the decline.
|
||||
- `expired`: record `maybe_failed_at = expires_at`. Retain original desktop IP and
|
||||
any prior approval, but leave the failure IP NULL: a timeout has no actor. A
|
||||
scheduled sweep records abandoned requests even when nobody polls again.
|
||||
|
||||
Failed challenges are unusable for approval or redemption. Their digests and
|
||||
audit metadata remain for diagnosing failures; do not physically delete the row
|
||||
when a user declines. A new attempt creates a new row and fresh tokens. Closing
|
||||
the desktop dialog stops polling; unused requests expire normally.
|
||||
|
||||
Invalid tokens, failed website sign-ins, throttling, and transient infrastructure
|
||||
errors do not terminally fail a valid challenge. Record those requests in the
|
||||
existing operational/abuse logging with their IP and failure category, without
|
||||
credentials. A second decision cannot replace the first actor or outcome.
|
||||
|
||||
## Atomic redemption and response loss
|
||||
|
||||
The website's approval records consent; it never creates or receives the desktop
|
||||
cookie. The desktop's authenticated poll performs redemption after approval:
|
||||
|
||||
1. Begin an InnoDB transaction on the primary and find the challenge by the device
|
||||
token digest with `SELECT ... FOR UPDATE`.
|
||||
2. Validate the immutable deadline and state after taking the lock. Handle
|
||||
failed/pending states without creating a session. Revalidate that the approved
|
||||
user exists and is permitted to log in under the ordinary session rules.
|
||||
3. For `approved`, insert a fresh `user_sessions` row owned by the deciding user,
|
||||
with `maybe_creation_type = 'device_approval'`, no impersonation, the current
|
||||
desktop redemption IP, and the ordinary web-session lifetime.
|
||||
4. Store its numeric ID, redemption IP/time, and `status = 'redeemed'` on the
|
||||
challenge in that same transaction. Generate the signed cookie using
|
||||
`HttpUserSessionManager::create_cookie` before commit, so signing failure rolls
|
||||
back too. Commit successfully before returning any credential to the caller.
|
||||
|
||||
Both approval/decline and redemption must serialize on the same challenge row;
|
||||
conditional updates include the expected state and the unexpired deadline, and
|
||||
must check rows affected. SQL for this flow belongs in `mysql_queries`, with
|
||||
executor/transaction support, not in the web handler. All three Rust enums in this
|
||||
change are database enums; separate client-facing status/failure enums include
|
||||
an unknown-value fallback. Desktop callers fail closed on an unknown state.
|
||||
|
||||
One-time means **at most one new session per challenge**. If the first successful
|
||||
HTTP response is lost, a retry with the same device token may return the same
|
||||
session until the original 20-minute deadline. For `redeemed`, read the linked
|
||||
session and verify its owner, creation type, absence of impersonation, expiry,
|
||||
deletion state, and the user's current eligibility before reconstructing its
|
||||
cookie. Never mint a replacement or resurrect a deleted session. After the
|
||||
deadline, return expiry without a cookie; the already-issued session keeps its
|
||||
normal lifetime, and the audit outcome remains `redeemed`.
|
||||
|
||||
The unique nullable session ID index prevents two challenges from referencing one
|
||||
session and supports retries without duplicating bearer credentials in this
|
||||
table. The reference is intentionally signed because the legacy `user_sessions.id`
|
||||
is signed. Like neighboring schemas, references are indexed logical links rather
|
||||
than cascading foreign keys; removal of a session must not erase the audit row.
|
||||
If the referenced session is missing, redemption fails closed.
|
||||
|
||||
## Implemented endpoints and clients
|
||||
|
||||
All endpoints are JSON POSTs under `/v1/login_challenges`:
|
||||
|
||||
| Endpoint | Authentication | Effect |
|
||||
|------------|---------------------------------------------|--------------------------------------------------------------|
|
||||
| create | None; limited by requesting IP | Returns device token, verification URL, code, expiry. |
|
||||
| review | Approval token + web session + Origin | Returns code, requesting IP, account, state, expiry. |
|
||||
| decide | Approval token + web session + Origin | Records the first Yes/No; never creates a desktop session. |
|
||||
| poll | Private device token | Reports state or atomically redeems approved consent. |
|
||||
|
||||
The API advertises five-second polling. Per-process IP limits are ten creates and
|
||||
120 other requests per minute, shared across Actix workers, with bounded memory.
|
||||
Production gateways can additionally enforce aggregate limits across replicas.
|
||||
A worker expires up to 1,000 abandoned challenges every 30 seconds. Audit retention
|
||||
is intentionally not an automatic purge policy in this change.
|
||||
|
||||
Both browser applications implement `/login/desktop`: `https://getartcraft.com`
|
||||
(including `www`) and `https://app.getartcraft.com`. They share the approval UI in
|
||||
`frontend/libs/login`. The API currently advertises the webapp URL. Local API
|
||||
requests advertise `http://localhost:4201/login/desktop`; the website also handles
|
||||
the route on port 4200. Run either frontend with `USE_LOCAL_API=1` so approval
|
||||
reaches the same local API; local origins cannot approve production challenges.
|
||||
The website signs in inline to preserve its origin-scoped pending token and
|
||||
mobile session header. The webapp preserves the return destination through login,
|
||||
signup, and password reset, bypassing billing redirects for desktop consent.
|
||||
Tokens stay in tab-scoped session storage, are removed from URL fragments before
|
||||
tracking, and are cleared on terminal results. Returning from authentication
|
||||
always requires explicit consent. The code stays visible for comparison; a single
|
||||
Approve or Decline click records the choice, without an extra checkbox.
|
||||
The desktop displays the QR code, code comparison, countdown, and status. Opening
|
||||
the website invokes the native system browser. Network errors back off up to 30
|
||||
seconds; closing the dialog stops polling and cancels the native handle. A response
|
||||
arriving after cancellation cannot install credentials.
|
||||
|
||||
Successful polling returns the signed value and the normal `Set-Cookie` header.
|
||||
Tauri commands use the Rust `artcraft_client` bindings for creation, polling, and
|
||||
ordinary session verification, all against `AppEnvConfigs.storyteller_host`. Rust
|
||||
keeps the private device token and signed session out of IPC/JavaScript; the UI
|
||||
holds only a local handle and public display data. After verifying the downstream
|
||||
session, Rust installs and persists the cookie in the native HTTP jar and updates
|
||||
the credential manager. No browser session is copied into the desktop.
|
||||
The desktop login/signup modal also uses Rust commands for its initial session
|
||||
check and password forms. Website/QR buttons appear only in login mode. Unknown
|
||||
optional feature flags in the session response are ignored by older desktops;
|
||||
malformed authentication data still fails verification. A stale initial session
|
||||
check cannot reset a pending or completed login.
|
||||
|
||||
## Integration tests
|
||||
|
||||
The backend harness uses the fixed local Unix socket `/tmp/mysql.sock` and local
|
||||
root account. It never reads `DATABASE_URL`, any environment database URL, DNS,
|
||||
or a TCP endpoint (including localhost SSH forwarding). Each test creates a new
|
||||
randomly named `artcraft_test_bridge_*` database, replays all migrations, and drops
|
||||
only its own database on success. Existing `storyteller` and `artcraft_test` data
|
||||
are not used. A failed test leaves its isolated database for diagnosis. The local
|
||||
MySQL instance must be running; there is no fallback to a remote database.
|
||||
|
||||
The HTTP endpoint suite is
|
||||
`crates/service/web/storyteller_web/src/http_server/endpoints/login_challenges/endpoint_integration_tests.rs`.
|
||||
It starts the real challenge handlers and `/v1/session` handler on an ephemeral
|
||||
`127.0.0.1` port. Separate browser and desktop HTTP clients maintain their own
|
||||
cookie jars; requests cannot follow redirects or use system proxies. Fixtures in
|
||||
`mysql_testing::fixtures::login_challenges` create passwordless users, Google email
|
||||
confirmation, feature flags (including `use_qt`), and signed browser sessions.
|
||||
The desktop cookie comes only from the real redemption response and is then sent
|
||||
to the real session endpoint. HTTP responses and SQL queries are not mocked.
|
||||
|
||||
From the services repository:
|
||||
|
||||
```sh
|
||||
SQLX_OFFLINE=true cargo test --offline -p storyteller-web --bin storyteller-web login_challenges
|
||||
cd frontend
|
||||
npm exec vitest -- run --config apps/artcraft-webapp/vite.config.ts src/pages/login/desktop-login.spec.tsx src/pages/login/login-bridge-continuation.spec.tsx
|
||||
npm exec vitest -- run --config apps/artcraft-website/vite.config.ts src/pages/login/desktop-login.spec.tsx
|
||||
```
|
||||
|
||||
`SQLX_OFFLINE` controls compile-time query metadata only. These tests still execute
|
||||
real SQL against local MySQL. Do not enable `skip_database_tests` for this run.
|
||||
|
||||
To run only the four HTTP endpoint tests with database fixtures:
|
||||
|
||||
```sh
|
||||
SQLX_OFFLINE=true cargo test --offline -p storyteller-web --bin storyteller-web login_challenges::endpoint_integration_tests
|
||||
```
|
||||
|
||||
From the desktop repository:
|
||||
|
||||
```sh
|
||||
SQLX_OFFLINE=true cargo test --offline -p artcraft --lib login_bridge_tests
|
||||
cd frontend
|
||||
npm exec vitest -- run --config libs/components/login-modal/vite.config.ts src/lib/DesktopLoginBridge.spec.tsx src/lib/login-modal.spec.tsx
|
||||
```
|
||||
|
||||
Coverage:
|
||||
|
||||
- Four HTTP endpoint integration tests with database fixtures: the complete
|
||||
create/review/approve/redeem/session flow, independent browser/desktop cookies,
|
||||
passwordless onboarding and `use_qt` serialization, concurrent redemption,
|
||||
lost-response recovery with a fresh client, no sessions before redemption,
|
||||
audit ownership/IPs/timestamps, decline, pending/approved expiry, downstream
|
||||
session revocation/expiry/bans, and immutable approval ownership across accounts.
|
||||
- Seven additional backend handler integration tests: both production browser origins, passwordless account approval, zero sessions
|
||||
before redemption, rejection/audit IPs, token separation, CSRF, MCP rejection,
|
||||
invalid approving sessions, concurrent decisions/redemptions, response-loss
|
||||
retries, abandoned expiry, expiry during lock contention, revocation, bans,
|
||||
signed-cookie authentication through the production session guard, and an issued
|
||||
session remaining valid after the challenge expires.
|
||||
- Twenty-five browser tests: API/consent interaction, code/IP display, decline,
|
||||
expiry, terminal/unknown states, lost-response retries, safe return URLs,
|
||||
website inline Google/password sign-in, and webapp login/signup/password-reset
|
||||
continuation without subscription gating or bridge credentials in SSO requests.
|
||||
- Thirteen desktop UI tests: IPC-only transport (JavaScript HTTP is forbidden),
|
||||
native browser invocation, QR/code values, polling, verified user results,
|
||||
decline, timeout, network backoff, cancellation, host/status diagnostics,
|
||||
login-only button placement, native password forms, and stale session checks.
|
||||
- Thirteen native tests: real API bindings against ephemeral loopback HTTP servers,
|
||||
downstream verification before cookie installation, persistence, retries,
|
||||
rejection, timeout, cancellation during HTTP, unknown states, no redirects,
|
||||
redacted errors, local/production origin isolation, visitor-only telemetry,
|
||||
forward-compatible feature flags, malformed session data, and native password
|
||||
login/signup/session recheck. Desktop tests do not connect to MySQL.
|
||||
|
||||
Frontend tests replace external Google/native boundaries; they do not
|
||||
perform a live Google OAuth exchange or drive a packaged Tauri GUI. Backend tests
|
||||
use real MySQL and the production cookie signer/session guard with Redis disabled;
|
||||
the HTTP suite also exercises the real session endpoint and its serialized body.
|
||||
These endpoint tests do not drive a live Google OAuth exchange or the Tauri GUI.
|
||||
Existing session-cache TTL behavior is unchanged. SQL lookups now reject expired
|
||||
sessions; bridge consent/redemption always checks MySQL directly.
|
||||
|
||||
Validation also covered both migration directions, nullable compatibility with
|
||||
legacy session writers, uniqueness constraints, fail-closed expiry defaults, and
|
||||
matching materialized schemas. The focused web TypeScript check and the production
|
||||
builds of both browser apps (including 53 dependency tasks) passed. Broad workspace TypeScript
|
||||
checks have existing errors. The desktop Nx build is blocked by the existing
|
||||
`api -> tauri-api -> api` cycle; its direct Vite login bundle builds but declaration
|
||||
checks report unavailable dependency outputs. A source-only desktop check reports
|
||||
four existing `model-list` imports of `src/index.js`, with no bridge-file errors.
|
||||
The full native Rust test target compiled successfully.
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use url::Url;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::common::responses::cover_image_links::CoverImageLinks;
|
||||
use crate::common::responses::media_file_cover_image_details::MediaFileDefaultCover;
|
||||
|
||||
// User-media list endpoints retain the deprecated bucket fields for compatibility.
|
||||
/// Everything we need to create a cover image.
|
||||
/// Cover images are small descriptive images that can be set for any media file.
|
||||
/// If a cover image is set, this is the path to the asset.
|
||||
#[derive(Serialize, Deserialize, ToSchema)]
|
||||
pub struct LegacyMediaFileCoverImageDetails {
|
||||
/// (DEPRECATED) URL path to the media file
|
||||
#[deprecated(note="This field doesn't point to the full URL. Use media_links instead to leverage the CDN.")]
|
||||
pub maybe_cover_image_public_bucket_path: Option<String>,
|
||||
|
||||
/// (DEPRECATED) Full URL to the media file
|
||||
#[deprecated(note="This points to the bucket. Use media_links instead to leverage the CDN.")]
|
||||
pub maybe_cover_image_public_bucket_url: Option<Url>,
|
||||
|
||||
/// Links to the cover image (CDN direct link, thumbnail template)
|
||||
/// If a cover image is set, this is the path to the asset.
|
||||
/// If a cover image is not set, use the information in `default_cover` instead.
|
||||
/// Rich CDN links to the media, including thumbnails, previews, and more.
|
||||
pub maybe_links: Option<CoverImageLinks>,
|
||||
|
||||
/// For items without a cover image, we can use one of our own.
|
||||
pub default_cover: MediaFileDefaultCover,
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use tokens::tokens::model_weights::ModelWeightToken;
|
||||
|
||||
/// Fields useful for enriching media file listings
|
||||
#[derive(Clone, Serialize, Deserialize, ToSchema)]
|
||||
pub struct MediaFileOriginDetails {
|
||||
/// Where the file came from (broadly)
|
||||
pub origin_category: MediaFileOriginCategory,
|
||||
|
||||
/// Where the file came from (specifically, a product area, eg. "face_animator")
|
||||
pub product_category: MediaFileOriginProductCategory,
|
||||
|
||||
/// If the file was produced by a model, the details.
|
||||
pub maybe_model: Option<MediaFileModelDetails>,
|
||||
}
|
||||
|
||||
/// Information about the model weights
|
||||
/// https://serde.rs/enum-representations.html#untagged
|
||||
#[derive(Clone, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(untagged)]
|
||||
pub enum MediaFileModelDetails {
|
||||
// Try the specific shape first so deserialization retains token and title.
|
||||
/// A model weight is typically a user-submitted model.
|
||||
/// We store lots of these.
|
||||
ModelWeight {
|
||||
/// The type of model weight
|
||||
model_type: MediaFileOriginModelType,
|
||||
/// The model token
|
||||
token: ModelWeightToken,
|
||||
/// The model title (typically only populated for `model_weights` models, not legacy tables such as `tts_models`.)
|
||||
title: String,
|
||||
},
|
||||
/// A system model is one that we maintain and update,
|
||||
/// eg. SadTalker, Wav2Lip, MocapNet, etc.
|
||||
SystemModel {
|
||||
/// The type of model weight
|
||||
model_type: MediaFileOriginModelType,
|
||||
},
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
pub mod cover_image_links;
|
||||
pub mod job_details;
|
||||
pub mod legacy_media_file_cover_image_details;
|
||||
pub mod media_file_cover_image_details;
|
||||
pub mod media_file_origin_details;
|
||||
pub mod media_links;
|
||||
pub mod pagination_cursors;
|
||||
pub mod simple_entity_stats;
|
||||
pub mod simple_generic_json_success;
|
||||
pub mod user_details_light;
|
||||
pub mod user_details_light;
|
||||
+2
-1
@@ -1,9 +1,10 @@
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
// TODO(bt,2025-08-20): Replace the storyteller-web version of this.
|
||||
|
||||
/// Simple stats that can be attached to any entity
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, ToSchema, Debug)]
|
||||
pub struct SimpleEntityStats {
|
||||
/// Number of positive ratings (or "likes") for this item
|
||||
pub positive_rating_count: u32,
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use utoipa::{IntoParams, ToSchema};
|
||||
|
||||
use crate::media_file::list::user_media_file_list_item::MediaFileForUserListItem;
|
||||
|
||||
pub const LIST_MEDIA_FILES_FOR_USER_V2_URL_PATH: &str = "/v1/media_files/list_v2/user/{username}";
|
||||
|
||||
#[derive(Serialize, Deserialize, ToSchema)]
|
||||
pub struct ListMediaFilesForUserV2PathInfo {
|
||||
pub username: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, ToSchema, IntoParams)]
|
||||
pub struct ListMediaFilesForUserV2QueryParams {
|
||||
pub sort_ascending: Option<bool>,
|
||||
pub page_size: Option<usize>,
|
||||
pub page_index: Option<usize>,
|
||||
|
||||
/// NB: This can be one (or more comma-separated values) from `MediaFileClass`,
|
||||
/// which are the broad category of media files: image, video, etc.
|
||||
///
|
||||
/// Usage:
|
||||
/// - `?filter_media_classes=audio`
|
||||
/// - `?filter_media_classes=image,video`
|
||||
/// - `?filter_media_classes=dimensional`
|
||||
/// - etc.
|
||||
pub filter_media_classes: Option<String>,
|
||||
|
||||
/// NB: This can be one (or more comma-separated values) from `MediaFileType`,
|
||||
/// which are mimetype-like / format-like categories of media files: glb, gltf,
|
||||
/// scene_json, jpg, png, mp4, wav, etc.
|
||||
///
|
||||
/// Usage:
|
||||
/// - `?filter_media_type=scene_json`
|
||||
/// - `?filter_media_type=glb,gltf,fbx`
|
||||
/// - `?filter_media_type=pmd,vmd,pmx`
|
||||
/// - `?filter_media_type=jpg,png,gif`
|
||||
/// - `?filter_media_type=wav,mp3`
|
||||
/// - `?filter_media_type=mp4`
|
||||
/// - etc.
|
||||
pub filter_media_type: Option<String>,
|
||||
|
||||
/// NB: This can be one (or more comma-separated values) from `MediaFileEngineCategory`.
|
||||
///
|
||||
/// Usage:
|
||||
/// - `?filter_engine_categories=scene`
|
||||
/// - `?filter_engine_categories=animation,character,object`
|
||||
/// - etc.
|
||||
pub filter_engine_categories: Option<String>,
|
||||
|
||||
/// Include user uploaded files in the results.
|
||||
/// By default, we do not return them unless this flag is set to true.
|
||||
pub include_user_uploads: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, ToSchema)]
|
||||
pub struct ListMediaFilesForUserV2SuccessResponse {
|
||||
pub success: bool,
|
||||
pub results: Vec<MediaFileForUserListItem>,
|
||||
pub pagination: UserMediaFileV2Pagination,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, ToSchema)]
|
||||
pub struct UserMediaFileV2Pagination {
|
||||
pub current: usize,
|
||||
pub has_more: bool,
|
||||
}
|
||||
@@ -1 +1,3 @@
|
||||
pub mod by_type;
|
||||
pub mod list_media_files_for_user_v2;
|
||||
pub mod user_media_file_list_item;
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use enums::common::visibility::Visibility;
|
||||
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
use tokens::tokens::prompts::PromptToken;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::common::responses::legacy_media_file_cover_image_details::LegacyMediaFileCoverImageDetails;
|
||||
use crate::common::responses::media_file_origin_details::MediaFileOriginDetails;
|
||||
use crate::common::responses::media_links::MediaLinks;
|
||||
use crate::common::responses::simple_entity_stats::SimpleEntityStats;
|
||||
|
||||
#[derive(Serialize, Deserialize, ToSchema)]
|
||||
pub struct MediaFileForUserListItem {
|
||||
pub token: MediaFileToken,
|
||||
|
||||
/// The coarse-grained class of media file: image, video, etc.
|
||||
pub media_class: MediaFileClass,
|
||||
|
||||
/// Type of media will dictate which fields are populated and what
|
||||
/// the frontend should display (eg. video player vs audio player).
|
||||
/// This is closer in meaning to a "mime type".
|
||||
pub media_type: MediaFileType,
|
||||
|
||||
/// If this is an engine/3D asset, this is the broad category (scene,
|
||||
/// animation, etc.) of that object.
|
||||
/// This can also be used for filtering in list/batch endpoints.
|
||||
pub maybe_engine_category: Option<MediaFileEngineCategory>,
|
||||
|
||||
/// If this is an engine/3D asset for an animation or a rig that can
|
||||
/// be animated with either (or both) skeletal or blend shape animations,
|
||||
/// this describes the animation regime used or supported.
|
||||
pub maybe_animation_type: Option<MediaFileAnimationType>,
|
||||
|
||||
/// Details where the media file came from.
|
||||
pub origin: MediaFileOriginDetails,
|
||||
|
||||
#[deprecated(note="Use MediaFileOriginDetails instead")]
|
||||
pub origin_category: MediaFileOriginCategory,
|
||||
|
||||
#[deprecated(note="Use MediaFileOriginDetails instead")]
|
||||
pub origin_product_category: MediaFileOriginProductCategory,
|
||||
|
||||
#[deprecated(note="Use MediaFileOriginDetails instead")]
|
||||
pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
|
||||
#[deprecated(note="Use MediaFileOriginDetails instead")]
|
||||
pub maybe_origin_model_token: Option<String>,
|
||||
|
||||
/// (DEPRECATED) URL path to the media file
|
||||
#[deprecated(note="This field doesn't point to the full URL. Use media_links instead to leverage the CDN.")]
|
||||
pub public_bucket_path: String,
|
||||
|
||||
/// (DEPRECATED) Full URL to the media file
|
||||
#[deprecated(note="This points to the bucket. Use media_links instead to leverage the CDN.")]
|
||||
pub public_bucket_url: String,
|
||||
|
||||
/// Rich CDN links to the media, including thumbnails, previews, and more.
|
||||
pub media_links: MediaLinks,
|
||||
|
||||
/// The foreign key to the prompt used to generate the media, if applicable.
|
||||
pub maybe_prompt_token: Option<PromptToken>,
|
||||
|
||||
/// Information about the cover image. Many media files do not require a cover image,
|
||||
/// e.g. image files, video files with thumbnails, audio files, etc.
|
||||
/// 3D files require them.
|
||||
pub cover_image: LegacyMediaFileCoverImageDetails,
|
||||
|
||||
pub creator_set_visibility: Visibility,
|
||||
|
||||
/// The file was uploaded by the user.
|
||||
/// This does not include files generated on the client side, like studio renders.
|
||||
pub is_user_upload: bool,
|
||||
|
||||
/// The file was created by the system.
|
||||
/// This includes files generated on the client side, like studio renders.
|
||||
pub is_intermediate_system_file: bool,
|
||||
|
||||
/// The name or title of the media file (optional)
|
||||
pub maybe_title: Option<String>,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
pub maybe_text_transcript: Option<String>,
|
||||
|
||||
/// For Comfy / Video Style Transfer jobs, this might include
|
||||
/// the name of the selected style.
|
||||
pub maybe_style_name: Option<StyleTransferName>,
|
||||
|
||||
/// Duration for audio and video files, if available.
|
||||
/// Measured in milliseconds.
|
||||
pub maybe_duration_millis: Option<u64>,
|
||||
|
||||
/// Statistics about the media file
|
||||
pub stats: SimpleEntityStats,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct CreateLoginChallengeRequest {}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct ReviewLoginChallengeRequest {
|
||||
pub approval_token: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct DecideLoginChallengeRequest {
|
||||
pub approval_token: String,
|
||||
pub approve: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct PollLoginChallengeRequest {
|
||||
pub device_token: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct CreateLoginChallengeResponse {
|
||||
pub success: bool,
|
||||
pub device_token: String,
|
||||
pub verification_url: String,
|
||||
pub confirmation_code: String,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub poll_interval_seconds: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct ReviewLoginChallengeResponse {
|
||||
pub success: bool,
|
||||
pub status: LoginChallengeState,
|
||||
pub maybe_failure_type: Option<LoginChallengeFailure>,
|
||||
pub confirmation_code: String,
|
||||
pub requesting_ip: String,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub username: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, ToSchema)]
|
||||
pub struct LoginChallengeResponse {
|
||||
pub success: bool,
|
||||
pub status: LoginChallengeState,
|
||||
pub maybe_failure_type: Option<LoginChallengeFailure>,
|
||||
/// Present only to the device after the redemption transaction commits.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub maybe_signed_session: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize, ToSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum LoginChallengeState {
|
||||
Pending,
|
||||
Approved,
|
||||
Redeemed,
|
||||
Failed,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize, ToSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum LoginChallengeFailure {
|
||||
UserDeclined,
|
||||
Expired,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
@@ -2,4 +2,5 @@ pub mod change_password;
|
||||
pub mod edit_email;
|
||||
pub mod edit_username;
|
||||
pub mod login;
|
||||
pub mod session_info;
|
||||
pub mod login_challenges;
|
||||
pub mod session_info;
|
||||
+14
-11
@@ -8,6 +8,7 @@ use enums::common::view_as::ViewAs;
|
||||
use errors::{anyhow, AnyhowResult};
|
||||
use mysql_queries::queries::media_files::delete::delete_media_file::delete_media_file_as_mod;
|
||||
use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs};
|
||||
use mysql_queries::queries::media_files::list::user_media_file_filters::UserMediaFileFilters;
|
||||
use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username;
|
||||
|
||||
use crate::cli_args::Args;
|
||||
@@ -42,23 +43,25 @@ pub async fn delete_all_files(username: &str, mysql: &Pool<MySql>) -> AnyhowResu
|
||||
info!("Querying page {page_index} of files...");
|
||||
|
||||
let media_files = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
username,
|
||||
maybe_filter_media_types: None,
|
||||
maybe_filter_media_classes: None,
|
||||
maybe_filter_engine_categories: None,
|
||||
include_user_uploads: true,
|
||||
page_size: 100,
|
||||
page_index,
|
||||
filters: UserMediaFileFilters {
|
||||
username,
|
||||
maybe_filter_media_types: None,
|
||||
maybe_filter_media_classes: None,
|
||||
maybe_filter_engine_categories: None,
|
||||
include_user_uploads: true,
|
||||
view_as: ViewAs::Moderator,
|
||||
},
|
||||
limit: 100,
|
||||
offset: page_index * 100,
|
||||
sort_ascending: false,
|
||||
view_as: ViewAs::Moderator,
|
||||
mysql_pool: &mysql,
|
||||
mysql_executor: mysql,
|
||||
}).await?;
|
||||
|
||||
if media_files.records.is_empty() {
|
||||
if media_files.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for file in media_files.records {
|
||||
for file in media_files {
|
||||
info!("Deleting file: {:?}", &file.token);
|
||||
delete_media_file_as_mod(&file.token, ECHELON_USER_TOKEN, mysql).await?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
use actix_cors::Cors;
|
||||
|
||||
pub fn add_artcraft_desktop(cors: Cors, _is_production: bool) -> Cors {
|
||||
cors
|
||||
// Stable identity sent by the Artcraft native HTTP bridge (dev and packaged apps).
|
||||
.allowed_origin("https://desktop.getartcraft.com")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use actix_web::dev::Transform;
|
||||
use actix_web::http::{header, Method, StatusCode};
|
||||
use actix_web::test::{self, TestRequest};
|
||||
use server_environment::ServerEnvironment;
|
||||
|
||||
use crate::cors::build_cors_config;
|
||||
use crate::testing::{assert_origin_invalid, assert_origin_ok};
|
||||
|
||||
const DESKTOP_ORIGIN: &str = "https://desktop.getartcraft.com";
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn desktop_requests_and_preflights_allow_credentials() {
|
||||
for environment in [ServerEnvironment::Production, ServerEnvironment::Development] {
|
||||
let service = build_cors_config(environment)
|
||||
.new_transform(test::ok_service())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for method in [Method::GET, Method::POST, Method::OPTIONS] {
|
||||
let mut request = TestRequest::default()
|
||||
.uri("/v1/session")
|
||||
.insert_header((header::ORIGIN, DESKTOP_ORIGIN));
|
||||
if method == Method::OPTIONS {
|
||||
request = request
|
||||
.insert_header((header::ACCESS_CONTROL_REQUEST_METHOD, "POST"))
|
||||
.insert_header((header::ACCESS_CONTROL_REQUEST_HEADERS, "content-type,session,x-artcraft-version"));
|
||||
}
|
||||
let response = test::call_service(&service, request.method(method).to_srv_request()).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers().get(header::ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(), DESKTOP_ORIGIN);
|
||||
assert_eq!(response.headers().get(header::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn desktop_origin_is_an_exact_https_allowlist_entry() {
|
||||
let cors = build_cors_config(ServerEnvironment::Production);
|
||||
for origin in [
|
||||
"http://desktop.getartcraft.com",
|
||||
"https://desktop.getartcraft.com:8443",
|
||||
"https://desktop.getartcraft.com.example.com",
|
||||
"https://sub.desktop.getartcraft.com",
|
||||
] {
|
||||
assert_origin_invalid(&cors, origin).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn existing_desktop_clients_remain_allowed() {
|
||||
let cors = build_cors_config(ServerEnvironment::Production);
|
||||
for origin in ["https://studio.storyteller.ai", "http://tauri.localhost", "tauri://localhost"] {
|
||||
assert_origin_ok(&cors, origin).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
pub (crate) mod artcraft_admin_dashboard;
|
||||
pub (crate) mod artcraft_2d;
|
||||
pub (crate) mod artcraft_3d;
|
||||
pub (crate) mod artcraft_desktop;
|
||||
pub (crate) mod artcraft_webapp;
|
||||
pub (crate) mod artcraft_website;
|
||||
pub (crate) mod development_only;
|
||||
@@ -11,4 +12,4 @@ pub (crate) mod storyteller;
|
||||
pub (crate) mod storyteller_board;
|
||||
pub (crate) mod storyteller_render;
|
||||
pub (crate) mod storyteller_studio;
|
||||
pub (crate) mod tauri;
|
||||
pub (crate) mod tauri;
|
||||
@@ -7,3 +7,86 @@ pub fn add_tauri(cors: Cors, _is_production: bool) -> Cors {
|
||||
// Tauri Mac
|
||||
.allowed_origin("tauri://localhost")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use actix_web::dev::Transform;
|
||||
use actix_web::http::{header, Method, StatusCode};
|
||||
use actix_web::test::{self, TestRequest};
|
||||
use server_environment::ServerEnvironment;
|
||||
|
||||
use crate::cors::build_cors_config;
|
||||
use crate::testing::assert_origin_invalid;
|
||||
|
||||
const WINDOWS_ORIGIN: &str = "http://tauri.localhost";
|
||||
const MAC_ORIGIN: &str = "tauri://localhost";
|
||||
|
||||
mod windows {
|
||||
use super::*;
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn requests_and_preflights_allow_credentials() {
|
||||
assert_requests_and_preflights_allow_credentials(WINDOWS_ORIGIN).await;
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn production_rejects_origin_variants() {
|
||||
let cors = build_cors_config(ServerEnvironment::Production);
|
||||
for origin in [
|
||||
"https://tauri.localhost",
|
||||
"http://tauri.localhost:8080",
|
||||
"http://tauri.localhost.example.com",
|
||||
"http://sub.tauri.localhost",
|
||||
] {
|
||||
assert_origin_invalid(&cors, origin).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod mac {
|
||||
use super::*;
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn requests_and_preflights_allow_credentials() {
|
||||
assert_requests_and_preflights_allow_credentials(MAC_ORIGIN).await;
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn production_rejects_origin_variants() {
|
||||
let cors = build_cors_config(ServerEnvironment::Production);
|
||||
for origin in [
|
||||
"http://localhost",
|
||||
"https://localhost",
|
||||
"tauri://localhost:8080",
|
||||
"tauri://localhost.example.com",
|
||||
"tauri://sub.localhost",
|
||||
] {
|
||||
assert_origin_invalid(&cors, origin).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_requests_and_preflights_allow_credentials(origin: &str) {
|
||||
for environment in [ServerEnvironment::Production, ServerEnvironment::Development] {
|
||||
let service = build_cors_config(environment)
|
||||
.new_transform(test::ok_service())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for method in [Method::GET, Method::POST, Method::OPTIONS] {
|
||||
let mut request = TestRequest::default()
|
||||
.uri("/v1/session")
|
||||
.insert_header((header::ORIGIN, origin));
|
||||
if method == Method::OPTIONS {
|
||||
request = request
|
||||
.insert_header((header::ACCESS_CONTROL_REQUEST_METHOD, "POST"))
|
||||
.insert_header((header::ACCESS_CONTROL_REQUEST_HEADERS, "content-type,session,x-artcraft-version"));
|
||||
}
|
||||
let response = test::call_service(&service, request.method(method).to_srv_request()).await;
|
||||
assert_eq!(response.status(), StatusCode::OK, "Origin: {}", origin);
|
||||
assert_eq!(response.headers().get(header::ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(), origin);
|
||||
assert_eq!(response.headers().get(header::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use log::info;
|
||||
use crate::configs::artcraft_2d::add_artcraft_2d;
|
||||
use crate::configs::artcraft_3d::add_artcraft_3d;
|
||||
use crate::configs::artcraft_admin_dashboard::add_artcraft_admin_dashboard;
|
||||
use crate::configs::artcraft_desktop::add_artcraft_desktop;
|
||||
use crate::configs::artcraft_webapp::add_artcraft_webapp;
|
||||
use crate::configs::artcraft_website::add_artcraft_website;
|
||||
use crate::configs::development_only::add_development_only;
|
||||
@@ -48,6 +49,7 @@ fn do_build_cors_config(is_production: bool) -> Cors {
|
||||
|
||||
// Artcraft
|
||||
cors = add_tauri(cors, is_production);
|
||||
cors = add_artcraft_desktop(cors, is_production);
|
||||
cors = add_artcraft_2d(cors, is_production);
|
||||
cors = add_artcraft_3d(cors, is_production);
|
||||
cors = add_artcraft_webapp(cors, is_production);
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use anyhow::bail;
|
||||
use log::{error, info};
|
||||
use std::io::Write;
|
||||
use log::info;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
use crate::ffmpeg::run_thumbnail_ffmpeg::run_thumbnail_ffmpeg;
|
||||
|
||||
pub struct FfmpegVideoFirstFrameToJpgThumbnailArgs<I: AsRef<Path>, O: AsRef<Path>> {
|
||||
pub input_video_path: I,
|
||||
pub output_jpg_path: O,
|
||||
@@ -24,26 +24,20 @@ pub fn ffmpeg_video_first_frame_to_jpg_thumbnail<I: AsRef<Path>, O: AsRef<Path>>
|
||||
command
|
||||
.arg("-nostdin")
|
||||
.arg("-y")
|
||||
.arg("-loglevel").arg("error")
|
||||
.arg("-nostats")
|
||||
.arg("-threads").arg("1")
|
||||
.arg("-filter_threads").arg("1")
|
||||
.arg("-sseof").arg("-1")
|
||||
.arg("-i").arg(args.input_video_path.as_ref())
|
||||
.arg("-threads").arg("1")
|
||||
.arg("-vf").arg("scale=320:320:force_original_aspect_ratio=decrease")
|
||||
.arg("-vframes").arg("1")
|
||||
.arg(args.output_jpg_path.as_ref());
|
||||
|
||||
info!("Calling ffmpeg (jpg thumbnail)...");
|
||||
|
||||
let output = command.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
error!("bad exit status: {}", output.status);
|
||||
|
||||
let _r = std::io::stdout().write_all(&output.stdout);
|
||||
let _r = std::io::stderr().write_all(&output.stderr);
|
||||
|
||||
bail!("ffmpeg jpg thumbnail failed: {:?}", output.status.to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
run_thumbnail_ffmpeg(command)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use anyhow::bail;
|
||||
use log::{error, info};
|
||||
use std::io::Write;
|
||||
use log::info;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
use crate::ffmpeg::run_thumbnail_ffmpeg::run_thumbnail_ffmpeg;
|
||||
|
||||
pub struct FfmpegVideoGifPreviewArgs<I: AsRef<Path>, O: AsRef<Path>> {
|
||||
pub input_video_path: I,
|
||||
pub output_gif_path: O,
|
||||
@@ -13,7 +13,7 @@ pub struct FfmpegVideoGifPreviewArgs<I: AsRef<Path>, O: AsRef<Path>> {
|
||||
|
||||
/// Extract a short animated GIF preview from a video file.
|
||||
///
|
||||
/// Takes the first 5 seconds, resampled to 10 fps, scaled to 360px wide
|
||||
/// Takes the first 5 seconds, resampled to 10 fps, scaled to fit within 360x360
|
||||
/// (preserving aspect ratio), with an optimized color palette.
|
||||
pub fn ffmpeg_video_gif_preview<I: AsRef<Path>, O: AsRef<Path>>(
|
||||
args: FfmpegVideoGifPreviewArgs<I, O>,
|
||||
@@ -23,27 +23,22 @@ pub fn ffmpeg_video_gif_preview<I: AsRef<Path>, O: AsRef<Path>>(
|
||||
command
|
||||
.arg("-nostdin")
|
||||
.arg("-y")
|
||||
.arg("-loglevel").arg("error")
|
||||
.arg("-nostats")
|
||||
.arg("-threads").arg("1")
|
||||
.arg("-filter_complex_threads").arg("1")
|
||||
.arg("-ss").arg("0")
|
||||
.arg("-to").arg("5")
|
||||
.arg("-i").arg(args.input_video_path.as_ref())
|
||||
.arg("-threads").arg("1")
|
||||
.arg("-filter_complex")
|
||||
.arg("fps=10,scale=360:-1[s]; [s]split[a][b]; [a]palettegen[palette]; [b][palette]paletteuse")
|
||||
.arg("fps=10,scale=360:360:force_original_aspect_ratio=decrease[s]; [s]split[a][b]; [a]palettegen[palette]; [b][palette]paletteuse")
|
||||
.arg("-frames:v").arg("50")
|
||||
.arg(args.output_gif_path.as_ref());
|
||||
|
||||
info!("Calling ffmpeg (gif preview)...");
|
||||
|
||||
let output = command.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
error!("bad exit status: {}", output.status);
|
||||
|
||||
let _r = std::io::stdout().write_all(&output.stdout);
|
||||
let _r = std::io::stderr().write_all(&output.stderr);
|
||||
|
||||
bail!("ffmpeg gif preview failed: {:?}", output.status.to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
run_thumbnail_ffmpeg(command)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -69,7 +64,7 @@ mod tests {
|
||||
|
||||
assert!(output_path.exists(), "output gif should exist");
|
||||
|
||||
// The input video is 640x480. scale=360:-1 should produce 360x270
|
||||
// The input video is 640x480. Fitting within 360x360 should produce 360x270
|
||||
// (maintaining 4:3 aspect ratio).
|
||||
// Use ffprobe to verify the GIF dimensions.
|
||||
let result = ffprobe::ffprobe(&output_path)
|
||||
@@ -82,4 +77,30 @@ mod tests {
|
||||
assert_eq!(video_stream.width, Some(360));
|
||||
assert_eq!(video_stream.height, Some(270));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tall_video_preview_has_bounded_dimensions_and_frame_count() {
|
||||
let temp_dir = TempDir::new_in("/tmp", "ffmpeg_tall_gif_test").unwrap();
|
||||
let input = temp_dir.path().join("tall.mp4");
|
||||
let output = temp_dir.path().join("preview.gif");
|
||||
let status = Command::new("ffmpeg")
|
||||
.args(["-nostdin", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=32x640:rate=10", "-t", "6", "-threads", "1"])
|
||||
.arg(&input)
|
||||
.status().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
ffmpeg_video_gif_preview(FfmpegVideoGifPreviewArgs {
|
||||
input_video_path: &input,
|
||||
output_gif_path: &output,
|
||||
}).unwrap();
|
||||
let probe = Command::new("ffprobe")
|
||||
.args(["-v", "error", "-count_frames", "-select_streams", "v:0", "-show_entries", "stream=width,height,nb_read_frames", "-of", "csv=p=0"])
|
||||
.arg(&output)
|
||||
.output().unwrap();
|
||||
assert!(probe.status.success());
|
||||
let values: Vec<u32> = std::str::from_utf8(&probe.stdout).unwrap().trim()
|
||||
.split(',').map(|value| value.parse().unwrap()).collect();
|
||||
assert_eq!(&values[..2], &[18, 360]);
|
||||
assert!(values[2] > 0 && values[2] <= 50);
|
||||
}
|
||||
}
|
||||
@@ -3,3 +3,4 @@ pub mod ffmpeg_transcode_to_mp4;
|
||||
pub mod ffmpeg_trim_and_resample;
|
||||
pub mod ffmpeg_video_first_frame_to_jpg_thumbnail;
|
||||
pub mod ffmpeg_video_gif_preview;
|
||||
mod run_thumbnail_ffmpeg;
|
||||
@@ -0,0 +1,75 @@
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::bail;
|
||||
use log::error;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
const THUMBNAIL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const MAX_ERROR_LOG_BYTES: u64 = 16 * 1024;
|
||||
|
||||
/// Synchronous: callers in async runtimes must use spawn_blocking. A timeout
|
||||
/// kills and reaps FFmpeg before returning, so it cannot outlive its job slot.
|
||||
pub(super) fn run_thumbnail_ffmpeg(mut command: Command) -> AnyhowResult<()> {
|
||||
// Avoid holding arbitrarily large FFmpeg stderr output in the Rust heap.
|
||||
let mut stderr = tempfile::tempfile()?;
|
||||
let mut child = command.stdout(Stdio::null()).stderr(Stdio::from(stderr.try_clone()?)).spawn()?;
|
||||
let result = wait_for_exit(&mut child, THUMBNAIL_TIMEOUT);
|
||||
if !matches!(&result, Ok(status) if status.success()) {
|
||||
let start = stderr.metadata()?.len().saturating_sub(MAX_ERROR_LOG_BYTES);
|
||||
stderr.seek(SeekFrom::Start(start))?;
|
||||
let mut bytes = Vec::new();
|
||||
stderr.take(MAX_ERROR_LOG_BYTES).read_to_end(&mut bytes)?;
|
||||
error!("Thumbnail FFmpeg failed: {:?}; stderr: {}", result, String::from_utf8_lossy(&bytes));
|
||||
}
|
||||
let status = result?;
|
||||
if !status.success() {
|
||||
bail!("thumbnail FFmpeg failed: {status}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_for_exit(child: &mut Child, timeout: Duration) -> AnyhowResult<ExitStatus> {
|
||||
let started_at = Instant::now();
|
||||
loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => return Ok(status),
|
||||
Ok(None) => {},
|
||||
Err(err) => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
if started_at.elapsed() >= timeout {
|
||||
let _ = child.kill();
|
||||
child.wait()?;
|
||||
bail!("thumbnail FFmpeg exceeded timeout of {} seconds", timeout.as_secs_f64());
|
||||
}
|
||||
thread::sleep(Duration::from_millis(25));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn timed_out_child_is_killed_and_reaped_before_returning() {
|
||||
let mut child = Command::new("sleep").arg("10").spawn().unwrap();
|
||||
let started_at = Instant::now();
|
||||
let result = wait_for_exit(&mut child, Duration::from_millis(50));
|
||||
assert!(result.unwrap_err().to_string().contains("exceeded timeout"));
|
||||
assert!(started_at.elapsed() < Duration::from_secs(3));
|
||||
assert!(!child.try_wait().unwrap().unwrap().success());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_normal_child_exit_status() {
|
||||
let mut child = Command::new("true").spawn().unwrap();
|
||||
assert!(wait_for_exit(&mut child, Duration::from_secs(3)).unwrap().success());
|
||||
}
|
||||
}
|
||||
@@ -56,24 +56,72 @@ schema metadata. For **new tables**, you may need to:
|
||||
|
||||
- Primary key `id` columns must be `BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT`. Do not use signed IDs.
|
||||
- Always create tables with `ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`.
|
||||
- Do not give application-owned state/status/type columns schema defaults. Keep required
|
||||
columns `NOT NULL` without a `DEFAULT`; application code must explicitly bind their
|
||||
initial value on every insert. Keep migrations and materialized schema docs in sync.
|
||||
|
||||
### Database enum values
|
||||
|
||||
- Use the Rust enums in `enums::by_table` for database state, status, failure, and type
|
||||
values. Do not hardcode their string representations in SQL or Rust business logic.
|
||||
- Bind enum values (or their `to_str()` representation) through SQL parameters, including
|
||||
`SET`, `WHERE`, `IN`, and `INSERT` values. For example, bind
|
||||
`UserLoginChallengeStatus::Approved.to_str()` to `status = ?`.
|
||||
- Query args and returned rows must use the enum type rather than `String`/`&str` for
|
||||
enum columns. Use SQLx column type overrides and the enum's MySQL decoder on reads;
|
||||
use exhaustive variant matches in application logic. Preserve existing stored values.
|
||||
|
||||
### Error Types
|
||||
|
||||
New queries should return `Result<T, sqlx::Error>` instead of the legacy `AnyhowResult<T>`.
|
||||
The `AnyhowResult` pattern is being phased out.
|
||||
|
||||
### Transaction Support
|
||||
### Named arguments and generic transactors
|
||||
|
||||
Use the executor-generic `Args` pattern for queries that need to work inside transactions:
|
||||
Every new or refactored query must take a single named `<QueryName>Args` struct containing
|
||||
all inputs, including `mysql_executor`. Do not use positional parameter lists, even for
|
||||
small queries. Keep the args struct in the same file as its query.
|
||||
|
||||
Use a generic transactor `T` bounded by `Executor<'c, Database = MySql>`. Do not constrain
|
||||
query functions to `MySqlConnection`, `MySqlPool`, a concrete transaction, or the legacy
|
||||
`Transactor` enum. This applies to all queries, not only ones currently used in transactions.
|
||||
Put the executor bound on the function; the args struct need not carry executor lifetimes
|
||||
or `PhantomData` when it simply stores `T`.
|
||||
|
||||
```rust
|
||||
pub struct MyQueryArgs<'a, 'c: 'a, E>
|
||||
where E: 'a + Executor<'c, Database = MySql>
|
||||
{
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct MyQueryArgs<'a, T> {
|
||||
pub field: &'a str,
|
||||
pub mysql_executor: E,
|
||||
pub phantom: PhantomData<&'c E>,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
pub async fn my_query<'c, T>(args: MyQueryArgs<'_, T>) -> Result<(), sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query!("UPDATE my_table SET name = ?", args.field)
|
||||
.execute(args.mysql_executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
This allows the same query function to accept both `&MySqlPool` and `&mut Transaction`.
|
||||
Call with `mysql_executor: &pool` for a pool, or `mysql_executor: &mut *tx` /
|
||||
`mysql_executor: &mut *conn` to reborrow the connection held by an owned SQLx transaction /
|
||||
pool connection. SQLx transactions do not themselves implement `Executor`; dereference
|
||||
them at the call site. Reuse the caller's transaction when atomicity or row locks matter.
|
||||
Do not acquire a new connection or begin/commit a transaction inside a query helper.
|
||||
|
||||
### One query per file, grouped by table
|
||||
|
||||
- Put each query in its own descriptively named `.rs` file under the table's directory
|
||||
module, e.g. `queries/user_login_challenges/decide_challenge.rs` or the existing
|
||||
`queries/users/user_sessions/find_redeemed_session.rs`.
|
||||
- Each query file contains one SQL statement/query function, its args struct, and any
|
||||
result type used only by that query. Put shared result types in a separate type file.
|
||||
- Keep `mod.rs` for module declarations; do not collect SQL in it or in catch-all files
|
||||
such as `challenge_queries.rs` or `session_queries.rs`.
|
||||
- Split multi-query operations into individual query files. Keep transaction sequencing
|
||||
and business decisions in the caller/service layer, with every statement using the
|
||||
same transaction when required.
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
use sqlx::{Executor, MySql, QueryBuilder};
|
||||
|
||||
use super::user_media_file_filters::UserMediaFileFilters;
|
||||
|
||||
pub struct CountMediaFilesForUserArgs<'a, T> {
|
||||
pub filters: UserMediaFileFilters<'a>,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Legacy numbered pagination needs an exact total. Count only this user's rows,
|
||||
/// without related-table lookups or global low-selectivity index intersections.
|
||||
pub async fn count_media_files_for_user<'c, T>(args: CountMediaFilesForUserArgs<'_, T>) -> Result<i64, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let mut query =
|
||||
QueryBuilder::<MySql>::new("SELECT COUNT(*) FROM media_files m FORCE INDEX (fk_maybe_creator_user_token)");
|
||||
args.filters.push_predicates(&mut query);
|
||||
query.build_query_scalar().fetch_one(args.mysql_executor).await
|
||||
}
|
||||
+49
-438
@@ -1,191 +1,10 @@
|
||||
use std::collections::HashSet;
|
||||
use sqlx::{Executor, MySql, QueryBuilder};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
|
||||
use sqlx::mysql::MySqlRow;
|
||||
use enums::by_table::entity_stats::stats_entity_type::StatsEntityType;
|
||||
use super::user_media_file_filters::UserMediaFileFilters;
|
||||
use super::user_media_file_list_item::{MediaFileListItem, MediaFileListItemInternal};
|
||||
|
||||
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
|
||||
use enums::traits::mysql_from_row::MySqlFromRow as _;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use enums::common::view_as::ViewAs;
|
||||
use enums::common::visibility::Visibility;
|
||||
use errors::AnyhowResult;
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
use tokens::tokens::prompts::PromptToken;
|
||||
use tokens::traits::mysql_token_from_row::MySqlTokenFromRow;
|
||||
use crate::helpers::boolean_converters::i8_to_bool;
|
||||
use crate::payloads::prompt_args::prompt_inner_payload::PromptInnerPayload;
|
||||
|
||||
pub struct MediaFileListPage {
|
||||
pub records: Vec<MediaFileListItem>,
|
||||
|
||||
pub sort_ascending: bool,
|
||||
|
||||
pub current_page: usize,
|
||||
pub total_page_count: usize,
|
||||
}
|
||||
|
||||
pub struct MediaFileListItem {
|
||||
pub token: MediaFileToken,
|
||||
|
||||
pub media_class: MediaFileClass,
|
||||
pub media_type: MediaFileType,
|
||||
|
||||
pub maybe_engine_category: Option<MediaFileEngineCategory>,
|
||||
pub maybe_animation_type: Option<MediaFileAnimationType>,
|
||||
|
||||
pub origin_category: MediaFileOriginCategory,
|
||||
pub origin_product_category: MediaFileOriginProductCategory,
|
||||
|
||||
pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
pub maybe_origin_model_token: Option<String>,
|
||||
|
||||
// NB: The title won't be populated for `tts_models` records or non-`model_weights` records.
|
||||
pub maybe_origin_model_title: Option<String>,
|
||||
|
||||
pub public_bucket_directory_hash: String,
|
||||
pub maybe_public_bucket_prefix: Option<String>,
|
||||
pub maybe_public_bucket_extension: Option<String>,
|
||||
|
||||
pub maybe_prompt_token: Option<PromptToken>,
|
||||
|
||||
pub creator_set_visibility: Visibility,
|
||||
|
||||
pub is_user_upload: bool,
|
||||
pub is_intermediate_system_file: bool,
|
||||
|
||||
pub maybe_file_cover_image_public_bucket_hash: Option<String>,
|
||||
pub maybe_file_cover_image_public_bucket_prefix: Option<String>,
|
||||
pub maybe_file_cover_image_public_bucket_extension: Option<String>,
|
||||
|
||||
pub maybe_title: Option<String>,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
pub maybe_text_transcript: Option<String>,
|
||||
pub maybe_prompt_args: Option<PromptInnerPayload>,
|
||||
|
||||
pub maybe_duration_millis: Option<u64>,
|
||||
|
||||
pub maybe_ratings_positive_count: Option<u32>,
|
||||
pub maybe_ratings_negative_count: Option<u32>,
|
||||
pub maybe_bookmark_count: Option<u32>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct ListMediaFileForUserArgs<'a> {
|
||||
pub username: &'a str,
|
||||
pub maybe_filter_media_types: Option<&'a HashSet<MediaFileType>>,
|
||||
pub maybe_filter_media_classes: Option<&'a HashSet<MediaFileClass>>,
|
||||
pub maybe_filter_engine_categories: Option<&'a HashSet<MediaFileEngineCategory>>,
|
||||
pub include_user_uploads: bool,
|
||||
pub page_size: usize,
|
||||
pub page_index: usize,
|
||||
pub sort_ascending: bool,
|
||||
pub view_as: ViewAs,
|
||||
pub mysql_pool: &'a MySqlPool,
|
||||
}
|
||||
|
||||
pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult<MediaFileListPage> {
|
||||
/// Let's figure out how many results we could have returned total
|
||||
let count_fields = select_total_count_field();
|
||||
let mut count_query_builder = query_builder(
|
||||
args.maybe_filter_media_types,
|
||||
args.maybe_filter_media_classes,
|
||||
args.maybe_filter_engine_categories,
|
||||
args.include_user_uploads,
|
||||
args.username,
|
||||
false,
|
||||
0,
|
||||
0,
|
||||
args.sort_ascending,
|
||||
args.view_as,
|
||||
count_fields.as_str(),
|
||||
true,
|
||||
);
|
||||
|
||||
let row_count_query = count_query_builder.build_query_scalar::<i64>();
|
||||
let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
|
||||
|
||||
/// Now fetch the actual results with all the fields
|
||||
let result_fields = select_result_fields();
|
||||
let mut query = query_builder(
|
||||
args.maybe_filter_media_types,
|
||||
args.maybe_filter_media_classes,
|
||||
args.maybe_filter_engine_categories,
|
||||
args.include_user_uploads,
|
||||
args.username,
|
||||
true,
|
||||
args.page_index,
|
||||
args.page_size,
|
||||
args.sort_ascending,
|
||||
args.view_as,
|
||||
result_fields.as_str(),
|
||||
false,
|
||||
);
|
||||
|
||||
let query = query.build_query_as::<MediaFileListItemInternal>();
|
||||
let results = query.fetch_all(args.mysql_pool).await?;
|
||||
|
||||
let number_of_pages = 1 + (row_count_result / args.page_size as i64) as usize;
|
||||
|
||||
let results = results.into_iter()
|
||||
.map(|record| {
|
||||
MediaFileListItem {
|
||||
token: record.token,
|
||||
media_class: record.media_class,
|
||||
media_type: record.media_type,
|
||||
maybe_engine_category: record.maybe_engine_category,
|
||||
maybe_animation_type: record.maybe_animation_type,
|
||||
origin_category: record.origin_category,
|
||||
origin_product_category: record.origin_product_category,
|
||||
maybe_origin_model_type: record.maybe_origin_model_type,
|
||||
maybe_origin_model_token: record.maybe_origin_model_token,
|
||||
maybe_origin_model_title: record.maybe_origin_model_title,
|
||||
public_bucket_directory_hash: record.public_bucket_directory_hash,
|
||||
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
|
||||
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
|
||||
maybe_prompt_token: record.maybe_prompt_token,
|
||||
creator_set_visibility: record.creator_set_visibility,
|
||||
is_user_upload: i8_to_bool(record.is_user_upload),
|
||||
is_intermediate_system_file: i8_to_bool(record.is_intermediate_system_file),
|
||||
maybe_file_cover_image_public_bucket_hash: record.maybe_file_cover_image_public_bucket_hash,
|
||||
maybe_file_cover_image_public_bucket_prefix: record.maybe_file_cover_image_public_bucket_prefix,
|
||||
maybe_file_cover_image_public_bucket_extension: record.maybe_file_cover_image_public_bucket_extension,
|
||||
maybe_title: record.maybe_title,
|
||||
maybe_text_transcript: record.maybe_text_transcript,
|
||||
maybe_prompt_args: record.maybe_other_prompt_args
|
||||
.as_deref()
|
||||
.map(|args| PromptInnerPayload::from_json(args))
|
||||
.transpose()
|
||||
.ok() // NB: Fail open
|
||||
.flatten(),
|
||||
maybe_duration_millis: record.maybe_duration_millis.map(|d| d as u64),
|
||||
maybe_ratings_positive_count: record.maybe_ratings_positive_count,
|
||||
maybe_ratings_negative_count: record.maybe_ratings_negative_count,
|
||||
maybe_bookmark_count: record.maybe_bookmark_count,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Ok(MediaFileListPage {
|
||||
records: results,
|
||||
sort_ascending: args.sort_ascending,
|
||||
current_page: args.page_index,
|
||||
total_page_count: number_of_pages,
|
||||
})
|
||||
}
|
||||
|
||||
fn select_result_fields() -> String {
|
||||
r#"
|
||||
const RESULT_FIELDS: &str = r#"
|
||||
m.id,
|
||||
m.token,
|
||||
|
||||
@@ -229,258 +48,50 @@ fn select_result_fields() -> String {
|
||||
|
||||
m.created_at,
|
||||
m.updated_at
|
||||
"#
|
||||
.to_string()
|
||||
"#;
|
||||
|
||||
pub struct ListMediaFileForUserArgs<'a, T> {
|
||||
pub filters: UserMediaFileFilters<'a>,
|
||||
pub limit: usize,
|
||||
pub offset: usize,
|
||||
pub sort_ascending: bool,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
fn select_total_count_field() -> String {
|
||||
r#"
|
||||
COUNT(m.id) AS total_count
|
||||
"#
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn query_builder<'a>(
|
||||
maybe_filter_media_types: Option<&HashSet<MediaFileType>>,
|
||||
maybe_filter_media_classes: Option<&HashSet<MediaFileClass>>,
|
||||
maybe_filter_engine_categories: Option<&HashSet<MediaFileEngineCategory>>,
|
||||
include_user_uploads: bool,
|
||||
username: &'a str,
|
||||
enforce_limits: bool,
|
||||
page_index: usize,
|
||||
page_size: usize,
|
||||
sort_ascending: bool,
|
||||
view_as: ViewAs,
|
||||
select_fields: &'a str,
|
||||
count_only: bool,
|
||||
) -> QueryBuilder<'a, MySql> {
|
||||
|
||||
// NB: The count query only needs the `users` join (for the username predicate).
|
||||
// The other joins are all 1:1 and contribute nothing to the count — MySQL does
|
||||
// not eliminate LEFT JOINs, so including them costs four index lookups per row.
|
||||
let from_clause = if count_only {
|
||||
r#"
|
||||
FROM media_files AS m
|
||||
LEFT OUTER JOIN users AS u
|
||||
ON m.maybe_creator_user_token = u.token
|
||||
"#
|
||||
} else {
|
||||
r#"
|
||||
FROM media_files AS m
|
||||
LEFT OUTER JOIN users AS u
|
||||
ON m.maybe_creator_user_token = u.token
|
||||
LEFT OUTER JOIN model_weights as w
|
||||
ON m.maybe_origin_model_token = w.token
|
||||
LEFT OUTER JOIN media_files as media_file_cover_image
|
||||
ON media_file_cover_image.token = m.maybe_cover_image_media_file_token
|
||||
LEFT OUTER JOIN entity_stats
|
||||
ON entity_stats.entity_type = "media_file"
|
||||
AND entity_stats.entity_token = m.token
|
||||
LEFT OUTER JOIN prompts
|
||||
ON prompts.token = m.maybe_prompt_token
|
||||
"#
|
||||
};
|
||||
|
||||
// NB: Query cannot be statically checked by sqlx
|
||||
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
|
||||
format!(r#"
|
||||
SELECT
|
||||
{select_fields}
|
||||
{from_clause}"#
|
||||
));
|
||||
|
||||
query_builder.push(" WHERE u.username = ");
|
||||
query_builder.push_bind(username);
|
||||
|
||||
//if let Some(media_type) = maybe_filter_media_type {
|
||||
// // FIXME: Binding shouldn't require to_str().
|
||||
// // Otherwise, it's calling the Display trait on the raw type which is resulting in an
|
||||
// // incorrect binding and runtime error.
|
||||
// query_builder.push(" AND m.media_type = ");
|
||||
// query_builder.push_bind(media_type.to_str());
|
||||
//}
|
||||
|
||||
if let Some(media_types) = maybe_filter_media_types {
|
||||
// NB: `WHERE IN` comma separated syntax will be wrong if list has zero length
|
||||
// We'll skip the predicate if the list isn't empty.
|
||||
if !media_types.is_empty() {
|
||||
query_builder.push(" AND m.media_type IN ( ");
|
||||
|
||||
let mut separated = query_builder.separated(", ");
|
||||
|
||||
for media_type in media_types.iter() {
|
||||
separated.push_bind(media_type.to_str());
|
||||
}
|
||||
|
||||
separated.push_unseparated(") ");
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(media_classes) = maybe_filter_media_classes {
|
||||
// NB: `WHERE IN` comma separated syntax will be wrong if list has zero length
|
||||
// We'll skip the predicate if the list isn't empty.
|
||||
if !media_classes.is_empty() {
|
||||
query_builder.push(" AND m.media_class IN ( ");
|
||||
|
||||
let mut separated = query_builder.separated(", ");
|
||||
|
||||
for media_class in media_classes.iter() {
|
||||
separated.push_bind(media_class.to_str());
|
||||
}
|
||||
|
||||
separated.push_unseparated(") ");
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(engine_categories) = maybe_filter_engine_categories {
|
||||
// NB: `WHERE IN` comma separated syntax will be wrong if list has zero length
|
||||
// We'll skip the predicate if the list isn't empty.
|
||||
if !engine_categories.is_empty() {
|
||||
query_builder.push(" AND m.maybe_engine_category IN ( ");
|
||||
|
||||
let mut separated = query_builder.separated(", ");
|
||||
|
||||
for engine_category in engine_categories.iter() {
|
||||
separated.push_bind(engine_category.to_str());
|
||||
}
|
||||
|
||||
separated.push_unseparated(") ");
|
||||
}
|
||||
}
|
||||
|
||||
if !include_user_uploads {
|
||||
query_builder.push(" AND NOT m.is_user_upload ");
|
||||
}
|
||||
|
||||
// TODO: Argument to control
|
||||
query_builder.push(" AND NOT m.is_intermediate_system_file ");
|
||||
|
||||
match view_as {
|
||||
ViewAs::Author | ViewAs::Moderator => {
|
||||
// NB(bt): Actually, mods don't want to see deleted files. We'll improve the moderator UI later.
|
||||
query_builder.push(" AND m.user_deleted_at IS NULL AND m.mod_deleted_at IS NULL ");
|
||||
}
|
||||
ViewAs::AnotherUser => {
|
||||
query_builder.push(" AND m.user_deleted_at IS NULL AND m.mod_deleted_at IS NULL ");
|
||||
// FIXME: Binding shouldn't require to_str().
|
||||
// Otherwise, it's calling the Display trait on the raw type which is resulting in an
|
||||
// incorrect binding and runtime error.
|
||||
query_builder.push(" AND m.creator_set_visibility = ");
|
||||
query_builder.push_bind(Visibility::Public.to_str());
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Ordering is meaningless for the single-row COUNT aggregate.
|
||||
if !count_only {
|
||||
if sort_ascending {
|
||||
query_builder.push(" ORDER BY m.created_at ASC ");
|
||||
} else {
|
||||
query_builder.push(" ORDER BY m.created_at DESC ");
|
||||
}
|
||||
}
|
||||
|
||||
if enforce_limits {
|
||||
let offset = page_index * page_size;
|
||||
query_builder.push(format!(" LIMIT {page_size} OFFSET {offset} "));
|
||||
}
|
||||
|
||||
query_builder
|
||||
}
|
||||
|
||||
struct MediaFileListItemInternal {
|
||||
id: i64,
|
||||
token: MediaFileToken,
|
||||
|
||||
media_class: MediaFileClass,
|
||||
media_type: MediaFileType,
|
||||
|
||||
maybe_engine_category: Option<MediaFileEngineCategory>,
|
||||
maybe_animation_type: Option<MediaFileAnimationType>,
|
||||
|
||||
origin_category: MediaFileOriginCategory,
|
||||
origin_product_category: MediaFileOriginProductCategory,
|
||||
|
||||
maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
maybe_origin_model_token: Option<String>,
|
||||
|
||||
// NB: The title won't be populated for `tts_models` records or non-`model_weights` records.
|
||||
maybe_origin_model_title: Option<String>,
|
||||
maybe_other_prompt_args: Option<String>,
|
||||
maybe_duration_millis: Option<i32>,
|
||||
|
||||
public_bucket_directory_hash: String,
|
||||
maybe_public_bucket_prefix: Option<String>,
|
||||
maybe_public_bucket_extension: Option<String>,
|
||||
|
||||
maybe_prompt_token: Option<PromptToken>,
|
||||
|
||||
creator_set_visibility: Visibility,
|
||||
|
||||
is_user_upload: i8,
|
||||
is_intermediate_system_file: i8,
|
||||
|
||||
maybe_file_cover_image_public_bucket_hash: Option<String>,
|
||||
maybe_file_cover_image_public_bucket_prefix: Option<String>,
|
||||
maybe_file_cover_image_public_bucket_extension: Option<String>,
|
||||
|
||||
maybe_title: Option<String>,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
maybe_text_transcript: Option<String>,
|
||||
|
||||
maybe_ratings_positive_count: Option<u32>,
|
||||
maybe_ratings_negative_count: Option<u32>,
|
||||
maybe_bookmark_count: Option<u32>,
|
||||
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
|
||||
// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually
|
||||
// implemented. Perhaps this will improve in the future, but for now manually constructed queries
|
||||
// cannot have type hints, eg. the following:
|
||||
//
|
||||
// m.token as `token: tokens::tokens::media_files::MediaFileToken`,
|
||||
// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,
|
||||
// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
|
||||
//
|
||||
// This results in the automatic mapping not being able to be found by name (for macro derive), and
|
||||
// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the
|
||||
// full "as" clause).
|
||||
impl FromRow<'_, MySqlRow> for MediaFileListItemInternal {
|
||||
fn from_row(row: &MySqlRow) -> Result<Self, sqlx::Error> {
|
||||
Ok(Self {
|
||||
id: row.try_get("id")?,
|
||||
token: MediaFileToken::new(row.try_get("token")?),
|
||||
media_class: MediaFileClass::try_from_mysql_row(row, "media_class")?,
|
||||
media_type: MediaFileType::try_from_mysql_row(row, "media_type")?,
|
||||
maybe_engine_category: MediaFileEngineCategory::try_from_mysql_row_nullable(row, "maybe_engine_category")?,
|
||||
maybe_animation_type: MediaFileAnimationType::try_from_mysql_row_nullable(row, "maybe_animation_type")?,
|
||||
origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?,
|
||||
origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?,
|
||||
maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?,
|
||||
maybe_origin_model_token: row.try_get("maybe_origin_model_token")?,
|
||||
maybe_origin_model_title: row.try_get("maybe_origin_model_title")?,
|
||||
maybe_other_prompt_args: row.try_get("maybe_other_prompt_args")?,
|
||||
maybe_duration_millis: row.try_get("maybe_duration_millis")?,
|
||||
public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?,
|
||||
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
|
||||
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
|
||||
maybe_prompt_token: PromptToken::try_from_mysql_row_nullable(row, "maybe_prompt_token")?,
|
||||
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
|
||||
is_user_upload: row.try_get("is_user_upload")?,
|
||||
is_intermediate_system_file: row.try_get("is_intermediate_system_file")?,
|
||||
maybe_file_cover_image_public_bucket_hash: row.try_get("maybe_file_cover_image_public_bucket_hash")?,
|
||||
maybe_file_cover_image_public_bucket_prefix: row.try_get("maybe_file_cover_image_public_bucket_prefix")?,
|
||||
maybe_file_cover_image_public_bucket_extension: row.try_get("maybe_file_cover_image_public_bucket_extension")?,
|
||||
maybe_title: row.try_get("maybe_title")?,
|
||||
maybe_text_transcript: row.try_get("maybe_text_transcript")?,
|
||||
maybe_ratings_positive_count: row.try_get("maybe_ratings_positive_count")?,
|
||||
maybe_ratings_negative_count: row.try_get("maybe_ratings_negative_count")?,
|
||||
maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
|
||||
created_at: row.try_get("created_at")?,
|
||||
updated_at: row.try_get("updated_at")?,
|
||||
})
|
||||
}
|
||||
/// Select a bounded page before joining potentially large prompt/metadata rows.
|
||||
/// `LIMIT` prevents MySQL from merging the derived table into the outer query.
|
||||
pub async fn list_media_files_for_user<'c, T>(
|
||||
args: ListMediaFileForUserArgs<'_, T>,
|
||||
) -> Result<Vec<MediaFileListItem>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let order = if args.sort_ascending { " ASC" } else { " DESC" };
|
||||
let mut query = QueryBuilder::<MySql>::new(format!("SELECT {RESULT_FIELDS} FROM (SELECT m.id "));
|
||||
// This index is in the July 2026 migrations. Restricting the scan to this user
|
||||
// avoids index-merge scans of the global deletion/class indexes on large tables.
|
||||
query.push("FROM media_files m FORCE INDEX (idx_creator_created_at) ");
|
||||
args.filters.push_predicates(&mut query);
|
||||
query.push(" ORDER BY m.created_at").push(order).push(", m.id").push(order);
|
||||
query.push(" LIMIT ").push_bind(args.limit as u64);
|
||||
query.push(" OFFSET ").push_bind(args.offset as u64);
|
||||
query.push(
|
||||
r#") AS page
|
||||
JOIN media_files m ON m.id = page.id
|
||||
LEFT JOIN model_weights w ON m.maybe_origin_model_token = w.token
|
||||
LEFT JOIN media_files media_file_cover_image ON media_file_cover_image.token = m.maybe_cover_image_media_file_token
|
||||
LEFT JOIN entity_stats ON entity_stats.entity_type = "#,
|
||||
);
|
||||
query.push_bind(StatsEntityType::MediaFile.to_str());
|
||||
query
|
||||
.push(
|
||||
r#" AND entity_stats.entity_token = m.token
|
||||
LEFT JOIN prompts ON prompts.token = m.maybe_prompt_token
|
||||
ORDER BY m.created_at"#,
|
||||
)
|
||||
.push(order)
|
||||
.push(", m.id")
|
||||
.push(order);
|
||||
let records = query.build_query_as::<MediaFileListItemInternal>().fetch_all(args.mysql_executor).await?;
|
||||
Ok(records.into_iter().map(MediaFileListItem::from).collect())
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod count_media_files_for_user;
|
||||
pub mod list_batch_generated_redux_media_files;
|
||||
pub mod list_featured_media_files;
|
||||
pub mod list_media_files;
|
||||
@@ -7,3 +8,5 @@ pub mod list_media_files_for_user;
|
||||
pub mod list_session_media_files_by_class;
|
||||
pub mod list_session_project_media_files;
|
||||
pub mod media_file_list_row;
|
||||
pub mod user_media_file_filters;
|
||||
pub mod user_media_file_list_item;
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use sqlx::{MySql, QueryBuilder};
|
||||
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use enums::common::view_as::ViewAs;
|
||||
use enums::common::visibility::Visibility;
|
||||
|
||||
/// Shared predicates keep the optional total and the returned page consistent.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct UserMediaFileFilters<'a> {
|
||||
pub username: &'a str,
|
||||
pub maybe_filter_media_types: Option<&'a HashSet<MediaFileType>>,
|
||||
pub maybe_filter_media_classes: Option<&'a HashSet<MediaFileClass>>,
|
||||
pub maybe_filter_engine_categories: Option<&'a HashSet<MediaFileEngineCategory>>,
|
||||
pub include_user_uploads: bool,
|
||||
pub view_as: ViewAs,
|
||||
}
|
||||
|
||||
impl<'a> UserMediaFileFilters<'a> {
|
||||
pub(super) fn push_predicates(&self, query: &mut QueryBuilder<'a, MySql>) {
|
||||
// Resolve the unique username once; subsequent scans are scoped to its token.
|
||||
query.push(" WHERE m.maybe_creator_user_token = (SELECT token FROM users WHERE username = ");
|
||||
query.push_bind(self.username).push(")");
|
||||
if let Some(values) = self.maybe_filter_media_types.filter(|values| !values.is_empty()) {
|
||||
query.push(" AND m.media_type IN (");
|
||||
let mut separated = query.separated(", ");
|
||||
for value in values {
|
||||
separated.push_bind(value.to_str());
|
||||
}
|
||||
separated.push_unseparated(")");
|
||||
}
|
||||
if let Some(values) = self.maybe_filter_media_classes.filter(|values| !values.is_empty()) {
|
||||
query.push(" AND m.media_class IN (");
|
||||
let mut separated = query.separated(", ");
|
||||
for value in values {
|
||||
separated.push_bind(value.to_str());
|
||||
}
|
||||
separated.push_unseparated(")");
|
||||
}
|
||||
if let Some(values) = self.maybe_filter_engine_categories.filter(|values| !values.is_empty()) {
|
||||
query.push(" AND m.maybe_engine_category IN (");
|
||||
let mut separated = query.separated(", ");
|
||||
for value in values {
|
||||
separated.push_bind(value.to_str());
|
||||
}
|
||||
separated.push_unseparated(")");
|
||||
}
|
||||
if !self.include_user_uploads {
|
||||
query.push(" AND NOT m.is_user_upload");
|
||||
}
|
||||
query.push(" AND NOT m.is_intermediate_system_file AND m.user_deleted_at IS NULL AND m.mod_deleted_at IS NULL");
|
||||
if self.view_as == ViewAs::AnotherUser {
|
||||
query.push(" AND m.creator_set_visibility = ").push_bind(Visibility::Public.to_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::{FromRow, Row};
|
||||
use sqlx::mysql::MySqlRow;
|
||||
|
||||
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
|
||||
use enums::traits::mysql_from_row::MySqlFromRow as _;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use enums::common::visibility::Visibility;
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
use tokens::tokens::prompts::PromptToken;
|
||||
use tokens::traits::mysql_token_from_row::MySqlTokenFromRow;
|
||||
use crate::helpers::boolean_converters::i8_to_bool;
|
||||
use crate::payloads::prompt_args::prompt_inner_payload::PromptInnerPayload;
|
||||
|
||||
pub struct MediaFileListItem {
|
||||
pub token: MediaFileToken,
|
||||
|
||||
pub media_class: MediaFileClass,
|
||||
pub media_type: MediaFileType,
|
||||
|
||||
pub maybe_engine_category: Option<MediaFileEngineCategory>,
|
||||
pub maybe_animation_type: Option<MediaFileAnimationType>,
|
||||
|
||||
pub origin_category: MediaFileOriginCategory,
|
||||
pub origin_product_category: MediaFileOriginProductCategory,
|
||||
|
||||
pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
pub maybe_origin_model_token: Option<String>,
|
||||
|
||||
// NB: The title won't be populated for `tts_models` records or non-`model_weights` records.
|
||||
pub maybe_origin_model_title: Option<String>,
|
||||
|
||||
pub public_bucket_directory_hash: String,
|
||||
pub maybe_public_bucket_prefix: Option<String>,
|
||||
pub maybe_public_bucket_extension: Option<String>,
|
||||
|
||||
pub maybe_prompt_token: Option<PromptToken>,
|
||||
|
||||
pub creator_set_visibility: Visibility,
|
||||
|
||||
pub is_user_upload: bool,
|
||||
pub is_intermediate_system_file: bool,
|
||||
|
||||
pub maybe_file_cover_image_public_bucket_hash: Option<String>,
|
||||
pub maybe_file_cover_image_public_bucket_prefix: Option<String>,
|
||||
pub maybe_file_cover_image_public_bucket_extension: Option<String>,
|
||||
|
||||
pub maybe_title: Option<String>,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
pub maybe_text_transcript: Option<String>,
|
||||
pub maybe_prompt_args: Option<PromptInnerPayload>,
|
||||
|
||||
pub maybe_duration_millis: Option<u64>,
|
||||
|
||||
pub maybe_ratings_positive_count: Option<u32>,
|
||||
pub maybe_ratings_negative_count: Option<u32>,
|
||||
pub maybe_bookmark_count: Option<u32>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub(super) struct MediaFileListItemInternal {
|
||||
id: i64,
|
||||
token: MediaFileToken,
|
||||
|
||||
media_class: MediaFileClass,
|
||||
media_type: MediaFileType,
|
||||
|
||||
maybe_engine_category: Option<MediaFileEngineCategory>,
|
||||
maybe_animation_type: Option<MediaFileAnimationType>,
|
||||
|
||||
origin_category: MediaFileOriginCategory,
|
||||
origin_product_category: MediaFileOriginProductCategory,
|
||||
|
||||
maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
maybe_origin_model_token: Option<String>,
|
||||
|
||||
// NB: The title won't be populated for `tts_models` records or non-`model_weights` records.
|
||||
maybe_origin_model_title: Option<String>,
|
||||
maybe_other_prompt_args: Option<String>,
|
||||
maybe_duration_millis: Option<i32>,
|
||||
|
||||
public_bucket_directory_hash: String,
|
||||
maybe_public_bucket_prefix: Option<String>,
|
||||
maybe_public_bucket_extension: Option<String>,
|
||||
|
||||
maybe_prompt_token: Option<PromptToken>,
|
||||
|
||||
creator_set_visibility: Visibility,
|
||||
|
||||
is_user_upload: i8,
|
||||
is_intermediate_system_file: i8,
|
||||
|
||||
maybe_file_cover_image_public_bucket_hash: Option<String>,
|
||||
maybe_file_cover_image_public_bucket_prefix: Option<String>,
|
||||
maybe_file_cover_image_public_bucket_extension: Option<String>,
|
||||
|
||||
maybe_title: Option<String>,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
maybe_text_transcript: Option<String>,
|
||||
|
||||
maybe_ratings_positive_count: Option<u32>,
|
||||
maybe_ratings_negative_count: Option<u32>,
|
||||
maybe_bookmark_count: Option<u32>,
|
||||
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
|
||||
// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually
|
||||
// implemented. Perhaps this will improve in the future, but for now manually constructed queries
|
||||
// cannot have type hints, eg. the following:
|
||||
//
|
||||
// m.token as `token: tokens::tokens::media_files::MediaFileToken`,
|
||||
// m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,
|
||||
// m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
|
||||
//
|
||||
// This results in the automatic mapping not being able to be found by name (for macro derive), and
|
||||
// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the
|
||||
// full "as" clause).
|
||||
impl FromRow<'_, MySqlRow> for MediaFileListItemInternal {
|
||||
fn from_row(row: &MySqlRow) -> Result<Self, sqlx::Error> {
|
||||
Ok(Self {
|
||||
id: row.try_get("id")?,
|
||||
token: MediaFileToken::new(row.try_get("token")?),
|
||||
media_class: MediaFileClass::try_from_mysql_row(row, "media_class")?,
|
||||
media_type: MediaFileType::try_from_mysql_row(row, "media_type")?,
|
||||
maybe_engine_category: MediaFileEngineCategory::try_from_mysql_row_nullable(row, "maybe_engine_category")?,
|
||||
maybe_animation_type: MediaFileAnimationType::try_from_mysql_row_nullable(row, "maybe_animation_type")?,
|
||||
origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?,
|
||||
origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?,
|
||||
maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?,
|
||||
maybe_origin_model_token: row.try_get("maybe_origin_model_token")?,
|
||||
maybe_origin_model_title: row.try_get("maybe_origin_model_title")?,
|
||||
maybe_other_prompt_args: row.try_get("maybe_other_prompt_args")?,
|
||||
maybe_duration_millis: row.try_get("maybe_duration_millis")?,
|
||||
public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?,
|
||||
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
|
||||
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
|
||||
maybe_prompt_token: PromptToken::try_from_mysql_row_nullable(row, "maybe_prompt_token")?,
|
||||
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
|
||||
is_user_upload: row.try_get("is_user_upload")?,
|
||||
is_intermediate_system_file: row.try_get("is_intermediate_system_file")?,
|
||||
maybe_file_cover_image_public_bucket_hash: row.try_get("maybe_file_cover_image_public_bucket_hash")?,
|
||||
maybe_file_cover_image_public_bucket_prefix: row.try_get("maybe_file_cover_image_public_bucket_prefix")?,
|
||||
maybe_file_cover_image_public_bucket_extension: row.try_get("maybe_file_cover_image_public_bucket_extension")?,
|
||||
maybe_title: row.try_get("maybe_title")?,
|
||||
maybe_text_transcript: row.try_get("maybe_text_transcript")?,
|
||||
maybe_ratings_positive_count: row.try_get("maybe_ratings_positive_count")?,
|
||||
maybe_ratings_negative_count: row.try_get("maybe_ratings_negative_count")?,
|
||||
maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
|
||||
created_at: row.try_get("created_at")?,
|
||||
updated_at: row.try_get("updated_at")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl From<MediaFileListItemInternal> for MediaFileListItem {
|
||||
fn from(record: MediaFileListItemInternal) -> Self {
|
||||
MediaFileListItem {
|
||||
token: record.token,
|
||||
media_class: record.media_class,
|
||||
media_type: record.media_type,
|
||||
maybe_engine_category: record.maybe_engine_category,
|
||||
maybe_animation_type: record.maybe_animation_type,
|
||||
origin_category: record.origin_category,
|
||||
origin_product_category: record.origin_product_category,
|
||||
maybe_origin_model_type: record.maybe_origin_model_type,
|
||||
maybe_origin_model_token: record.maybe_origin_model_token,
|
||||
maybe_origin_model_title: record.maybe_origin_model_title,
|
||||
public_bucket_directory_hash: record.public_bucket_directory_hash,
|
||||
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
|
||||
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
|
||||
maybe_prompt_token: record.maybe_prompt_token,
|
||||
creator_set_visibility: record.creator_set_visibility,
|
||||
is_user_upload: i8_to_bool(record.is_user_upload),
|
||||
is_intermediate_system_file: i8_to_bool(record.is_intermediate_system_file),
|
||||
maybe_file_cover_image_public_bucket_hash: record.maybe_file_cover_image_public_bucket_hash,
|
||||
maybe_file_cover_image_public_bucket_prefix: record.maybe_file_cover_image_public_bucket_prefix,
|
||||
maybe_file_cover_image_public_bucket_extension: record.maybe_file_cover_image_public_bucket_extension,
|
||||
maybe_title: record.maybe_title,
|
||||
maybe_text_transcript: record.maybe_text_transcript,
|
||||
maybe_prompt_args: record.maybe_other_prompt_args
|
||||
.as_deref()
|
||||
.map(|args| PromptInnerPayload::from_json(args))
|
||||
.transpose()
|
||||
.ok() // NB: Fail open
|
||||
.flatten(),
|
||||
maybe_duration_millis: record.maybe_duration_millis.map(|d| d as u64),
|
||||
maybe_ratings_positive_count: record.maybe_ratings_positive_count,
|
||||
maybe_ratings_negative_count: record.maybe_ratings_negative_count,
|
||||
maybe_bookmark_count: record.maybe_bookmark_count,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
+23
-15
@@ -27,6 +27,11 @@ pub struct VideoMediaFileWithoutThumbnail {
|
||||
pub id: i64,
|
||||
pub token: MediaFileToken,
|
||||
pub created_at: DateTime<Utc>,
|
||||
/// Database statement-start time, shared by every row in this page.
|
||||
pub database_read_at: DateTime<Utc>,
|
||||
/// Our job's successful finalization time, not the provider's completion time.
|
||||
/// Uploads, legacy records, and jobs still being finalized can have no timestamp.
|
||||
pub maybe_generation_completed_at: Option<DateTime<Utc>>,
|
||||
pub maybe_thumbnail_version: Option<u8>,
|
||||
pub public_bucket_directory_hash: String,
|
||||
pub maybe_public_bucket_prefix: Option<String>,
|
||||
@@ -51,22 +56,25 @@ pub async fn list_video_media_files_without_thumbnails_for_job<'e, 'c, E>(
|
||||
VideoMediaFileWithoutThumbnail,
|
||||
r#"
|
||||
SELECT
|
||||
id,
|
||||
token as `token: MediaFileToken`,
|
||||
created_at as `created_at: DateTime<Utc>`,
|
||||
maybe_thumbnail_version as `maybe_thumbnail_version: u8`,
|
||||
public_bucket_directory_hash,
|
||||
maybe_public_bucket_prefix,
|
||||
maybe_public_bucket_extension
|
||||
FROM media_files
|
||||
media.id,
|
||||
media.token as `token: MediaFileToken`,
|
||||
media.created_at as `created_at: DateTime<Utc>`,
|
||||
NOW(6) as `database_read_at!: DateTime<Utc>`,
|
||||
jobs.successfully_completed_at as `maybe_generation_completed_at?: DateTime<Utc>`,
|
||||
media.maybe_thumbnail_version as `maybe_thumbnail_version: u8`,
|
||||
media.public_bucket_directory_hash,
|
||||
media.maybe_public_bucket_prefix,
|
||||
media.maybe_public_bucket_extension
|
||||
FROM media_files AS media
|
||||
LEFT JOIN generic_inference_jobs AS jobs ON jobs.token = media.maybe_source_job_token
|
||||
WHERE
|
||||
id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)
|
||||
AND id < ?
|
||||
AND media_class = ?
|
||||
AND maybe_thumbnail_version IS NULL
|
||||
AND user_deleted_at IS NULL
|
||||
AND mod_deleted_at IS NULL
|
||||
ORDER BY id DESC
|
||||
media.id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)
|
||||
AND media.id < ?
|
||||
AND media.media_class = ?
|
||||
AND media.maybe_thumbnail_version IS NULL
|
||||
AND media.user_deleted_at IS NULL
|
||||
AND media.mod_deleted_at IS NULL
|
||||
ORDER BY media.id DESC
|
||||
LIMIT ?
|
||||
"#,
|
||||
max_lookback_hours,
|
||||
|
||||
@@ -41,6 +41,7 @@ pub mod unsubscribe_reason;
|
||||
pub mod uploaded_video_notes;
|
||||
pub mod uploaded_videos;
|
||||
pub mod user_impersonation_requests;
|
||||
pub mod user_login_challenges;
|
||||
pub mod user_referral_codes;
|
||||
pub mod user_referrals;
|
||||
pub mod user_daily_spends;
|
||||
@@ -52,4 +53,4 @@ pub mod voice_conversion;
|
||||
pub mod voice_designer;
|
||||
pub mod wallet_ledger_entries;
|
||||
pub mod wallets;
|
||||
pub mod web_referrals;
|
||||
pub mod web_referrals;
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
use tokens::tokens::user_login_challenges::UserLoginChallengeToken;
|
||||
|
||||
pub struct CreateChallengeArgs<'a, T> {
|
||||
pub token: &'a UserLoginChallengeToken,
|
||||
pub approval_hash: &'a [u8],
|
||||
pub device_hash: &'a [u8],
|
||||
pub confirmation_code: &'a str,
|
||||
pub ip_address: &'a str,
|
||||
pub status: UserLoginChallengeStatus,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
pub async fn create_challenge<'c, T>(args: CreateChallengeArgs<'_, T>) -> Result<(), sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO user_login_challenges
|
||||
(token, approval_token_sha256, device_token_sha256, confirmation_code, ip_address_creation, status, created_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, NOW(), NOW() + INTERVAL 20 MINUTE)
|
||||
"#,
|
||||
args.token.as_str(),
|
||||
args.approval_hash,
|
||||
args.device_hash,
|
||||
args.confirmation_code,
|
||||
args.ip_address,
|
||||
args.status.to_str()
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct DecideChallengeArgs<'a, T> {
|
||||
pub challenge_id: u64,
|
||||
pub user_token: &'a str,
|
||||
pub ip_address: &'a str,
|
||||
pub approve: bool,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
pub async fn decide_challenge<'c, T>(args: DecideChallengeArgs<'_, T>) -> Result<bool, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let status = if args.approve { UserLoginChallengeStatus::Approved } else { UserLoginChallengeStatus::Failed };
|
||||
let failure = if args.approve { None } else { Some(UserLoginChallengeFailureType::UserDeclined) };
|
||||
let failure_ip = if args.approve { None } else { Some(args.ip_address) };
|
||||
let affected = sqlx::query!(
|
||||
r#"
|
||||
UPDATE user_login_challenges SET status = ?, maybe_deciding_user_token = ?,
|
||||
maybe_ip_address_decision = ?, maybe_decided_at = NOW(), maybe_failure_type = ?,
|
||||
maybe_ip_address_failure = ?, maybe_failed_at = IF(?, NULL, NOW())
|
||||
WHERE id = ? AND status = ? AND expires_at > NOW()
|
||||
"#,
|
||||
status.to_str(),
|
||||
args.user_token,
|
||||
args.ip_address,
|
||||
failure.map(|value| value.to_str()),
|
||||
failure_ip,
|
||||
args.approve,
|
||||
args.challenge_id,
|
||||
UserLoginChallengeStatus::Pending.to_str()
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok(affected == 1)
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct ExpireAbandonedChallengesArgs<T> {
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Bounded sweep also audits abandoned attempts that are never polled again.
|
||||
pub async fn expire_abandoned_challenges<'c, T>(args: ExpireAbandonedChallengesArgs<T>) -> Result<u64, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let result = sqlx::query!(
|
||||
r#"
|
||||
UPDATE user_login_challenges SET status = ?, maybe_failure_type = ?, maybe_failed_at = expires_at
|
||||
WHERE status IN (?, ?) AND expires_at <= NOW() ORDER BY expires_at LIMIT 1000
|
||||
"#,
|
||||
UserLoginChallengeStatus::Failed.to_str(),
|
||||
UserLoginChallengeFailureType::Expired.to_str(),
|
||||
UserLoginChallengeStatus::Pending.to_str(),
|
||||
UserLoginChallengeStatus::Approved.to_str()
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct ExpireChallengeArgs<T> {
|
||||
pub challenge_id: u64,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Audit expiration of a pending/approved challenge in the caller's transaction.
|
||||
pub async fn expire_challenge<'c, T>(args: ExpireChallengeArgs<T>) -> Result<(), sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query!(
|
||||
r#"
|
||||
UPDATE user_login_challenges SET status = ?, maybe_failure_type = ?, maybe_failed_at = expires_at
|
||||
WHERE id = ? AND status IN (?, ?) AND expires_at <= NOW()
|
||||
"#,
|
||||
UserLoginChallengeStatus::Failed.to_str(),
|
||||
UserLoginChallengeFailureType::Expired.to_str(),
|
||||
args.challenge_id,
|
||||
UserLoginChallengeStatus::Pending.to_str(),
|
||||
UserLoginChallengeStatus::Approved.to_str()
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct IsChallengeExpiredArgs<T> {
|
||||
pub challenge_id: u64,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Run AFTER acquiring the row lock: NOW() on the locking statement can precede
|
||||
/// time spent waiting for another transaction. Never extend the original deadline.
|
||||
pub async fn is_challenge_expired<'c, T>(args: IsChallengeExpiredArgs<T>) -> Result<bool, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let result = sqlx::query!(
|
||||
r#"
|
||||
SELECT expires_at <= NOW() AS expired FROM user_login_challenges WHERE id = ?
|
||||
"#,
|
||||
args.challenge_id
|
||||
)
|
||||
.fetch_one(args.mysql_executor)
|
||||
.await?;
|
||||
Ok(result.expired != 0)
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
use super::login_challenge::LoginChallenge;
|
||||
|
||||
pub struct LockByApprovalArgs<'a, T> {
|
||||
pub approval_hash: &'a [u8],
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Use the caller's transaction to retain the row lock for subsequent decisions.
|
||||
/// The approval token cannot be used in device lookups.
|
||||
pub async fn lock_by_approval<'c, T>(args: LockByApprovalArgs<'_, T>) -> Result<Option<LoginChallenge>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query_as!(
|
||||
LoginChallenge,
|
||||
r#"
|
||||
SELECT id, confirmation_code, status AS `status: UserLoginChallengeStatus`, maybe_deciding_user_token, maybe_redeemed_user_session_id,
|
||||
maybe_failure_type AS `maybe_failure_type: UserLoginChallengeFailureType`, ip_address_creation, expires_at
|
||||
FROM user_login_challenges WHERE approval_token_sha256 = ? FOR UPDATE
|
||||
"#,
|
||||
args.approval_hash
|
||||
)
|
||||
.fetch_optional(args.mysql_executor)
|
||||
.await
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
use super::login_challenge::LoginChallenge;
|
||||
|
||||
pub struct LockByDeviceArgs<'a, T> {
|
||||
pub device_hash: &'a [u8],
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Use the caller's transaction to retain the row lock throughout redemption.
|
||||
pub async fn lock_by_device<'c, T>(args: LockByDeviceArgs<'_, T>) -> Result<Option<LoginChallenge>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query_as!(
|
||||
LoginChallenge,
|
||||
r#"
|
||||
SELECT id, confirmation_code, status AS `status: UserLoginChallengeStatus`, maybe_deciding_user_token, maybe_redeemed_user_session_id,
|
||||
maybe_failure_type AS `maybe_failure_type: UserLoginChallengeFailureType`, ip_address_creation, expires_at
|
||||
FROM user_login_challenges WHERE device_token_sha256 = ? FOR UPDATE
|
||||
"#,
|
||||
args.device_hash
|
||||
)
|
||||
.fetch_optional(args.mysql_executor)
|
||||
.await
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
|
||||
pub struct LoginChallenge {
|
||||
pub id: u64,
|
||||
pub confirmation_code: String,
|
||||
pub status: UserLoginChallengeStatus,
|
||||
pub maybe_deciding_user_token: Option<String>,
|
||||
pub maybe_redeemed_user_session_id: Option<i64>,
|
||||
pub maybe_failure_type: Option<UserLoginChallengeFailureType>,
|
||||
pub ip_address_creation: String,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
pub struct MarkRedeemedArgs<'a, T> {
|
||||
pub challenge_id: u64,
|
||||
pub session_id: i64,
|
||||
pub ip_address: &'a str,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
pub async fn mark_redeemed<'c, T>(args: MarkRedeemedArgs<'_, T>) -> Result<bool, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let affected = sqlx::query!(
|
||||
r#"
|
||||
UPDATE user_login_challenges SET status = ?, maybe_redeemed_user_session_id = ?,
|
||||
maybe_ip_address_redemption = ?, maybe_redeemed_at = NOW()
|
||||
WHERE id = ? AND status = ? AND expires_at > NOW() AND maybe_redeemed_user_session_id IS NULL
|
||||
"#,
|
||||
UserLoginChallengeStatus::Redeemed.to_str(),
|
||||
args.session_id,
|
||||
args.ip_address,
|
||||
args.challenge_id,
|
||||
UserLoginChallengeStatus::Approved.to_str()
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok(affected == 1)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
pub mod create_challenge;
|
||||
pub mod decide_challenge;
|
||||
pub mod expire_abandoned_challenges;
|
||||
pub mod expire_challenge;
|
||||
pub mod is_challenge_expired;
|
||||
pub mod lock_by_approval;
|
||||
pub mod lock_by_device;
|
||||
pub mod login_challenge;
|
||||
pub mod mark_redeemed;
|
||||
@@ -0,0 +1,6 @@
|
||||
pub struct BridgeSession {
|
||||
pub id: i64,
|
||||
pub token: String,
|
||||
pub user_token: String,
|
||||
pub username: String,
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use sqlx::{Executor, MySql};
|
||||
use tokens::tokens::user_sessions::UserSessionToken;
|
||||
|
||||
pub struct CreateBridgeSessionArgs<'a, T> {
|
||||
pub user_token: &'a str,
|
||||
pub ip_address: &'a str,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Returns the inserted session ID, or None if the user cannot sign in.
|
||||
/// Must share the transaction holding the challenge lock. A failed/expired
|
||||
/// redemption rolls back this INSERT; no session is minted when merely approving.
|
||||
pub async fn create_bridge_session<'c, T>(args: CreateBridgeSessionArgs<'_, T>) -> Result<Option<i64>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
let token = UserSessionToken::generate();
|
||||
let inserted = sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO user_sessions (token, user_token, ip_address_creation, maybe_creation_type, expires_at)
|
||||
SELECT ?, token, ?, ?, NOW() + INTERVAL 1 YEAR FROM users
|
||||
WHERE token = ? AND is_banned = FALSE AND user_deleted_at IS NULL AND mod_deleted_at IS NULL
|
||||
"#,
|
||||
token.as_str(),
|
||||
args.ip_address,
|
||||
UserWebSessionCreationType::DeviceApproval.to_str(),
|
||||
args.user_token
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
.await?;
|
||||
if inserted.rows_affected() != 1 {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(inserted.last_insert_id() as i64))
|
||||
}
|
||||
+4
-1
@@ -1,6 +1,7 @@
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
use tokens::tokens::user_sessions::UserSessionToken;
|
||||
@@ -36,14 +37,16 @@ INSERT INTO user_sessions (
|
||||
user_token,
|
||||
maybe_impersonation_user_token,
|
||||
ip_address_creation,
|
||||
maybe_creation_type,
|
||||
expires_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
"#,
|
||||
session_token.as_str(),
|
||||
args.user_token.as_str(),
|
||||
args.impersonator_user_token.as_str(),
|
||||
args.ip_address,
|
||||
UserWebSessionCreationType::Impersonation.to_str(),
|
||||
args.expires_at,
|
||||
)
|
||||
.execute(args.mysql_executor)
|
||||
|
||||
+4
-1
@@ -1,4 +1,5 @@
|
||||
use anyhow::anyhow;
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use log::info;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
@@ -17,13 +18,15 @@ INSERT INTO user_sessions (
|
||||
token,
|
||||
user_token,
|
||||
ip_address_creation,
|
||||
maybe_creation_type,
|
||||
expires_at
|
||||
)
|
||||
VALUES ( ?, ?, ?, NOW() + interval 1 year )
|
||||
VALUES ( ?, ?, ?, ?, NOW() + interval 1 year )
|
||||
"#,
|
||||
session_token,
|
||||
user_token.to_string(),
|
||||
ip_address.to_string(),
|
||||
UserWebSessionCreationType::DirectLogin.to_str(),
|
||||
)
|
||||
.execute(mysql_pool)
|
||||
.await;
|
||||
|
||||
+4
-1
@@ -1,3 +1,4 @@
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use log::info;
|
||||
use sqlx::{Executor, MySql};
|
||||
use tokens::tokens::user_sessions::UserSessionToken;
|
||||
@@ -19,13 +20,15 @@ INSERT INTO user_sessions (
|
||||
token,
|
||||
user_token,
|
||||
ip_address_creation,
|
||||
maybe_creation_type,
|
||||
expires_at
|
||||
)
|
||||
VALUES ( ?, ?, ?, NOW() + interval 1 year )
|
||||
VALUES ( ?, ?, ?, ?, NOW() + interval 1 year )
|
||||
"#,
|
||||
session_token.as_str(),
|
||||
user_token.as_str(),
|
||||
ip_address,
|
||||
UserWebSessionCreationType::DirectLogin.to_str(),
|
||||
)
|
||||
.execute(mysql_executor)
|
||||
.await?;
|
||||
|
||||
+4
-1
@@ -1,4 +1,5 @@
|
||||
use anyhow::anyhow;
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use log::info;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
@@ -22,13 +23,15 @@ INSERT INTO user_sessions (
|
||||
token,
|
||||
user_token,
|
||||
ip_address_creation,
|
||||
maybe_creation_type,
|
||||
expires_at
|
||||
)
|
||||
VALUES ( ?, ?, ?, NOW() + interval 1 year )
|
||||
VALUES ( ?, ?, ?, ?, NOW() + interval 1 year )
|
||||
"#,
|
||||
session_token.as_str(),
|
||||
user_token.as_str(),
|
||||
ip_address,
|
||||
UserWebSessionCreationType::DirectLogin.to_str(),
|
||||
);
|
||||
|
||||
let query_result = transactor.execute(query).await;
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
use super::bridge_session::BridgeSession;
|
||||
|
||||
pub struct FindApprovingSessionArgs<'a, T> {
|
||||
pub session_token: &'a str,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Fresh database lookup, not a cached session. Impersonated sessions cannot consent.
|
||||
pub async fn find_approving_session<'c, T>(args: FindApprovingSessionArgs<'_, T>) -> Result<Option<BridgeSession>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query_as!(
|
||||
BridgeSession,
|
||||
r#"
|
||||
SELECT s.id, s.token, s.user_token, u.username FROM user_sessions s JOIN users u ON u.token = s.user_token
|
||||
WHERE s.token = ? AND s.deleted_at IS NULL AND s.expires_at > NOW()
|
||||
AND s.maybe_impersonation_user_token IS NULL AND u.is_banned = FALSE
|
||||
AND u.user_deleted_at IS NULL AND u.mod_deleted_at IS NULL
|
||||
"#,
|
||||
args.session_token
|
||||
)
|
||||
.fetch_optional(args.mysql_executor)
|
||||
.await
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use sqlx::{Executor, MySql};
|
||||
|
||||
use super::bridge_session::BridgeSession;
|
||||
|
||||
pub struct FindRedeemedSessionArgs<'a, T> {
|
||||
pub session_id: i64,
|
||||
pub user_token: &'a str,
|
||||
pub mysql_executor: T,
|
||||
}
|
||||
|
||||
/// Returns the one previously minted session, only while it and its owner are live.
|
||||
pub async fn find_redeemed_session<'c, T>(args: FindRedeemedSessionArgs<'_, T>) -> Result<Option<BridgeSession>, sqlx::Error>
|
||||
where
|
||||
T: Executor<'c, Database = MySql>,
|
||||
{
|
||||
sqlx::query_as!(
|
||||
BridgeSession,
|
||||
r#"
|
||||
SELECT s.id, s.token, s.user_token, u.username FROM user_sessions s JOIN users u ON u.token = s.user_token
|
||||
WHERE s.id = ? AND s.user_token = ? AND s.maybe_creation_type = ?
|
||||
AND s.deleted_at IS NULL AND s.expires_at > NOW() AND s.maybe_impersonation_user_token IS NULL
|
||||
AND u.is_banned = FALSE AND u.user_deleted_at IS NULL AND u.mod_deleted_at IS NULL
|
||||
"#,
|
||||
args.session_id,
|
||||
args.user_token,
|
||||
UserWebSessionCreationType::DeviceApproval.to_str()
|
||||
)
|
||||
.fetch_optional(args.mysql_executor)
|
||||
.await
|
||||
}
|
||||
+1
@@ -169,6 +169,7 @@ LEFT OUTER JOIN user_roles
|
||||
ON users.user_role_slug = user_roles.slug
|
||||
WHERE user_sessions.token = ?
|
||||
AND user_sessions.deleted_at IS NULL
|
||||
AND user_sessions.expires_at > NOW()
|
||||
AND users.user_deleted_at IS NULL
|
||||
AND users.mod_deleted_at IS NULL
|
||||
"#,
|
||||
|
||||
+1
@@ -26,6 +26,7 @@ SELECT
|
||||
FROM user_sessions
|
||||
WHERE token = ?
|
||||
AND deleted_at IS NULL
|
||||
AND expires_at > NOW()
|
||||
"#,
|
||||
session_token,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
pub mod bridge_session;
|
||||
pub mod create_bridge_session;
|
||||
pub mod create_impersonated_user_session;
|
||||
pub mod create_user_session;
|
||||
pub mod create_user_session_with_executor;
|
||||
pub mod create_user_session_with_transactor;
|
||||
pub mod delete_user_session;
|
||||
pub mod find_approving_session;
|
||||
pub mod find_redeemed_session;
|
||||
pub mod get_user_session_by_token;
|
||||
pub mod get_user_session_by_token_light;
|
||||
@@ -0,0 +1,90 @@
|
||||
//! Mutators/inspection for disposable bridge integration databases only.
|
||||
use sqlx::{MySqlPool, Row};
|
||||
use chrono::{DateTime, Utc};
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_failure_type::UserLoginChallengeFailureType;
|
||||
use enums::by_table::user_login_challenges::user_login_challenge_status::UserLoginChallengeStatus;
|
||||
use enums::by_table::user_sessions::user_web_session_creation_type::UserWebSessionCreationType;
|
||||
use enums::by_table::users::user_feature_flag::UserFeatureFlag;
|
||||
use tokens::tokens::user_login_challenges::UserLoginChallengeToken;
|
||||
use tokens::tokens::users::UserToken;
|
||||
|
||||
use super::users::{create_test_user, TestUser};
|
||||
|
||||
pub struct ChallengeAudit {
|
||||
pub status: UserLoginChallengeStatus,
|
||||
pub failure: Option<UserLoginChallengeFailureType>,
|
||||
pub creation_ip: String,
|
||||
pub failure_ip: Option<String>,
|
||||
pub session_id: Option<i64>,
|
||||
pub lifetime_seconds: i64,
|
||||
pub deciding_user: Option<String>,
|
||||
pub decision_ip: Option<String>,
|
||||
pub redemption_ip: Option<String>,
|
||||
pub decided_at: Option<DateTime<Utc>>,
|
||||
pub redeemed_at: Option<DateTime<Utc>>,
|
||||
pub failed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
pub async fn create_passwordless_user(pool: &MySqlPool, flags: &[UserFeatureFlag]) -> TestUser {
|
||||
let user = create_test_user(pool).await.expect("create passwordless fixture");
|
||||
set_passwordless(pool, &user.user_token).await;
|
||||
let flags = flags.iter().map(UserFeatureFlag::to_str).collect::<Vec<_>>().join(",");
|
||||
sqlx::query("UPDATE users SET maybe_feature_flags = ? WHERE token = ?")
|
||||
.bind(flags).bind(user.user_token.as_str()).execute(pool).await.expect("set fixture feature flags");
|
||||
user
|
||||
}
|
||||
|
||||
pub async fn audit(pool: &MySqlPool, device_hash: &[u8]) -> ChallengeAudit {
|
||||
let row = sqlx::query("SELECT status, maybe_failure_type, ip_address_creation, maybe_ip_address_failure, maybe_redeemed_user_session_id, TIMESTAMPDIFF(SECOND, created_at, expires_at) AS lifetime, maybe_deciding_user_token, maybe_ip_address_decision, maybe_ip_address_redemption, maybe_decided_at, maybe_redeemed_at, maybe_failed_at FROM user_login_challenges WHERE device_token_sha256 = ?").bind(device_hash).fetch_one(pool).await.expect("challenge audit");
|
||||
ChallengeAudit {
|
||||
status: row.get("status"), failure: row.get("maybe_failure_type"), creation_ip: row.get("ip_address_creation"),
|
||||
failure_ip: row.get("maybe_ip_address_failure"), session_id: row.get("maybe_redeemed_user_session_id"), lifetime_seconds: row.get("lifetime"),
|
||||
deciding_user: row.get("maybe_deciding_user_token"), decision_ip: row.get("maybe_ip_address_decision"),
|
||||
redemption_ip: row.get("maybe_ip_address_redemption"), decided_at: row.get("maybe_decided_at"),
|
||||
redeemed_at: row.get("maybe_redeemed_at"), failed_at: row.get("maybe_failed_at"),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn bridge_session_count(pool: &MySqlPool) -> i64 {
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM user_sessions WHERE maybe_creation_type = ?")
|
||||
.bind(UserWebSessionCreationType::DeviceApproval.to_str()).fetch_one(pool).await.expect("bridge session count")
|
||||
}
|
||||
|
||||
pub async fn assert_challenge_status_is_explicit(pool: &MySqlPool) {
|
||||
let column = sqlx::query("SELECT COLUMN_DEFAULT, IS_NULLABLE FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'user_login_challenges' AND COLUMN_NAME = 'status'")
|
||||
.fetch_one(pool).await.expect("inspect migrated status column");
|
||||
assert!(column.get::<Option<String>, _>("COLUMN_DEFAULT").is_none());
|
||||
assert_eq!(column.get::<String, _>("IS_NULLABLE"), "NO");
|
||||
|
||||
// Every other required field is supplied; MySQL must reject the omitted status.
|
||||
let token = UserLoginChallengeToken::generate();
|
||||
let error = sqlx::query("INSERT INTO user_login_challenges (token, approval_token_sha256, device_token_sha256, confirmation_code, ip_address_creation, expires_at) VALUES (?, ?, ?, ?, ?, NOW() + INTERVAL 20 MINUTE)")
|
||||
.bind(token.as_str()).bind(&[0u8; 32][..]).bind(&[1u8; 32][..])
|
||||
.bind("BCDFGHJK").bind("192.0.2.1").execute(pool).await.expect_err("status must be explicit");
|
||||
let error = error.as_database_error().expect("MySQL missing-column error");
|
||||
assert_eq!(error.try_downcast_ref::<sqlx::mysql::MySqlDatabaseError>().unwrap().number(), 1364);
|
||||
}
|
||||
|
||||
pub async fn set_challenge_deadline(pool: &MySqlPool, device_hash: &[u8], seconds_from_now: i32) {
|
||||
sqlx::query("UPDATE user_login_challenges SET expires_at = TIMESTAMPADD(SECOND, ?, NOW()) WHERE device_token_sha256 = ?").bind(seconds_from_now).bind(device_hash).execute(pool).await.expect("change test deadline");
|
||||
}
|
||||
|
||||
pub async fn expire_session(pool: &MySqlPool, token: &str) {
|
||||
sqlx::query("UPDATE user_sessions SET expires_at = NOW() - INTERVAL 1 SECOND WHERE token = ?").bind(token).execute(pool).await.expect("expire test session");
|
||||
}
|
||||
|
||||
pub async fn revoke_session(pool: &MySqlPool, token: &str) {
|
||||
sqlx::query("UPDATE user_sessions SET deleted_at = NOW() WHERE token = ?").bind(token).execute(pool).await.expect("revoke test session");
|
||||
}
|
||||
|
||||
pub async fn set_passwordless(pool: &MySqlPool, user: &UserToken) {
|
||||
sqlx::query("UPDATE users SET is_without_password = TRUE, email_confirmed_by_google = TRUE WHERE token = ?").bind(user.as_str()).execute(pool).await.expect("passwordless test account");
|
||||
}
|
||||
|
||||
pub async fn ban_user(pool: &MySqlPool, user: &UserToken) {
|
||||
sqlx::query("UPDATE users SET is_banned = TRUE WHERE token = ?").bind(user.as_str()).execute(pool).await.expect("ban test account");
|
||||
}
|
||||
|
||||
pub async fn impersonate_session(pool: &MySqlPool, token: &str, actor: &UserToken) {
|
||||
sqlx::query("UPDATE user_sessions SET maybe_impersonation_user_token = ? WHERE token = ?").bind(actor.as_str()).bind(token).execute(pool).await.expect("test impersonation");
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod mcp_sessions;
|
||||
pub mod login_challenges;
|
||||
pub mod media_files;
|
||||
pub mod users;
|
||||
pub mod wallets;
|
||||
@@ -0,0 +1,36 @@
|
||||
//! Disposable databases on the local MySQL instance only. No application or test
|
||||
//! database URL is read, so production configuration cannot redirect these tests.
|
||||
|
||||
use sqlx::mysql::{MySqlConnectOptions, MySqlPoolOptions, MySqlSslMode};
|
||||
use sqlx::MySqlPool;
|
||||
use tokens::tokens::users::UserToken;
|
||||
|
||||
use crate::schema::ensure_schema;
|
||||
|
||||
pub struct IsolatedTestDatabase {
|
||||
pub pool: MySqlPool,
|
||||
admin_pool: MySqlPool,
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl IsolatedTestDatabase {
|
||||
pub async fn create() -> Self {
|
||||
// Deliberately no URL/env parsing, DNS, TCP (including forwarded local ports),
|
||||
// or socket overrides. This is the local Homebrew MySQL socket. Only a new
|
||||
// randomly named database can be created/dropped; existing schemas are unused.
|
||||
let options = MySqlConnectOptions::new().socket("/tmp/mysql.sock").username("root").ssl_mode(MySqlSslMode::Disabled);
|
||||
let admin_pool = MySqlPoolOptions::new().max_connections(1).connect_with(options.clone()).await.expect("local MySQL test connection");
|
||||
let name = format!("artcraft_test_bridge_{}", UserToken::generate().as_str());
|
||||
assert!(name.len() <= 64 && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'));
|
||||
sqlx::query(&format!("CREATE DATABASE `{name}`")).execute(&admin_pool).await.expect("create disposable database");
|
||||
let pool = MySqlPoolOptions::new().max_connections(8).connect_with(options.database(&name)).await.expect("connect disposable database");
|
||||
ensure_schema(&pool).await;
|
||||
Self { pool, admin_pool, name }
|
||||
}
|
||||
|
||||
pub async fn destroy(self) {
|
||||
self.pool.close().await;
|
||||
sqlx::query(&format!("DROP DATABASE `{}`", self.name)).execute(&self.admin_pool).await.expect("drop our disposable database");
|
||||
self.admin_pool.close().await;
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@
|
||||
|
||||
pub mod fixtures;
|
||||
pub mod guard;
|
||||
pub mod isolated;
|
||||
pub mod pool;
|
||||
pub mod schema;
|
||||
pub mod serial;
|
||||
@@ -0,0 +1,309 @@
|
||||
//! Executes the actual library queries against migrated, disposable local MySQL.
|
||||
//! IsolatedTestDatabase uses a fixed Unix socket, never DATABASE_URL/MYSQL_URL.
|
||||
use std::collections::HashSet;
|
||||
use std::time::Instant;
|
||||
|
||||
use enums::by_table::entity_stats::stats_entity_type::StatsEntityType;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use enums::by_table::prompts::prompt_type::PromptType;
|
||||
use enums::common::view_as::ViewAs;
|
||||
use enums::common::visibility::Visibility;
|
||||
use mysql_queries::queries::media_files::list::count_media_files_for_user::{
|
||||
count_media_files_for_user, CountMediaFilesForUserArgs,
|
||||
};
|
||||
use mysql_queries::queries::media_files::list::list_media_files_for_user::{
|
||||
list_media_files_for_user, ListMediaFileForUserArgs,
|
||||
};
|
||||
use mysql_queries::queries::media_files::list::user_media_file_filters::UserMediaFileFilters;
|
||||
use mysql_testing::fixtures::users::create_test_user;
|
||||
use mysql_testing::isolated::IsolatedTestDatabase;
|
||||
use sqlx::{MySql, MySqlPool, QueryBuilder};
|
||||
use tokens::tokens::users::UserToken;
|
||||
|
||||
const CREATED_AT: &str = "2025-01-01 12:00:00";
|
||||
|
||||
#[tokio::test]
|
||||
async fn user_media_library_preserves_filters_visibility_joins_and_pagination() {
|
||||
let db = IsolatedTestDatabase::create().await;
|
||||
let owner = create_test_user(&db.pool).await.unwrap();
|
||||
let other = create_test_user(&db.pool).await.unwrap();
|
||||
let rows = [
|
||||
("m_public_image", MediaFileClass::Image, MediaFileType::Png, Visibility::Public, true),
|
||||
("m_private_video", MediaFileClass::Video, MediaFileType::Mp4, Visibility::Private, false),
|
||||
("m_hidden_audio", MediaFileClass::Audio, MediaFileType::Mp3, Visibility::Hidden, false),
|
||||
("m_public_mesh", MediaFileClass::Mesh, MediaFileType::Glb, Visibility::Public, false),
|
||||
("m_private_splat", MediaFileClass::Splat, MediaFileType::Spz, Visibility::Private, false),
|
||||
("m_legacy_mesh", MediaFileClass::Dimensional, MediaFileType::Glb, Visibility::Public, false),
|
||||
];
|
||||
for (token, class, kind, visibility, upload) in rows {
|
||||
insert_fixture(&db.pool, Fixture { owner: &owner.user_token, token, class, kind, visibility, upload }).await;
|
||||
}
|
||||
for token in ["m_intermediate", "m_user_deleted", "m_mod_deleted"] {
|
||||
insert_fixture(&db.pool, Fixture::image(&owner.user_token, token)).await;
|
||||
}
|
||||
insert_fixture(&db.pool, Fixture::image(&other.user_token, "m_other_user")).await;
|
||||
sqlx::query("UPDATE media_files SET is_intermediate_system_file = true WHERE token = 'm_intermediate'")
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE media_files SET user_deleted_at = NOW() WHERE token = 'm_user_deleted'")
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE media_files SET mod_deleted_at = NOW() WHERE token = 'm_mod_deleted'")
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE media_files SET maybe_engine_category = ? WHERE token = 'm_public_mesh'")
|
||||
.bind(MediaFileEngineCategory::Object.to_str())
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO prompts (token, prompt_type, creator_ip_address, maybe_other_args) VALUES ('p_library', ?, '127.0.0.1', ?)")
|
||||
.bind(PromptType::ArtcraftApp.to_str()).bind(r#"{"fd":true}"#).execute(&db.pool).await.unwrap();
|
||||
sqlx::query("UPDATE media_files SET maybe_cover_image_media_file_token = 'm_intermediate', maybe_prompt_token = 'p_library', maybe_duration_millis = 1234, maybe_title = 'Fixture title' WHERE token = 'm_private_video'").execute(&db.pool).await.unwrap();
|
||||
sqlx::query("INSERT INTO entity_stats (entity_type, entity_token, ratings_positive_count, bookmark_count) VALUES (?, 'm_private_video', 3, 7)")
|
||||
.bind(StatsEntityType::MediaFile.to_str()).execute(&db.pool).await.unwrap();
|
||||
|
||||
let filters = filters(&owner.username);
|
||||
for (view_as, expected) in [
|
||||
(
|
||||
ViewAs::Author,
|
||||
vec!["m_legacy_mesh", "m_private_splat", "m_public_mesh", "m_hidden_audio", "m_private_video", "m_public_image"],
|
||||
),
|
||||
(
|
||||
ViewAs::Moderator,
|
||||
vec!["m_legacy_mesh", "m_private_splat", "m_public_mesh", "m_hidden_audio", "m_private_video", "m_public_image"],
|
||||
),
|
||||
(ViewAs::AnotherUser, vec!["m_legacy_mesh", "m_public_mesh", "m_public_image"]),
|
||||
] {
|
||||
let selected = UserMediaFileFilters { view_as, ..filters };
|
||||
let records = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters: selected,
|
||||
limit: 60,
|
||||
offset: 0,
|
||||
sort_ascending: false,
|
||||
mysql_executor: &db.pool,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(records.iter().map(|r| r.token.as_str()).collect::<Vec<_>>(), expected);
|
||||
assert_eq!(
|
||||
count_media_files_for_user(CountMediaFilesForUserArgs { filters: selected, mysql_executor: &db.pool })
|
||||
.await
|
||||
.unwrap(),
|
||||
expected.len() as i64
|
||||
);
|
||||
}
|
||||
let records = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters,
|
||||
limit: 60,
|
||||
offset: 0,
|
||||
sort_ascending: true,
|
||||
mysql_executor: &db.pool,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(records.first().unwrap().token.as_str(), "m_public_image");
|
||||
let video = &records[1];
|
||||
assert_eq!(video.maybe_file_cover_image_public_bucket_hash.as_deref(), Some("fixture_hash"));
|
||||
assert_eq!(video.maybe_prompt_args.as_ref().unwrap().used_face_detailer, Some(true));
|
||||
assert_eq!(video.maybe_ratings_positive_count, Some(3));
|
||||
assert_eq!(video.maybe_bookmark_count, Some(7));
|
||||
assert_eq!(video.maybe_duration_millis, Some(1234));
|
||||
assert_eq!(video.maybe_title.as_deref(), Some("Fixture title"));
|
||||
|
||||
// All timestamps tie. The id tiebreaker must avoid duplicates/omissions.
|
||||
let mut paged = vec![];
|
||||
for offset in [0, 2, 4, 6] {
|
||||
let records = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters,
|
||||
limit: 2,
|
||||
offset,
|
||||
sort_ascending: true,
|
||||
mysql_executor: &db.pool,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
paged.extend(records.into_iter().map(|r| r.token.to_string()));
|
||||
}
|
||||
assert_eq!(paged, rows.iter().map(|r| r.0.to_string()).collect::<Vec<_>>());
|
||||
let classes = HashSet::from([MediaFileClass::Image, MediaFileClass::Video]);
|
||||
let types = HashSet::from([MediaFileType::Glb]);
|
||||
let engines = HashSet::from([MediaFileEngineCategory::Object]);
|
||||
let empty = HashSet::new();
|
||||
for (selected, expected) in [
|
||||
(UserMediaFileFilters { maybe_filter_media_classes: Some(&classes), ..filters }, 2),
|
||||
(UserMediaFileFilters { maybe_filter_media_types: Some(&types), ..filters }, 2),
|
||||
(UserMediaFileFilters { maybe_filter_engine_categories: Some(&engines), ..filters }, 1),
|
||||
(UserMediaFileFilters { include_user_uploads: false, ..filters }, 5),
|
||||
(UserMediaFileFilters { maybe_filter_media_classes: Some(&empty), ..filters }, 6),
|
||||
(UserMediaFileFilters { username: "' OR 1=1 --", ..filters }, 0),
|
||||
] {
|
||||
let records = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters: selected,
|
||||
limit: 60,
|
||||
offset: 0,
|
||||
sort_ascending: false,
|
||||
mysql_executor: &db.pool,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), expected);
|
||||
assert_eq!(
|
||||
count_media_files_for_user(CountMediaFilesForUserArgs { filters: selected, mysql_executor: &db.pool })
|
||||
.await
|
||||
.unwrap(),
|
||||
expected as i64
|
||||
);
|
||||
}
|
||||
// Queries must honor an existing transaction, including uncommitted changes.
|
||||
let mut tx = db.pool.begin().await.unwrap();
|
||||
sqlx::query("UPDATE media_files SET user_deleted_at = NOW() WHERE token = 'm_private_video'")
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
count_media_files_for_user(CountMediaFilesForUserArgs { filters, mysql_executor: &mut *tx }).await.unwrap(),
|
||||
5
|
||||
);
|
||||
assert_eq!(
|
||||
list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters,
|
||||
limit: 60,
|
||||
offset: 0,
|
||||
sort_ascending: false,
|
||||
mysql_executor: &mut *tx
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
5
|
||||
);
|
||||
tx.rollback().await.unwrap();
|
||||
assert_eq!(
|
||||
count_media_files_for_user(CountMediaFilesForUserArgs { filters, mysql_executor: &db.pool }).await.unwrap(),
|
||||
6
|
||||
);
|
||||
db.destroy().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "large local fixture; run explicitly with --ignored --nocapture"]
|
||||
async fn user_media_library_large_page_reads_are_bounded() {
|
||||
let db = IsolatedTestDatabase::create().await;
|
||||
let owner = create_test_user(&db.pool).await.unwrap();
|
||||
let other = create_test_user(&db.pool).await.unwrap();
|
||||
// 60k owner rows, followed by 240k newer files from another account.
|
||||
for chunk in 0..300 {
|
||||
let mut query = QueryBuilder::<MySql>::new("INSERT INTO media_files (token, origin_category, origin_product_category, media_type, media_class, checksum_sha2, public_bucket_directory_hash, creator_ip_address, maybe_creator_user_token, creator_set_visibility, is_intermediate_system_file, is_user_upload, created_at) ");
|
||||
query.push_values(0..1000, |mut row, index| {
|
||||
row
|
||||
.push_bind(format!("m_bench_{}", chunk * 1000 + index))
|
||||
.push_bind(MediaFileOriginCategory::Upload.to_str())
|
||||
.push_bind(MediaFileOriginProductCategory::Unknown.to_str())
|
||||
.push_bind(MediaFileType::Png.to_str())
|
||||
.push_bind(MediaFileClass::Image.to_str())
|
||||
.push_bind("0".repeat(64))
|
||||
.push_bind("fixture_hash")
|
||||
.push_bind("127.0.0.1")
|
||||
.push_bind(if chunk < 60 { owner.user_token.as_str() } else { other.user_token.as_str() })
|
||||
.push_bind(Visibility::Private.to_str())
|
||||
.push_bind(false)
|
||||
.push_bind(true)
|
||||
.push_bind(CREATED_AT);
|
||||
});
|
||||
query.build().execute(&db.pool).await.unwrap();
|
||||
}
|
||||
sqlx::query("ANALYZE TABLE media_files").execute(&db.pool).await.unwrap();
|
||||
let classes = HashSet::from([
|
||||
MediaFileClass::Image,
|
||||
MediaFileClass::Video,
|
||||
MediaFileClass::Audio,
|
||||
MediaFileClass::Dimensional,
|
||||
MediaFileClass::Mesh,
|
||||
MediaFileClass::Splat,
|
||||
]);
|
||||
let filters = UserMediaFileFilters { maybe_filter_media_classes: Some(&classes), ..filters(&owner.username) };
|
||||
let mut conn = db.pool.acquire().await.unwrap();
|
||||
let before = scan_reads(&mut conn).await;
|
||||
let start = Instant::now();
|
||||
let records = list_media_files_for_user(ListMediaFileForUserArgs {
|
||||
filters,
|
||||
limit: 61,
|
||||
offset: 0,
|
||||
sort_ascending: false,
|
||||
mysql_executor: &mut *conn,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let elapsed = start.elapsed();
|
||||
let scanned = scan_reads(&mut conn).await - before;
|
||||
assert_eq!(records.len(), 61);
|
||||
assert_eq!(records.first().unwrap().token.as_str(), "m_bench_59999");
|
||||
assert!(scanned < 200, "page query scanned {scanned} sequential index entries");
|
||||
let start = Instant::now();
|
||||
assert_eq!(
|
||||
count_media_files_for_user(CountMediaFilesForUserArgs { filters, mysql_executor: &mut *conn }).await.unwrap(),
|
||||
60_000
|
||||
);
|
||||
println!(
|
||||
"300k fixtures / 60k owned: page={elapsed:?}, sequential index reads={scanned}; optional count={:?}",
|
||||
start.elapsed()
|
||||
);
|
||||
drop(conn);
|
||||
db.destroy().await;
|
||||
}
|
||||
|
||||
struct Fixture<'a> {
|
||||
owner: &'a UserToken,
|
||||
token: &'a str,
|
||||
class: MediaFileClass,
|
||||
kind: MediaFileType,
|
||||
visibility: Visibility,
|
||||
upload: bool,
|
||||
}
|
||||
|
||||
impl<'a> Fixture<'a> {
|
||||
fn image(owner: &'a UserToken, token: &'a str) -> Self {
|
||||
Self {
|
||||
owner,
|
||||
token,
|
||||
class: MediaFileClass::Image,
|
||||
kind: MediaFileType::Png,
|
||||
visibility: Visibility::Public,
|
||||
upload: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filters(username: &str) -> UserMediaFileFilters<'_> {
|
||||
UserMediaFileFilters {
|
||||
username,
|
||||
maybe_filter_media_types: None,
|
||||
maybe_filter_media_classes: None,
|
||||
maybe_filter_engine_categories: None,
|
||||
include_user_uploads: true,
|
||||
view_as: ViewAs::Author,
|
||||
}
|
||||
}
|
||||
|
||||
async fn insert_fixture(pool: &MySqlPool, fixture: Fixture<'_>) {
|
||||
sqlx::query("INSERT INTO media_files (token, origin_category, origin_product_category, media_type, media_class, checksum_sha2, public_bucket_directory_hash, creator_ip_address, maybe_creator_user_token, creator_set_visibility, is_intermediate_system_file, is_user_upload, created_at) VALUES (?, ?, ?, ?, ?, ?, 'fixture_hash', '127.0.0.1', ?, ?, false, ?, ?)")
|
||||
.bind(fixture.token).bind(MediaFileOriginCategory::Upload.to_str()).bind(MediaFileOriginProductCategory::Unknown.to_str())
|
||||
.bind(fixture.kind.to_str()).bind(fixture.class.to_str()).bind("0".repeat(64)).bind(fixture.owner.as_str())
|
||||
.bind(fixture.visibility.to_str()).bind(fixture.upload).bind(CREATED_AT).execute(pool).await.unwrap();
|
||||
}
|
||||
|
||||
async fn scan_reads(conn: &mut sqlx::MySqlConnection) -> u64 {
|
||||
let values: Vec<(String, String)> =
|
||||
sqlx::query_as("SHOW SESSION STATUS WHERE Variable_name IN ('Handler_read_next', 'Handler_read_prev')")
|
||||
.fetch_all(conn)
|
||||
.await
|
||||
.unwrap();
|
||||
values.into_iter().map(|(_, value)| value.parse::<u64>().unwrap()).sum()
|
||||
}
|
||||
@@ -24,7 +24,9 @@ pub mod uploaded_video_notes;
|
||||
pub mod uploaded_videos;
|
||||
pub mod usages;
|
||||
pub mod user_bookmarks;
|
||||
pub mod user_login_challenges;
|
||||
pub mod user_ratings;
|
||||
pub mod user_sessions;
|
||||
pub mod user_spend_events;
|
||||
pub mod users;
|
||||
pub mod voice_conversion_models;
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod user_login_challenge_failure_type;
|
||||
pub mod user_login_challenge_status;
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[cfg(test)]
|
||||
use strum::{EnumCount, EnumIter};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Stored in `user_login_challenges.maybe_failure_type` (VARCHAR(32)).
|
||||
///
|
||||
/// Terminal outcome for a failed device approval request. Transient transport/database errors and invalid credentials are not challenge outcomes.
|
||||
///
|
||||
/// Add new values as needed, but never change stored values without a migration.
|
||||
#[cfg_attr(test, derive(EnumIter, EnumCount))]
|
||||
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
|
||||
pub enum UserLoginChallengeFailureType {
|
||||
/// An authenticated user explicitly declined the request.
|
||||
#[serde(rename = "user_declined")]
|
||||
UserDeclined,
|
||||
|
||||
/// The 20-minute deadline passed before the challenge was redeemed.
|
||||
#[serde(rename = "expired")]
|
||||
Expired,
|
||||
}
|
||||
|
||||
impl_enum_display_and_debug_using_to_str!(UserLoginChallengeFailureType);
|
||||
impl_mysql_enum_coders!(UserLoginChallengeFailureType);
|
||||
impl_mysql_from_row!(UserLoginChallengeFailureType);
|
||||
|
||||
impl UserLoginChallengeFailureType {
|
||||
pub fn to_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UserDeclined => "user_declined",
|
||||
Self::Expired => "expired",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Result<Self, String> {
|
||||
match value {
|
||||
"user_declined" => Ok(Self::UserDeclined),
|
||||
"expired" => Ok(Self::Expired),
|
||||
_ => Err(format!("invalid UserLoginChallengeFailureType value: {:?}", value)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn all_variants() -> BTreeSet<Self> {
|
||||
BTreeSet::from([Self::UserDeclined, Self::Expired])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::UserLoginChallengeFailureType;
|
||||
use crate::test_helpers::assert_serialization;
|
||||
use strum::{EnumCount, IntoEnumIterator};
|
||||
|
||||
const STORED_VALUES: &[(UserLoginChallengeFailureType, &str)] = &[(UserLoginChallengeFailureType::UserDeclined, "user_declined"), (UserLoginChallengeFailureType::Expired, "expired")];
|
||||
const MAX_STORED_LENGTH: usize = 32;
|
||||
|
||||
#[test]
|
||||
fn stored_values_and_serialization_are_stable() {
|
||||
for &(variant, stored) in STORED_VALUES {
|
||||
assert_serialization(variant, stored);
|
||||
assert_eq!(variant.to_str(), stored);
|
||||
assert_eq!(UserLoginChallengeFailureType::from_str(stored).unwrap(), variant);
|
||||
assert_eq!(format!("{}", variant), stored);
|
||||
assert_eq!(format!("{:?}", variant), stored);
|
||||
assert!(!stored.is_empty());
|
||||
assert!(stored.len() <= MAX_STORED_LENGTH);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_variants_are_covered_and_round_trip() {
|
||||
let variants = UserLoginChallengeFailureType::all_variants();
|
||||
assert_eq!(variants.len(), UserLoginChallengeFailureType::COUNT);
|
||||
assert_eq!(variants.len(), STORED_VALUES.len());
|
||||
for variant in UserLoginChallengeFailureType::iter() {
|
||||
assert!(variants.contains(&variant));
|
||||
assert_eq!(UserLoginChallengeFailureType::from_str(variant.to_str()).unwrap(), variant);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_stored_values_are_rejected() {
|
||||
for value in ["", "unknown", "USER_DECLINED"] {
|
||||
assert!(UserLoginChallengeFailureType::from_str(value).is_err());
|
||||
assert!(serde_json::from_str::<UserLoginChallengeFailureType>(&format!("{:?}", value)).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[cfg(test)]
|
||||
use strum::{EnumCount, EnumIter};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Stored in `user_login_challenges.status` (VARCHAR(16)).
|
||||
///
|
||||
/// Persisted state of a device approval request. Always check expires_at independently; a pending or approved row can already be expired.
|
||||
///
|
||||
/// Add new values as needed, but never change stored values without a migration.
|
||||
#[cfg_attr(test, derive(EnumIter, EnumCount))]
|
||||
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
|
||||
pub enum UserLoginChallengeStatus {
|
||||
/// Waiting for an authenticated user's explicit decision.
|
||||
#[serde(rename = "pending")]
|
||||
Pending,
|
||||
|
||||
/// Approved for the deciding user; no session has been minted yet.
|
||||
#[serde(rename = "approved")]
|
||||
Approved,
|
||||
|
||||
/// Exactly one session has been minted. Retries must never mint another.
|
||||
#[serde(rename = "redeemed")]
|
||||
Redeemed,
|
||||
|
||||
/// Terminal failure, described by maybe_failure_type.
|
||||
#[serde(rename = "failed")]
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl_enum_display_and_debug_using_to_str!(UserLoginChallengeStatus);
|
||||
impl_mysql_enum_coders!(UserLoginChallengeStatus);
|
||||
impl_mysql_from_row!(UserLoginChallengeStatus);
|
||||
|
||||
impl UserLoginChallengeStatus {
|
||||
pub fn to_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Pending => "pending",
|
||||
Self::Approved => "approved",
|
||||
Self::Redeemed => "redeemed",
|
||||
Self::Failed => "failed",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Result<Self, String> {
|
||||
match value {
|
||||
"pending" => Ok(Self::Pending),
|
||||
"approved" => Ok(Self::Approved),
|
||||
"redeemed" => Ok(Self::Redeemed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
_ => Err(format!("invalid UserLoginChallengeStatus value: {:?}", value)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn all_variants() -> BTreeSet<Self> {
|
||||
BTreeSet::from([Self::Pending, Self::Approved, Self::Redeemed, Self::Failed])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::UserLoginChallengeStatus;
|
||||
use crate::test_helpers::assert_serialization;
|
||||
use strum::{EnumCount, IntoEnumIterator};
|
||||
|
||||
const STORED_VALUES: &[(UserLoginChallengeStatus, &str)] = &[(UserLoginChallengeStatus::Pending, "pending"), (UserLoginChallengeStatus::Approved, "approved"), (UserLoginChallengeStatus::Redeemed, "redeemed"), (UserLoginChallengeStatus::Failed, "failed")];
|
||||
const MAX_STORED_LENGTH: usize = 16;
|
||||
|
||||
#[test]
|
||||
fn stored_values_and_serialization_are_stable() {
|
||||
for &(variant, stored) in STORED_VALUES {
|
||||
assert_serialization(variant, stored);
|
||||
assert_eq!(variant.to_str(), stored);
|
||||
assert_eq!(UserLoginChallengeStatus::from_str(stored).unwrap(), variant);
|
||||
assert_eq!(format!("{}", variant), stored);
|
||||
assert_eq!(format!("{:?}", variant), stored);
|
||||
assert!(!stored.is_empty());
|
||||
assert!(stored.len() <= MAX_STORED_LENGTH);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_variants_are_covered_and_round_trip() {
|
||||
let variants = UserLoginChallengeStatus::all_variants();
|
||||
assert_eq!(variants.len(), UserLoginChallengeStatus::COUNT);
|
||||
assert_eq!(variants.len(), STORED_VALUES.len());
|
||||
for variant in UserLoginChallengeStatus::iter() {
|
||||
assert!(variants.contains(&variant));
|
||||
assert_eq!(UserLoginChallengeStatus::from_str(variant.to_str()).unwrap(), variant);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_stored_values_are_rejected() {
|
||||
for value in ["", "unknown", "PENDING"] {
|
||||
assert!(UserLoginChallengeStatus::from_str(value).is_err());
|
||||
assert!(serde_json::from_str::<UserLoginChallengeStatus>(&format!("{:?}", value)).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod user_web_session_creation_type;
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[cfg(test)]
|
||||
use strum::{EnumCount, EnumIter};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Stored in `user_sessions.maybe_creation_type` (VARCHAR(32)).
|
||||
///
|
||||
/// How a cookie-based user session was created; NULL in MySQL means unknown. This is provenance, not an authentication policy or an MCP/API session type.
|
||||
///
|
||||
/// Add new values as needed, but never change stored values without a migration.
|
||||
#[cfg_attr(test, derive(EnumIter, EnumCount))]
|
||||
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
|
||||
pub enum UserWebSessionCreationType {
|
||||
/// A normal password or SSO login/signup, including direct desktop password login.
|
||||
#[serde(rename = "direct_login")]
|
||||
DirectLogin,
|
||||
|
||||
/// A new session authorized through a user_login_challenges device approval.
|
||||
#[serde(rename = "device_approval")]
|
||||
DeviceApproval,
|
||||
|
||||
/// An explicitly consented staff impersonation; existing impersonation rules still apply.
|
||||
#[serde(rename = "impersonation")]
|
||||
Impersonation,
|
||||
}
|
||||
|
||||
impl_enum_display_and_debug_using_to_str!(UserWebSessionCreationType);
|
||||
impl_mysql_enum_coders!(UserWebSessionCreationType);
|
||||
impl_mysql_from_row!(UserWebSessionCreationType);
|
||||
|
||||
impl UserWebSessionCreationType {
|
||||
pub fn to_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::DirectLogin => "direct_login",
|
||||
Self::DeviceApproval => "device_approval",
|
||||
Self::Impersonation => "impersonation",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Result<Self, String> {
|
||||
match value {
|
||||
"direct_login" => Ok(Self::DirectLogin),
|
||||
"device_approval" => Ok(Self::DeviceApproval),
|
||||
"impersonation" => Ok(Self::Impersonation),
|
||||
_ => Err(format!("invalid UserWebSessionCreationType value: {:?}", value)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn all_variants() -> BTreeSet<Self> {
|
||||
BTreeSet::from([Self::DirectLogin, Self::DeviceApproval, Self::Impersonation])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::UserWebSessionCreationType;
|
||||
use crate::test_helpers::assert_serialization;
|
||||
use strum::{EnumCount, IntoEnumIterator};
|
||||
|
||||
const STORED_VALUES: &[(UserWebSessionCreationType, &str)] = &[(UserWebSessionCreationType::DirectLogin, "direct_login"), (UserWebSessionCreationType::DeviceApproval, "device_approval"), (UserWebSessionCreationType::Impersonation, "impersonation")];
|
||||
const MAX_STORED_LENGTH: usize = 32;
|
||||
|
||||
#[test]
|
||||
fn stored_values_and_serialization_are_stable() {
|
||||
for &(variant, stored) in STORED_VALUES {
|
||||
assert_serialization(variant, stored);
|
||||
assert_eq!(variant.to_str(), stored);
|
||||
assert_eq!(UserWebSessionCreationType::from_str(stored).unwrap(), variant);
|
||||
assert_eq!(format!("{}", variant), stored);
|
||||
assert_eq!(format!("{:?}", variant), stored);
|
||||
assert!(!stored.is_empty());
|
||||
assert!(stored.len() <= MAX_STORED_LENGTH);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_variants_are_covered_and_round_trip() {
|
||||
let variants = UserWebSessionCreationType::all_variants();
|
||||
assert_eq!(variants.len(), UserWebSessionCreationType::COUNT);
|
||||
assert_eq!(variants.len(), STORED_VALUES.len());
|
||||
for variant in UserWebSessionCreationType::iter() {
|
||||
assert!(variants.contains(&variant));
|
||||
assert_eq!(UserWebSessionCreationType::from_str(variant.to_str()).unwrap(), variant);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_stored_values_are_rejected() {
|
||||
for value in ["", "unknown", "DIRECT_LOGIN"] {
|
||||
assert!(UserWebSessionCreationType::from_str(value).is_err());
|
||||
assert!(serde_json::from_str::<UserWebSessionCreationType>(&format!("{:?}", value)).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ pub(crate) enum TokenPrefix {
|
||||
UserBookmark,
|
||||
UserReferralCode,
|
||||
UserSession,
|
||||
UserLoginChallenge,
|
||||
UserSpendEvent,
|
||||
UserSubscription,
|
||||
VoiceConversionModel,
|
||||
@@ -145,6 +146,7 @@ impl PrefixGenerator for TokenPrefix {
|
||||
Self::UserBookmark => "ub_",
|
||||
Self::UserReferralCode => "urc_",
|
||||
Self::UserSession => "session_",
|
||||
Self::UserLoginChallenge => "login_",
|
||||
Self::UserSpendEvent => "spend_",
|
||||
Self::UserSubscription => "subscription_", // NB: Previously "SUB:"
|
||||
Self::VoiceConversionModel => "vcm_",
|
||||
|
||||
@@ -40,6 +40,7 @@ pub mod user_bookmarks;
|
||||
pub mod user_impersonation_requests;
|
||||
pub mod user_referral_codes;
|
||||
pub mod user_sessions;
|
||||
pub mod user_login_challenges;
|
||||
pub mod user_spend_events;
|
||||
pub mod user_subscriptions;
|
||||
pub mod users;
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::prefixes::TokenPrefix;
|
||||
|
||||
/// Non-secret identifier for a login challenge. This never grants access.
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize, ToSchema)]
|
||||
#[cfg_attr(feature = "database", derive(sqlx::Type))]
|
||||
#[cfg_attr(feature = "database", sqlx(transparent))]
|
||||
pub struct UserLoginChallengeToken(pub String);
|
||||
|
||||
impl_string_token!(UserLoginChallengeToken);
|
||||
impl_mysql_token_from_row!(UserLoginChallengeToken);
|
||||
impl_crockford_generator!(UserLoginChallengeToken, 32usize, TokenPrefix::UserLoginChallenge, CrockfordLower);
|
||||
@@ -39,6 +39,8 @@ server_environment = { path = "../../../lib/server_environment" }
|
||||
# External
|
||||
actix-web.workspace = true
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
futures.workspace = true
|
||||
log.workspace = true
|
||||
serde.workspace = true
|
||||
serde_derive.workspace = true
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# Video thumbnail worker
|
||||
|
||||
## Scheduling and resource settings
|
||||
|
||||
| Environment variable | Default | Meaning |
|
||||
| ------------------------------------------ | ------- | -------------------------------------------- |
|
||||
| VIDEO_THUMBNAIL_CONCURRENCY | 1 | Maximum simultaneous complete file attempts |
|
||||
| VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS | 5000 | Idle polling interval |
|
||||
| VIDEO_THUMBNAIL_RETRY_CACHE_CAPACITY | 10000 | Maximum cached failed media IDs |
|
||||
| VIDEO_THUMBNAIL_RETRY_INITIAL_DELAY_MILLIS | 30000 | Backoff after the first failed attempt |
|
||||
| VIDEO_THUMBNAIL_RETRY_MAX_DELAY_MILLIS | 1800000 | Maximum backoff (configurable up to one day) |
|
||||
|
||||
Failures back off for 30 seconds, 60 seconds, 120 seconds, and so on, capped at
|
||||
30 minutes by default. Successful retries clear their failure history. The
|
||||
cache retains only IDs, attempt counts, and monotonic timestamps. At capacity,
|
||||
the least recently failed entry is evicted. Restarts and eviction reset that
|
||||
entry's history; no failure state is persisted in the database. Invalid files
|
||||
are periodically retried rather than permanently marked failed.
|
||||
|
||||
Files in backoff are skipped, and pagination advances even across pages made
|
||||
entirely of skipped files. Within each page, untried files precede due retries.
|
||||
The worker starts at most the configured number of attempts, admitting another
|
||||
as soon as a slot is free. The concurrency limit covers downloads, processing,
|
||||
uploads, and the database update. Raising it does not coordinate other pods;
|
||||
multiple pods still require distinct shard assignments.
|
||||
|
||||
FFmpeg runs on Tokio's blocking pool, one process at a time per file. Each
|
||||
process uses one codec thread and one filter thread, and is killed and reaped
|
||||
after 60 seconds if it has not finished. Stderr is spooled to a temporary file;
|
||||
only its final 16 KiB is read into memory on failure. GIF previews fit within
|
||||
360x360 and contain at most 50 frames (five seconds at ten frames per second).
|
||||
Portrait GIFs are consequently smaller than the former 360px-wide previews.
|
||||
|
||||
These controls reduce memory use but do not guarantee an OOM-free decode:
|
||||
source resolution, codec reference frames, and multiple concurrent files still
|
||||
matter. Keep concurrency at one initially. With the previously observed
|
||||
approximately 560 MiB container peak, two jobs are not proven safe under a 1 GiB
|
||||
memory limit. Measure representative large inputs before raising concurrency.
|
||||
|
||||
On cgroup v2, `thumbnail_memory` logs container `current_bytes`, `peak_bytes`,
|
||||
and `limit_bytes` every ten seconds, including FFmpeg children and charged file
|
||||
cache. Usage at or above 80% of the limit is logged as a warning. The peak is a
|
||||
container lifetime high-water mark, not a per-file allocation. Memory reporting
|
||||
is advisory; it does not pause admission or replace the container memory limit.
|
||||
Unavailable metrics (including on macOS) are reported without blocking work.
|
||||
|
||||
## Timing
|
||||
|
||||
The worker emits `thumbnail_timing` followed by a JSON object at the start and
|
||||
end of every attempt. Filter by `media_file_token` to correlate events.
|
||||
|
||||
- `event=started`: before downloading the source video.
|
||||
- `event=completed`: JPG and GIF uploaded and the thumbnail version updated in
|
||||
the database, so the thumbnails are available to API readers.
|
||||
- `event=failed`: an attempt ended unsuccessfully; exclude this from successful
|
||||
thumbnail latency percentiles.
|
||||
|
||||
`since_generation_completed_ms` measures time from
|
||||
`generic_inference_jobs.successfully_completed_at` to the event. The source job
|
||||
is found through `media_files.maybe_source_job_token`. This timestamp records
|
||||
our backend finalizing the generation job after ingesting its output. It does
|
||||
not measure when the external provider originally finished generating video,
|
||||
nor is it an exact transaction commit timestamp.
|
||||
|
||||
For uploads, older unlinked media, or a job whose completion has not yet been
|
||||
written when the page is fetched, `generation_completed_at` and
|
||||
`since_generation_completed_ms` are JSON `null`. They are never replaced with
|
||||
media creation time. `since_media_created_ms` is reported separately for all
|
||||
files. `media_files.updated_at` is unsuitable because subsequent edits and the
|
||||
thumbnail update change it.
|
||||
|
||||
Additional measurements:
|
||||
|
||||
- `processing_elapsed_ms`: monotonic time spent on this attempt; zero at start.
|
||||
- `page_wait_ms`: time from fetching the page until this file starts, including
|
||||
waiting for earlier files in the same page.
|
||||
- `database_query_ms`: total time spent fetching the page, including pool wait.
|
||||
- `thumbnail_stage`: JSON events with `stage`, `success`, and `duration_ms` for
|
||||
download, JPG generation/upload, GIF generation/upload, and database update.
|
||||
- `thumbnail_page`: fetched row count and query duration, including empty pages.
|
||||
|
||||
## Clock handling
|
||||
|
||||
The listing query returns `NOW(6)` as `database_read_at` with each row. Both that
|
||||
timestamp and the generation completion timestamp come from the database.
|
||||
Worker wall-clock time is not used to calculate latency.
|
||||
|
||||
MySQL evaluates `NOW(6)` at statement start. We bracket the query with two Rust
|
||||
`Instant`s and use their midpoint as the estimated local instant corresponding
|
||||
to `database_read_at`. Every event advances this database time using monotonic
|
||||
elapsed time, including time queued behind other files in the page.
|
||||
|
||||
This estimate has up to `clock_uncertainty_ms` of error in either direction
|
||||
(half the query duration, rounded up), plus the source timestamps' existing
|
||||
one-second precision. For example, a 20 ms query contributes up to 10 ms of
|
||||
clock-mapping uncertainty. A slow query reports a larger uncertainty; do not
|
||||
interpret millisecond-valued fields as millisecond accuracy. Negative values
|
||||
are preserved to expose timestamp anomalies. No extra per-file clock query is
|
||||
needed.
|
||||
|
||||
These measurements end at backend thumbnail availability. Client polling,
|
||||
caching, and UI refresh delays happen afterward and need separate measurement.
|
||||
@@ -5,3 +5,7 @@ SERVER_ENVIRONMENT='development'
|
||||
|
||||
# Don't collide with storyteller-web in development
|
||||
HTTP_BIND_ADDRESS=0.0.0.0:13339
|
||||
|
||||
# Limit whole thumbnail attempts, including FFmpeg and downloads.
|
||||
VIDEO_THUMBNAIL_CONCURRENCY=1
|
||||
VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS=5000
|
||||
@@ -1,19 +1,30 @@
|
||||
use std::time::Duration;
|
||||
use std::future::Future;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::{stream, StreamExt};
|
||||
use log::{error, info, warn};
|
||||
|
||||
use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::{
|
||||
list_video_media_files_without_thumbnails_for_job,
|
||||
ListVideoMediaFilesWithoutThumbnailsArgs,
|
||||
VideoMediaFileWithoutThumbnail,
|
||||
};
|
||||
|
||||
use crate::job::alert_on_error::alert_pager_and_return_err;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::job::process_single_media_file::process_single_media_file;
|
||||
use crate::job::thumbnail_timing::ThumbnailTiming;
|
||||
use crate::job::memory_monitor::monitor_memory;
|
||||
use crate::job::retry_backoff::RetryBackoff;
|
||||
|
||||
pub async fn main_loop(deps: JobDependencies) {
|
||||
let config = &deps.worker_config;
|
||||
let mut retry_cache = RetryBackoff::new(config.retry_cache_capacity, config.retry_initial_delay, config.retry_max_delay);
|
||||
let memory_monitor = tokio::spawn(monitor_memory(config.concurrency.get()));
|
||||
|
||||
while !deps.application_shutdown.get() {
|
||||
let processed_any = match run_batch_cycle(&deps).await {
|
||||
let processed_any = match run_batch_cycle(&deps, &mut retry_cache).await {
|
||||
Ok(count) => {
|
||||
if count > 0 {
|
||||
info!("Processed {} video thumbnail(s) this cycle.", count);
|
||||
@@ -37,12 +48,13 @@ pub async fn main_loop(deps: JobDependencies) {
|
||||
}
|
||||
}
|
||||
|
||||
memory_monitor.abort();
|
||||
warn!("Video thumbnail job main loop is shut down.");
|
||||
}
|
||||
|
||||
/// Run one full pagination cycle: keep querying pages of media files without thumbnails
|
||||
/// until there are no more results. Returns the total number of items processed.
|
||||
async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
|
||||
async fn run_batch_cycle(deps: &JobDependencies, retry_cache: &mut RetryBackoff) -> anyhow::Result<u64> {
|
||||
let mut maybe_cursor: Option<i64> = None;
|
||||
let mut total_processed: u64 = 0;
|
||||
|
||||
@@ -51,6 +63,7 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
|
||||
break;
|
||||
}
|
||||
|
||||
let query_started_at = Instant::now();
|
||||
let result = list_video_media_files_without_thumbnails_for_job(
|
||||
ListVideoMediaFilesWithoutThumbnailsArgs {
|
||||
custom_max_lookback_hours: deps.custom_max_lookback_hours,
|
||||
@@ -59,41 +72,43 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
|
||||
executor: &deps.mysql_pool,
|
||||
},
|
||||
).await?;
|
||||
let query_finished_at = Instant::now();
|
||||
|
||||
info!(
|
||||
"thumbnail_page fetched_count={} database_query_ms={}",
|
||||
result.media_files.len(),
|
||||
query_finished_at.duration_since(query_started_at).as_millis(),
|
||||
);
|
||||
|
||||
if result.media_files.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for media_file in &result.media_files {
|
||||
if deps.application_shutdown.get() {
|
||||
break;
|
||||
}
|
||||
|
||||
// If sharding is enabled, skip media files that don't belong to this shard.
|
||||
if let Some(shard_info) = &deps.shard_info {
|
||||
let modulus = (media_file.id as u64) % (shard_info.number_of_shards as u64);
|
||||
if modulus != shard_info.shard_index as u64 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match process_single_media_file(deps, media_file).await {
|
||||
Ok(()) => {
|
||||
let _ = deps.job_stats.increment_success_count();
|
||||
total_processed += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Failed to generate thumbnail for media file {}: {:?}",
|
||||
media_file.token.as_str(),
|
||||
err,
|
||||
);
|
||||
let _ = deps.job_stats.increment_failure_count();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Advance using the database page, even when every row is in backoff or
|
||||
// belongs to another shard. Otherwise failed files could hide older good files.
|
||||
maybe_cursor = result.next_cursor;
|
||||
let media_files = result.media_files.into_iter().filter(|media_file| {
|
||||
deps.shard_info.as_ref().is_none_or(|shard| {
|
||||
(media_file.id as u64) % u64::from(shard.number_of_shards) == u64::from(shard.shard_index)
|
||||
})
|
||||
}).collect();
|
||||
|
||||
total_processed += process_page(media_files, deps.worker_config.concurrency, retry_cache, |media_file| async move {
|
||||
// Already-running attempts are drained on shutdown; no new attempts start.
|
||||
if deps.application_shutdown.get() {
|
||||
return None;
|
||||
}
|
||||
let timing = ThumbnailTiming::start(&media_file, query_started_at, query_finished_at);
|
||||
let result = process_single_media_file(deps, &media_file, &timing).await;
|
||||
timing.finish(result.is_ok());
|
||||
if result.is_ok() {
|
||||
let _ = deps.job_stats.increment_success_count();
|
||||
} else {
|
||||
let _ = deps.job_stats.increment_failure_count();
|
||||
}
|
||||
Some(result)
|
||||
}).await;
|
||||
|
||||
if maybe_cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
@@ -104,3 +119,141 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
|
||||
|
||||
Ok(total_processed)
|
||||
}
|
||||
|
||||
/// Only keep `concurrency` attempts in flight, including their download, FFmpeg,
|
||||
/// and upload work. Complete one before admitting another; never spawn a page of
|
||||
/// tasks that merely wait for a semaphore while retaining downloaded videos.
|
||||
async fn process_page<F, Fut>(
|
||||
mut media_files: Vec<VideoMediaFileWithoutThumbnail>,
|
||||
concurrency: NonZeroUsize,
|
||||
retry_cache: &mut RetryBackoff,
|
||||
process: F,
|
||||
) -> u64
|
||||
where
|
||||
F: Fn(VideoMediaFileWithoutThumbnail) -> Fut,
|
||||
Fut: Future<Output = Option<anyhow::Result<()>>>,
|
||||
{
|
||||
let fetched_count = media_files.len();
|
||||
let now = Instant::now();
|
||||
media_files.retain(|file| retry_cache.is_ready(file.id, now));
|
||||
// Keep database ordering within each group, but let untried files precede retries.
|
||||
media_files.sort_by_key(|file| retry_cache.has_failed(file.id));
|
||||
info!(
|
||||
"thumbnail_dispatch eligible_count={} backoff_skipped_count={} concurrency={}",
|
||||
media_files.len(), fetched_count - media_files.len(), concurrency,
|
||||
);
|
||||
|
||||
let mut pending = stream::iter(media_files).map(|file| {
|
||||
let id = file.id;
|
||||
let token = file.token.clone();
|
||||
let future = process(file);
|
||||
async move { (id, token, future.await) }
|
||||
}).buffer_unordered(concurrency.get());
|
||||
let mut completed = 0;
|
||||
while let Some((id, token, maybe_result)) = pending.next().await {
|
||||
match maybe_result {
|
||||
Some(Ok(())) => {
|
||||
retry_cache.record_success(id);
|
||||
completed += 1;
|
||||
}
|
||||
Some(Err(err)) => {
|
||||
let retry = retry_cache.record_failure(id, Instant::now());
|
||||
warn!(
|
||||
"thumbnail_retry media_file_token={} failed_attempts={} retry_delay_ms={} error={:?}",
|
||||
token.as_str(), retry.attempts, retry.delay.as_millis(), err,
|
||||
);
|
||||
}
|
||||
None => {}, // Shutdown: this file was not attempted.
|
||||
}
|
||||
}
|
||||
completed
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use chrono::DateTime;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
mod dispatch {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn fills_available_slots_without_exceeding_configured_concurrency() {
|
||||
for concurrency in 1..=4 {
|
||||
let active = AtomicUsize::new(0);
|
||||
let peak = AtomicUsize::new(0);
|
||||
let mut cache = retry_cache();
|
||||
let completed = process_page((1..=20).map(media_file).collect(), limit(concurrency), &mut cache, |_| async {
|
||||
let count = active.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
peak.fetch_max(count, Ordering::SeqCst);
|
||||
tokio::task::yield_now().await;
|
||||
active.fetch_sub(1, Ordering::SeqCst);
|
||||
Some(Ok(()))
|
||||
}).await;
|
||||
assert_eq!(completed, 20);
|
||||
assert_eq!(peak.load(Ordering::SeqCst), concurrency);
|
||||
assert_eq!(active.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_failure_page_does_not_prevent_processing_a_later_good_page() {
|
||||
let mut cache = retry_cache();
|
||||
for id in 1..=100 {
|
||||
cache.record_failure(id, Instant::now());
|
||||
}
|
||||
let completed = process_page((1..=100).map(media_file).collect(), limit(2), &mut cache, |_| async {
|
||||
panic!("A file in backoff must not run");
|
||||
}).await;
|
||||
assert_eq!(completed, 0);
|
||||
let completed = process_page(vec![media_file(101)], limit(2), &mut cache, |_| async { Some(Ok(())) }).await;
|
||||
assert_eq!(completed, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn untried_files_precede_due_retries_and_failure_does_not_stop_progress() {
|
||||
let mut cache = retry_cache();
|
||||
cache.record_failure(1, Instant::now() - Duration::from_secs(60));
|
||||
let order = Mutex::new(Vec::new());
|
||||
let completed = process_page(vec![media_file(1), media_file(2)], limit(1), &mut cache, |file| {
|
||||
let order = ℴ
|
||||
async move {
|
||||
order.lock().unwrap().push(file.id);
|
||||
Some(if file.id == 1 { Err(anyhow::anyhow!("invalid video")) } else { Ok(()) })
|
||||
}
|
||||
}).await;
|
||||
assert_eq!(completed, 1);
|
||||
assert_eq!(*order.lock().unwrap(), vec![2, 1]);
|
||||
assert!(!cache.is_ready(1, Instant::now()));
|
||||
assert!(!cache.has_failed(2));
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_cache() -> RetryBackoff {
|
||||
RetryBackoff::new(limit(1000), Duration::from_secs(30), Duration::from_secs(1800))
|
||||
}
|
||||
|
||||
fn limit(value: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(value).unwrap()
|
||||
}
|
||||
|
||||
fn media_file(id: i64) -> VideoMediaFileWithoutThumbnail {
|
||||
let now = DateTime::from_timestamp(1_000_000, 0).unwrap();
|
||||
VideoMediaFileWithoutThumbnail {
|
||||
id,
|
||||
token: serde_json::from_value(json!(format!("m_test_{id}"))).unwrap(),
|
||||
created_at: now,
|
||||
database_read_at: now,
|
||||
maybe_generation_completed_at: Some(now),
|
||||
maybe_thumbnail_version: None,
|
||||
public_bucket_directory_hash: String::new(),
|
||||
maybe_public_bucket_prefix: None,
|
||||
maybe_public_bucket_extension: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use log::{info, warn};
|
||||
use serde_json::json;
|
||||
use tokio::fs;
|
||||
|
||||
struct MemoryUsage {
|
||||
current_bytes: u64,
|
||||
maybe_peak_bytes: Option<u64>,
|
||||
maybe_limit_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// cgroup v2 accounts for the Rust worker, FFmpeg children, and charged file cache.
|
||||
pub async fn monitor_memory(concurrency: usize) {
|
||||
let root = Path::new("/sys/fs/cgroup");
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(10));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let Some(usage) = read_memory_usage(root).await else {
|
||||
info!("Thumbnail container memory measurements unavailable (requires readable cgroup v2 memory files)");
|
||||
return;
|
||||
};
|
||||
let fields = json!({
|
||||
"configured_concurrency": concurrency,
|
||||
"current_bytes": usage.current_bytes,
|
||||
"peak_bytes": usage.maybe_peak_bytes,
|
||||
"limit_bytes": usage.maybe_limit_bytes,
|
||||
});
|
||||
if usage.maybe_limit_bytes.is_some_and(|limit| usage.current_bytes >= limit.saturating_mul(4) / 5) {
|
||||
warn!("thumbnail_memory {}", fields);
|
||||
} else {
|
||||
info!("thumbnail_memory {}", fields);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_memory_usage(root: &Path) -> Option<MemoryUsage> {
|
||||
let current_bytes = read_number(&root.join("memory.current")).await?;
|
||||
let maybe_peak_bytes = read_number(&root.join("memory.peak")).await;
|
||||
let maybe_limit_bytes = read_number(&root.join("memory.max")).await;
|
||||
Some(MemoryUsage { current_bytes, maybe_peak_bytes, maybe_limit_bytes })
|
||||
}
|
||||
|
||||
async fn read_number(path: &Path) -> Option<u64> {
|
||||
// "max" denotes no cgroup memory limit; old kernels may lack memory.peak.
|
||||
fs::read_to_string(path).await.ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tempdir::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reads_container_usage_and_handles_unlimited_or_missing_optional_metrics() {
|
||||
let directory = TempDir::new("thumbnail_memory_test").unwrap();
|
||||
let root = directory.path();
|
||||
fs::write(root.join("memory.current"), "1234\n").await.unwrap();
|
||||
fs::write(root.join("memory.max"), "max\n").await.unwrap();
|
||||
let usage = read_memory_usage(root).await.unwrap();
|
||||
assert_eq!(usage.current_bytes, 1234);
|
||||
assert_eq!(usage.maybe_peak_bytes, None);
|
||||
assert_eq!(usage.maybe_limit_bytes, None);
|
||||
|
||||
fs::write(root.join("memory.peak"), "2345\n").await.unwrap();
|
||||
fs::write(root.join("memory.max"), "4096\n").await.unwrap();
|
||||
let usage = read_memory_usage(root).await.unwrap();
|
||||
assert_eq!(usage.maybe_peak_bytes, Some(2345));
|
||||
assert_eq!(usage.maybe_limit_bytes, Some(4096));
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
pub mod alert_on_error;
|
||||
pub mod main_loop;
|
||||
pub mod process_single_media_file;
|
||||
pub mod thumbnail_timing;
|
||||
pub mod retry_backoff;
|
||||
pub mod memory_monitor;
|
||||
@@ -1,4 +1,6 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use log::{error, info};
|
||||
use tempdir::TempDir;
|
||||
@@ -11,6 +13,7 @@ use mysql_queries::queries::media_files::thumbnails::list_video_media_files_with
|
||||
use mysql_queries::queries::media_files::thumbnails::update_video_media_file_with_thumbnail::update_video_media_file_with_thumbnail;
|
||||
|
||||
use crate::job::alert_on_error::alert_pager_and_return_err;
|
||||
use crate::job::thumbnail_timing::ThumbnailTiming;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
|
||||
/// A downloaded video file alongside its owning temp directory.
|
||||
@@ -24,6 +27,7 @@ pub struct DownloadedFile {
|
||||
pub async fn process_single_media_file(
|
||||
deps: &JobDependencies,
|
||||
media_file: &VideoMediaFileWithoutThumbnail,
|
||||
timing: &ThumbnailTiming<'_>,
|
||||
) -> anyhow::Result<()> {
|
||||
|
||||
info!(
|
||||
@@ -33,8 +37,11 @@ pub async fn process_single_media_file(
|
||||
media_file.created_at,
|
||||
);
|
||||
|
||||
let downloaded = match download_video(deps, media_file).await {
|
||||
Ok(d) => d,
|
||||
let stage_started_at = Instant::now();
|
||||
let result = download_video(deps, media_file).await;
|
||||
timing.log_stage("download", stage_started_at, result.is_ok());
|
||||
let downloaded = match result {
|
||||
Ok(d) => Arc::new(d),
|
||||
Err(err) => {
|
||||
error!("Failed to download video for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "Video download failed", err);
|
||||
@@ -53,12 +60,19 @@ pub async fn process_single_media_file(
|
||||
// Generate jpg thumbnail
|
||||
let jpg_path = downloaded.temp_dir.path().join("thumbnail.jpg");
|
||||
|
||||
if let Err(err) = ffmpeg_video_first_frame_to_jpg_thumbnail(
|
||||
FfmpegVideoFirstFrameToJpgThumbnailArgs {
|
||||
input_video_path: &downloaded.file_path,
|
||||
output_jpg_path: &jpg_path,
|
||||
},
|
||||
) {
|
||||
let stage_started_at = Instant::now();
|
||||
// Retain the owning temp directory inside the blocking task, even if the
|
||||
// async caller is cancelled. Await the task before releasing this job's slot.
|
||||
let source = Arc::clone(&downloaded);
|
||||
let destination = jpg_path.clone();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
ffmpeg_video_first_frame_to_jpg_thumbnail(FfmpegVideoFirstFrameToJpgThumbnailArgs {
|
||||
input_video_path: &source.file_path,
|
||||
output_jpg_path: &destination,
|
||||
})
|
||||
}).await.map_err(anyhow::Error::from).and_then(|result| result);
|
||||
timing.log_stage("jpg_generation", stage_started_at, result.is_ok());
|
||||
if let Err(err) = result {
|
||||
error!("Failed to generate JPG thumbnail for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "JPG thumbnail generation failed", err.into());
|
||||
}
|
||||
@@ -67,7 +81,10 @@ pub async fn process_single_media_file(
|
||||
|
||||
let jpg_object_path = format!("{video_object_path}{VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}");
|
||||
|
||||
if let Err(err) = deps.public_bucket_client.upload_filename(&jpg_object_path, &jpg_path).await {
|
||||
let stage_started_at = Instant::now();
|
||||
let result = deps.public_bucket_client.upload_filename(&jpg_object_path, &jpg_path).await;
|
||||
timing.log_stage("jpg_upload", stage_started_at, result.is_ok());
|
||||
if let Err(err) = result {
|
||||
error!("Failed to upload JPG thumbnail for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "JPG thumbnail upload failed", err);
|
||||
}
|
||||
@@ -77,12 +94,17 @@ pub async fn process_single_media_file(
|
||||
// Generate gif thumbnail
|
||||
let gif_path = downloaded.temp_dir.path().join("thumbnail.gif");
|
||||
|
||||
if let Err(err) = ffmpeg_video_gif_preview(
|
||||
FfmpegVideoGifPreviewArgs {
|
||||
input_video_path: &downloaded.file_path,
|
||||
output_gif_path: &gif_path,
|
||||
},
|
||||
) {
|
||||
let stage_started_at = Instant::now();
|
||||
let source = Arc::clone(&downloaded);
|
||||
let destination = gif_path.clone();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
ffmpeg_video_gif_preview(FfmpegVideoGifPreviewArgs {
|
||||
input_video_path: &source.file_path,
|
||||
output_gif_path: &destination,
|
||||
})
|
||||
}).await.map_err(anyhow::Error::from).and_then(|result| result);
|
||||
timing.log_stage("gif_generation", stage_started_at, result.is_ok());
|
||||
if let Err(err) = result {
|
||||
error!("Failed to generate GIF preview for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "GIF preview generation failed", err.into());
|
||||
}
|
||||
@@ -91,7 +113,10 @@ pub async fn process_single_media_file(
|
||||
|
||||
let gif_object_path = format!("{video_object_path}{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX}");
|
||||
|
||||
if let Err(err) = deps.public_bucket_client.upload_filename(&gif_object_path, &gif_path).await {
|
||||
let stage_started_at = Instant::now();
|
||||
let result = deps.public_bucket_client.upload_filename(&gif_object_path, &gif_path).await;
|
||||
timing.log_stage("gif_upload", stage_started_at, result.is_ok());
|
||||
if let Err(err) = result {
|
||||
error!("Failed to upload GIF preview for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "GIF preview upload failed", err);
|
||||
}
|
||||
@@ -101,11 +126,14 @@ pub async fn process_single_media_file(
|
||||
info!("Marking thumbnail job for {:?} done", media_file.token);
|
||||
|
||||
// Mark the media file as having a thumbnail in the database.
|
||||
if let Err(err) = update_video_media_file_with_thumbnail(
|
||||
let stage_started_at = Instant::now();
|
||||
let result = update_video_media_file_with_thumbnail(
|
||||
&media_file.token,
|
||||
CURRENT_VIDEO_THUMBNAIL_VERSION,
|
||||
&deps.mysql_pool,
|
||||
).await {
|
||||
).await;
|
||||
timing.log_stage("database_update", stage_started_at, result.is_ok());
|
||||
if let Err(err) = result {
|
||||
error!("Failed to update thumbnail version for {}: {:?}", media_file.token.as_str(), err);
|
||||
return alert_pager_and_return_err(&deps.pager, "Thumbnail DB update failed", err.into());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Process-local failure history. Memory is bounded; restarts/eviction forget failures.
|
||||
pub struct RetryBackoff {
|
||||
failures: HashMap<i64, Failure>,
|
||||
capacity: NonZeroUsize,
|
||||
initial_delay: Duration,
|
||||
max_delay: Duration,
|
||||
}
|
||||
|
||||
struct Failure {
|
||||
attempts: u32,
|
||||
last_failed_at: Instant,
|
||||
retry_at: Instant,
|
||||
}
|
||||
|
||||
pub struct RetrySchedule {
|
||||
pub attempts: u32,
|
||||
pub delay: Duration,
|
||||
}
|
||||
|
||||
impl RetryBackoff {
|
||||
pub fn new(capacity: NonZeroUsize, initial_delay: Duration, max_delay: Duration) -> Self {
|
||||
Self { failures: HashMap::new(), capacity, initial_delay, max_delay }
|
||||
}
|
||||
|
||||
pub fn is_ready(&self, id: i64, now: Instant) -> bool {
|
||||
self.failures.get(&id).is_none_or(|failure| now >= failure.retry_at)
|
||||
}
|
||||
|
||||
pub fn has_failed(&self, id: i64) -> bool {
|
||||
self.failures.contains_key(&id)
|
||||
}
|
||||
|
||||
pub fn record_success(&mut self, id: i64) {
|
||||
self.failures.remove(&id);
|
||||
}
|
||||
|
||||
pub fn record_failure(&mut self, id: i64, now: Instant) -> RetrySchedule {
|
||||
if !self.failures.contains_key(&id) && self.failures.len() == self.capacity.get() {
|
||||
// Evict the least recently failed entry. Skipping a cached failure must not
|
||||
// refresh its position or allocate an unbounded auxiliary eviction queue.
|
||||
if let Some(oldest_id) = self.failures.iter()
|
||||
.min_by_key(|(_, failure)| failure.last_failed_at)
|
||||
.map(|(id, _)| *id)
|
||||
{
|
||||
self.failures.remove(&oldest_id);
|
||||
}
|
||||
}
|
||||
|
||||
let attempts = self.failures.get(&id)
|
||||
.map(|failure| failure.attempts.saturating_add(1))
|
||||
.unwrap_or(1);
|
||||
let multiplier = 1u32 << attempts.saturating_sub(1).min(31);
|
||||
let delay = self.initial_delay.saturating_mul(multiplier).min(self.max_delay);
|
||||
self.failures.insert(id, Failure { attempts, last_failed_at: now, retry_at: now + delay });
|
||||
|
||||
RetrySchedule { attempts, delay }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn retries_at_the_deadline_with_exponential_delay_and_a_cap() {
|
||||
let mut cache = cache(10);
|
||||
let mut now = Instant::now();
|
||||
for (index, seconds) in [30, 60, 120, 120, 120].into_iter().enumerate() {
|
||||
let retry = cache.record_failure(1, now);
|
||||
assert_eq!(retry.attempts, index as u32 + 1);
|
||||
assert_eq!(retry.delay, Duration::from_secs(seconds));
|
||||
assert!(!cache.is_ready(1, now + retry.delay - Duration::from_millis(1)));
|
||||
now += retry.delay;
|
||||
assert!(cache.is_ready(1, now));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_forgets_failures_and_resets_the_backoff() {
|
||||
let mut cache = cache(10);
|
||||
let now = Instant::now();
|
||||
cache.record_failure(1, now);
|
||||
cache.record_failure(1, now);
|
||||
cache.record_success(1);
|
||||
assert!(cache.is_ready(1, now));
|
||||
assert!(!cache.has_failed(1));
|
||||
assert_eq!(cache.record_failure(1, now).attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capacity_is_bounded_and_evicts_the_least_recent_failure() {
|
||||
let mut cache = cache(2);
|
||||
let now = Instant::now();
|
||||
cache.record_failure(1, now);
|
||||
cache.record_failure(2, now + Duration::from_secs(1));
|
||||
cache.record_failure(1, now + Duration::from_secs(2));
|
||||
cache.record_failure(3, now + Duration::from_secs(3));
|
||||
assert_eq!(cache.failures.len(), 2);
|
||||
assert!(cache.has_failed(1));
|
||||
assert!(!cache.has_failed(2));
|
||||
assert!(cache.has_failed(3));
|
||||
for id in 4..1000 {
|
||||
cache.record_failure(id, now + Duration::from_secs(id as u64));
|
||||
assert_eq!(cache.failures.len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
fn cache(capacity: usize) -> RetryBackoff {
|
||||
RetryBackoff::new(NonZeroUsize::new(capacity).unwrap(), Duration::from_secs(30), Duration::from_secs(120))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use log::info;
|
||||
use serde_json::json;
|
||||
|
||||
use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::VideoMediaFileWithoutThumbnail;
|
||||
|
||||
/// Measures the complete thumbnail attempt: download through uploads and DB update.
|
||||
pub struct ThumbnailTiming<'a> {
|
||||
media_file: &'a VideoMediaFileWithoutThumbnail,
|
||||
clock: DatabaseClock,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
struct DatabaseClock {
|
||||
database_read_at: DateTime<Utc>,
|
||||
query_started_at: Instant,
|
||||
query_finished_at: Instant,
|
||||
}
|
||||
|
||||
struct LatencySnapshot {
|
||||
estimated_database_at: DateTime<Utc>,
|
||||
maybe_since_generation_completed_ms: Option<i64>,
|
||||
since_media_created_ms: i64,
|
||||
}
|
||||
|
||||
impl<'a> ThumbnailTiming<'a> {
|
||||
pub fn start(
|
||||
media_file: &'a VideoMediaFileWithoutThumbnail,
|
||||
query_started_at: Instant,
|
||||
query_finished_at: Instant,
|
||||
) -> Self {
|
||||
let timing = Self {
|
||||
media_file,
|
||||
clock: DatabaseClock { database_read_at: media_file.database_read_at, query_started_at, query_finished_at },
|
||||
started_at: Instant::now(),
|
||||
};
|
||||
|
||||
timing.log_event("started", timing.started_at);
|
||||
timing
|
||||
}
|
||||
|
||||
pub fn finish(&self, success: bool) {
|
||||
self.log_event(if success { "completed" } else { "failed" }, Instant::now());
|
||||
}
|
||||
|
||||
pub fn log_stage(&self, stage: &str, started_at: Instant, success: bool) {
|
||||
info!(
|
||||
"thumbnail_stage {}",
|
||||
json!({
|
||||
"media_file_token": self.media_file.token.as_str(),
|
||||
"stage": stage,
|
||||
"success": success,
|
||||
"duration_ms": started_at.elapsed().as_millis(),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
fn log_event(&self, event: &str, now: Instant) {
|
||||
let snapshot =
|
||||
self.clock.snapshot_at(now, self.media_file.created_at, self.media_file.maybe_generation_completed_at);
|
||||
let query_duration = self.clock.query_finished_at.duration_since(self.clock.query_started_at);
|
||||
|
||||
info!(
|
||||
"thumbnail_timing {}",
|
||||
json!({
|
||||
"event": event,
|
||||
"media_file_token": self.media_file.token.as_str(),
|
||||
"media_created_at": self.media_file.created_at,
|
||||
"generation_completed_at": self.media_file.maybe_generation_completed_at,
|
||||
"database_read_at": self.clock.database_read_at,
|
||||
"estimated_database_at": snapshot.estimated_database_at,
|
||||
"since_generation_completed_ms": snapshot.maybe_since_generation_completed_ms,
|
||||
"since_media_created_ms": snapshot.since_media_created_ms,
|
||||
"processing_elapsed_ms": now.duration_since(self.started_at).as_millis(),
|
||||
"page_wait_ms": self.started_at.duration_since(self.clock.query_finished_at).as_millis(),
|
||||
"database_query_ms": query_duration.as_millis(),
|
||||
"clock_uncertainty_ms": query_duration.as_nanos().div_ceil(2_000_000),
|
||||
"source_timestamp_precision_ms": 1000,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl DatabaseClock {
|
||||
fn snapshot_at(
|
||||
&self,
|
||||
now: Instant,
|
||||
media_created_at: DateTime<Utc>,
|
||||
maybe_generation_completed_at: Option<DateTime<Utc>>,
|
||||
) -> LatencySnapshot {
|
||||
// NOW(6) is sampled at statement start, somewhere between these two Instants.
|
||||
// Use the midpoint as an estimate with +/- half the query duration uncertainty.
|
||||
// This includes pool wait, execution, and network time; anchoring NOW() at the
|
||||
// response instead would systematically omit query time from every latency.
|
||||
// The completion/creation columns themselves have only whole-second precision.
|
||||
let query_duration = self.query_finished_at.duration_since(self.query_started_at);
|
||||
let midpoint = self.query_started_at + query_duration / 2;
|
||||
let elapsed = TimeDelta::from_std(now.duration_since(midpoint))
|
||||
.expect("thumbnail measurement duration fits in chrono::TimeDelta");
|
||||
let estimated_database_at = self.database_read_at + elapsed;
|
||||
|
||||
LatencySnapshot {
|
||||
estimated_database_at,
|
||||
// Preserve missing and negative values instead of fabricating a zero delay.
|
||||
maybe_since_generation_completed_ms: maybe_generation_completed_at
|
||||
.map(|completed_at| (estimated_database_at - completed_at).num_milliseconds()),
|
||||
since_media_created_ms: (estimated_database_at - media_created_at).num_milliseconds(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
|
||||
mod database_clock {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn includes_query_time_and_time_spent_waiting_behind_other_files() {
|
||||
let clock = clock_with_two_second_query();
|
||||
let created_at = clock.database_read_at - TimeDelta::seconds(12);
|
||||
let completed_at = clock.database_read_at - TimeDelta::seconds(10);
|
||||
let started_at = clock.query_finished_at + Duration::from_secs(60);
|
||||
|
||||
let start = clock.snapshot_at(started_at, created_at, Some(completed_at));
|
||||
let end = clock.snapshot_at(started_at + Duration::from_secs(8), created_at, Some(completed_at));
|
||||
|
||||
assert_eq!(start.maybe_since_generation_completed_ms, Some(71_000));
|
||||
assert_eq!(end.maybe_since_generation_completed_ms, Some(79_000));
|
||||
assert_eq!(start.since_media_created_ms, 73_000);
|
||||
assert_eq!(end.estimated_database_at - start.estimated_database_at, TimeDelta::seconds(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_job_completion_does_not_become_media_creation_or_zero() {
|
||||
let clock = clock_with_two_second_query();
|
||||
let created_at = clock.database_read_at - TimeDelta::seconds(30);
|
||||
|
||||
let snapshot = clock.snapshot_at(clock.query_finished_at, created_at, None);
|
||||
|
||||
assert_eq!(snapshot.maybe_since_generation_completed_ms, None);
|
||||
assert_eq!(snapshot.since_media_created_ms, 31_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_latency_is_preserved_for_diagnosing_timestamp_anomalies() {
|
||||
let clock = clock_with_two_second_query();
|
||||
let completed_at = clock.database_read_at + TimeDelta::seconds(2);
|
||||
|
||||
let snapshot = clock.snapshot_at(clock.query_finished_at, clock.database_read_at, Some(completed_at));
|
||||
|
||||
assert_eq!(snapshot.maybe_since_generation_completed_ms, Some(-1_000));
|
||||
}
|
||||
}
|
||||
|
||||
fn clock_with_two_second_query() -> DatabaseClock {
|
||||
let query_started_at = Instant::now();
|
||||
DatabaseClock {
|
||||
// Intentionally unrelated to the worker's wall clock.
|
||||
database_read_at: DateTime::from_timestamp(1_000_000, 0).unwrap(),
|
||||
query_started_at,
|
||||
query_finished_at: query_started_at + Duration::from_secs(2),
|
||||
}
|
||||
}
|
||||
}
|
||||
Loaded 100 of 205 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user