From 2c8fb4be965cb0770d2c599950b46da671482530 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 26 Apr 2021 02:42:35 -0400 Subject: [PATCH 01/18] Initial commit; a few messages to bootstrap with --- .gitignore | 1 + README.md | 4 ++++ protos/cesium/cesium.proto | 10 ++++++++++ protos/twitch-gateway/twitch-gateway.proto | 9 +++++++++ 4 files changed, 24 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 protos/cesium/cesium.proto create mode 100644 protos/twitch-gateway/twitch-gateway.proto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b25c15b81f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/README.md b/README.md new file mode 100644 index 0000000000..dbee9aedbd --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +storyteller-protos +================== +Our shared protos for all of our services and applications. + diff --git a/protos/cesium/cesium.proto b/protos/cesium/cesium.proto new file mode 100644 index 0000000000..2b98417b60 --- /dev/null +++ b/protos/cesium/cesium.proto @@ -0,0 +1,10 @@ + +package storyteller.cesium; + +message CesiumWarpRequest { + optional string twitch_username = 1; + optional string twitch_channel = 2; + optional double latitude = 3; + optional double longitude = 4; +} + diff --git a/protos/twitch-gateway/twitch-gateway.proto b/protos/twitch-gateway/twitch-gateway.proto new file mode 100644 index 0000000000..dfe8d4eb09 --- /dev/null +++ b/protos/twitch-gateway/twitch-gateway.proto @@ -0,0 +1,9 @@ + +package storyteller.twitchgateway; + +message TwitchMessage { + optional string username = 1; + optional string channel = 2; + optional string message_contents = 3; +} + From e5abaf2abfee547a8b5b3bb8b7b7126aec6dad5f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 27 Apr 2021 03:11:52 -0400 Subject: [PATCH 02/18] add new fields to twitchmessage (renumbered the tags) --- protos/twitch-gateway/twitch-gateway.proto | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/protos/twitch-gateway/twitch-gateway.proto b/protos/twitch-gateway/twitch-gateway.proto index dfe8d4eb09..c85c7abaf8 100644 --- a/protos/twitch-gateway/twitch-gateway.proto +++ b/protos/twitch-gateway/twitch-gateway.proto @@ -2,8 +2,14 @@ package storyteller.twitchgateway; message TwitchMessage { - optional string username = 1; - optional string channel = 2; - optional string message_contents = 3; + optional int64 user_id = 1; + optional string username = 2; + optional bool is_mod = 3; + optional bool is_subscribed = 4; + + optional string channel = 5; + + // Actual message. Doesn't need to be trimmed. + optional string message_contents = 6; } From a3051ded476fa133bd153a91845e0cd625afe59f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 27 Apr 2021 03:17:23 -0400 Subject: [PATCH 03/18] comment --- protos/twitch-gateway/twitch-gateway.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/protos/twitch-gateway/twitch-gateway.proto b/protos/twitch-gateway/twitch-gateway.proto index c85c7abaf8..9b4f8d44c1 100644 --- a/protos/twitch-gateway/twitch-gateway.proto +++ b/protos/twitch-gateway/twitch-gateway.proto @@ -7,6 +7,7 @@ message TwitchMessage { optional bool is_mod = 3; optional bool is_subscribed = 4; + // Name of the channel, prefixed with hash. eg. `#vocodes`. optional string channel = 5; // Actual message. Doesn't need to be trimmed. From 108f080920cbc533b2adb582d31183f9bbcca3b0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 29 Apr 2021 20:00:17 -0400 Subject: [PATCH 04/18] add unreal engine protos --- protos/unreal/unreal.proto | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 protos/unreal/unreal.proto diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto new file mode 100644 index 0000000000..4b768f9f41 --- /dev/null +++ b/protos/unreal/unreal.proto @@ -0,0 +1,38 @@ + +// The payloads delivered to unreal engine over the pubsub bus. +message UnrealEventPayloadV1 { + enum PayloadType { + DO_NOT_USE = 0; + VOCODES_TTS = 1; + LEVEL_WARP = 2; + CESIUM_WARP = 3; + CESIUM_TIME_CHANGE = 4; + SPAWN_CREATURE = 5; + CHANGE_MUSIC = 6; + } + + // Type of the payload. + optional PayloadType payload_type = 1; + + // A binary-encoded proto whose type is based on the above value. + optional bytes payload = 2; +} + +// Warp within a Cesium geospatial world. +message CesiumWarpRequest { + // Required + optional double latitude = 1; + + // Required + optional double longitude = 2; + + // Optional. Fields describing the user that sent the message. + optional int64 twitch_user_id = 3; + optional string twitch_username = 4; + optional bool twitch_user_is_mod = 5; + optional bool twitch_user_is_subscribed = 6; + + // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. + optional string twitch_channel = 7; +} + From 2fcb93f60a8e312404c7b20cc7451d64184c119a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 29 Apr 2021 20:11:50 -0400 Subject: [PATCH 05/18] add package to unreal protos --- protos/unreal/unreal.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 4b768f9f41..9043d6664b 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -1,3 +1,4 @@ +package storyteller.unreal; // The payloads delivered to unreal engine over the pubsub bus. message UnrealEventPayloadV1 { From f24ccba7604339d860be00f8b4df99f6fd09d347 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 1 May 2021 01:52:38 -0400 Subject: [PATCH 06/18] force proto3, protoc 3.6 compatible --- protos/unreal/unreal.proto | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 9043d6664b..9f48d3bea3 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -1,3 +1,5 @@ +syntax = 'proto3'; + package storyteller.unreal; // The payloads delivered to unreal engine over the pubsub bus. @@ -13,27 +15,27 @@ message UnrealEventPayloadV1 { } // Type of the payload. - optional PayloadType payload_type = 1; + PayloadType payload_type = 1; // A binary-encoded proto whose type is based on the above value. - optional bytes payload = 2; + bytes payload = 2; } // Warp within a Cesium geospatial world. message CesiumWarpRequest { // Required - optional double latitude = 1; + double latitude = 1; // Required - optional double longitude = 2; + double longitude = 2; // Optional. Fields describing the user that sent the message. - optional int64 twitch_user_id = 3; - optional string twitch_username = 4; - optional bool twitch_user_is_mod = 5; - optional bool twitch_user_is_subscribed = 6; + int64 twitch_user_id = 3; + string twitch_username = 4; + bool twitch_user_is_mod = 5; + bool twitch_user_is_subscribed = 6; // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. - optional string twitch_channel = 7; + string twitch_channel = 7; } From e86757d405aab3e9c5120b62c1eae76c03b75f1e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 1 May 2021 16:08:59 -0400 Subject: [PATCH 07/18] add debug message to payload --- protos/unreal/unreal.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 9f48d3bea3..00e7da3ab4 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -19,6 +19,9 @@ message UnrealEventPayloadV1 { // A binary-encoded proto whose type is based on the above value. bytes payload = 2; + + // An optional debug message. + string debug_message = 3; } // Warp within a Cesium geospatial world. From 590dc6756b7c48195ae729ce8d26660d7bb3e43c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 16:05:52 -0400 Subject: [PATCH 08/18] vastly more protos for the unreal firehose --- protos/twitch-gateway/twitch-gateway.proto | 1 + protos/unreal/unreal.proto | 160 ++++++++++++++++++++- 2 files changed, 155 insertions(+), 6 deletions(-) diff --git a/protos/twitch-gateway/twitch-gateway.proto b/protos/twitch-gateway/twitch-gateway.proto index 9b4f8d44c1..1fbcb886b9 100644 --- a/protos/twitch-gateway/twitch-gateway.proto +++ b/protos/twitch-gateway/twitch-gateway.proto @@ -1,6 +1,7 @@ package storyteller.twitchgateway; +// These are raw messages from Twitch that we send to the PubSub enricher. message TwitchMessage { optional int64 user_id = 1; optional string username = 2; diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 00e7da3ab4..e41100519a 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -4,26 +4,120 @@ package storyteller.unreal; // The payloads delivered to unreal engine over the pubsub bus. message UnrealEventPayloadV1 { + + // If people Retweet us or mention us, we should show that in the engine! + enum SourceType { + DO_NOT_USE = 0; + TWITCH = 1; + TWITTER = 2; + DISCORD = 3; + YOUTUBE = 4; + REDDIT = 5; + TIKTOK = 6; + } + enum PayloadType { DO_NOT_USE = 0; + + // Game events. These can come from many sources (control panel, Twitch, Discord) VOCODES_TTS = 1; LEVEL_WARP = 2; CESIUM_WARP = 3; CESIUM_TIME_CHANGE = 4; SPAWN_CREATURE = 5; - CHANGE_MUSIC = 6; + //CHANGE_MUSIC = 6; + + // Twitter-specific events. + TWITTER_FOLLOW = 200; + TWITTER_RETWEET = 201; + TWITTER_MENTION = 202; + + // Discord-specific events. + DISCORD_JOINED = 300; + + // YouTube-specific events. + YOUTUBE_SUBSCRIBED = 400; + + // Are there non-chat Reddit events? } - // Type of the payload. - PayloadType payload_type = 1; + // Source of the event. + SourceType source_type = 1; - // A binary-encoded proto whose type is based on the above value. - bytes payload = 2; + // Type of the payload. + PayloadType payload_type = 2; + + // A binary-encoded proto whose type is based on PayloadType + bytes payload_data = 3; + + // A binary-encoded proto whose type is based on SourceType + bytes source_data = 4; // An optional debug message. - string debug_message = 3; + string debug_message = 5; } + +// ============================== SOURCE PROTOS ============================== // + +message TwitchMetadata { + // Optional. Fields describing the user that sent the message. + int64 user_id = 1; + string username = 2; + bool user_is_mod = 3; + bool user_is_subscribed = 4; + + // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. + string ch_channel = 5; +} + + +message TwitterMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message YoutubeMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message DiscordMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message YoutubeMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message RedditMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message TikTokMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +// ============================== GAME EVENT PROTOS ============================== // + + +// Change time in Cesium. +message CesiumTimeChangeRequest { + // Required + int64 hour = 1; +} + + // Warp within a Cesium geospatial world. message CesiumWarpRequest { // Required @@ -32,13 +126,67 @@ message CesiumWarpRequest { // Required double longitude = 2; + // TODO: Remove these. // Optional. Fields describing the user that sent the message. int64 twitch_user_id = 3; string twitch_username = 4; bool twitch_user_is_mod = 5; bool twitch_user_is_subscribed = 6; + // TODO: Remove these. // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. string twitch_channel = 7; } + +// Change the level +message LevelWarpRequest { + // Required. Level to warp to. + string level_slug = 1; +} + + +// Spawn a creature +message SpawnCreatureRequest { + // Required. Name of the creature. + int64 name_slug = 1; +} + + +// Do text to speech +message VocodesTtsRequest { + // Required. Speaker to use. + string voice_slug = 1; + // Required. Text to speak. + double text = 2; +} + + +// ============================== TWITTER EVENT PROTOS ============================== // + + +message TwitterFollow { + // Intentionally blank +} + +message TwitterRetweet { + // Intentionally blank +} + +message TwitterMention { + // Intentionally blank +} + + +// ============================== DISCORD EVENT PROTOS ============================== // + +message DiscordJoined { + // Intentionally blank +} + +// ============================== YOUTUBE EVENT PROTOS ============================== // + +message YoutubeSubscribed { + // Intentionally blank +} + From bf5f427a610db448e0194758a7ddbc0e2b114e27 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 16:08:02 -0400 Subject: [PATCH 09/18] fix protos for C++ --- protos/unreal/unreal.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index e41100519a..50eeb64ba9 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -7,7 +7,7 @@ message UnrealEventPayloadV1 { // If people Retweet us or mention us, we should show that in the engine! enum SourceType { - DO_NOT_USE = 0; + ST_DO_NOT_USE = 0; TWITCH = 1; TWITTER = 2; DISCORD = 3; @@ -17,7 +17,7 @@ message UnrealEventPayloadV1 { } enum PayloadType { - DO_NOT_USE = 0; + PT_DO_NOT_USE = 0; // Game events. These can come from many sources (control panel, Twitch, Discord) VOCODES_TTS = 1; From 3217f96e6f93b10f3b823e89bd811ca2ed1b9969 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 16:10:39 -0400 Subject: [PATCH 10/18] fix duplication --- protos/unreal/unreal.proto | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 50eeb64ba9..dabb227b78 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -60,6 +60,7 @@ message UnrealEventPayloadV1 { // ============================== SOURCE PROTOS ============================== // + message TwitchMetadata { // Optional. Fields describing the user that sent the message. int64 user_id = 1; @@ -78,12 +79,6 @@ message TwitterMetadata { } -message YoutubeMetadata { - // Optional. Fields describing the user that sent the message. - string username = 1; -} - - message DiscordMetadata { // Optional. Fields describing the user that sent the message. string username = 1; From 0775b0f13fec6e6390843ad3e2ee49163a376952 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 16:31:44 -0400 Subject: [PATCH 11/18] not necessary anymore --- protos/cesium/cesium.proto | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 protos/cesium/cesium.proto diff --git a/protos/cesium/cesium.proto b/protos/cesium/cesium.proto deleted file mode 100644 index 2b98417b60..0000000000 --- a/protos/cesium/cesium.proto +++ /dev/null @@ -1,10 +0,0 @@ - -package storyteller.cesium; - -message CesiumWarpRequest { - optional string twitch_username = 1; - optional string twitch_channel = 2; - optional double latitude = 3; - optional double longitude = 4; -} - From 367b1bbc9f721c871dfc056c664020f46c9523e1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 17:21:26 -0400 Subject: [PATCH 12/18] huge refactor to ingestion protos --- protos/pubsub-events/pubsub-events.proto | 174 +++++++++++++++++++++ protos/twitch-gateway/twitch-gateway.proto | 17 -- protos/unreal/unreal.proto | 25 +-- 3 files changed, 188 insertions(+), 28 deletions(-) create mode 100644 protos/pubsub-events/pubsub-events.proto delete mode 100644 protos/twitch-gateway/twitch-gateway.proto diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto new file mode 100644 index 0000000000..12a51d53d0 --- /dev/null +++ b/protos/pubsub-events/pubsub-events.proto @@ -0,0 +1,174 @@ +syntax = 'proto2'; + +package storyteller.pubsub; + +// Ingestion platforms are "dumb" and shouldn't have much logic in them. +// (Granted, we might eventually grow to send them back messages.) +// +// The "enricher" is the smarts. It does denylisting, rate limiting, +// service calls, transforms, etc. +// +// What if we need to scale? Several ideas: Sharding (redis db indices), +// brokers with read-once semantics (our own intelligence - UUIDs with +// transactional semantics in MySQL). +// +// Events coming from multiple sources +message PubsubEventPayloadV1 { + + // TODO: This should all be merged in the future, but I need to settle on proto2 vs proto3 + // And I also need to figure out package names, etc. + enum IngestionSourceType { + IST_DO_NOT_USE = 0; + IST_ADMIN_PANEL = 1; + IST_VOCODES_WEBSITE = 2; + IST_TWITCH = 3; + IST_TWITTER = 4; + IST_DISCORD = 5; + IST_YOUTUBE = 6; + IST_REDDIT = 7; + IST_TIKTOK = 8; + } + + // These will differ from the "enriched" types. + enum IngestionPayloadType { + IPT_DO_NOT_USE = 0; + + // Twitch-specific events. + TWITCH_MESSAGE = 100; // This can encode lots of things. + TWITCH_SUBSCRIBED = 101; + + // Twitter-specific events. + TWITTER_FOLLOW = 200; + TWITTER_RETWEET = 201; + TWITTER_MENTION = 202; + + // Discord-specific events. + DISCORD_MESSAGE = 300; + DISCORD_JOINED = 301; + + // YouTube-specific events. + YOUTUBE_SUBSCRIBED = 400; + + // Reddit-specific events. + REDDIT_MESSAGE = 500; + } + + // Source of the event. + IngestionSourceType ingestion_source_type = 1; + IngestionPayloadType ingestion_payload_type = 2; +} + +// ============================== SOURCE PROTOS ============================== // + + +message IngestionTwitchMetadata { + // Optional. Fields describing the user that sent the message. + int64 user_id = 1; + string username = 2; + bool user_is_mod = 3; + bool user_is_subscribed = 4; + + // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. + string channel = 5; +} + + +message IngestionTwitterMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message IngestionDiscordMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message IngestionYoutubeMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message RedditMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +message IngestionTikTokMetadata { + // Optional. Fields describing the user that sent the message. + string username = 1; +} + + +// ============================== TWITCH PAYLOAD PROTOS ============================== // + + +// These are raw messages from Twitch that we send to the PubSub enricher. +message IngestionTwitchMessage { + // TODO: DEPRECATE THESE + optional int64 user_id = 1; + optional string username = 2; + optional bool is_mod = 3; + optional bool is_subscribed = 4; + + // TODO: DEPRECATE THESE + // Name of the channel, prefixed with hash. eg. `#vocodes`. + optional string channel = 5; + + // Actual message. Doesn't need to be trimmed. + optional string message_contents = 6; +} + + +message IngestionTwitchSubscribed { + // Intentionally blank for now +} + + +// ============================== TWITTER PAYLOAD PROTOS ============================== // + + +message IngestionTwitterFollow { + // Intentionally blank for now +} +message IngestionTwitterRetweet { + // Intentionally blank for now +} +message IngestionTwitterMention { + // Intentionally blank for now +} + + +// ============================== DISCORD PAYLOAD PROTOS ============================== // + + +message IngestionDiscordMessage { + // Raw message. Needs to be trimmed + optional string message_contents = 6; +} + +message IngestionDiscordJoined { + // Intentionally blank for now +} + + +// ============================== YOUTUBE PAYLOAD PROTOS ============================== // + + +message IngestionYoutubeSubscribed { + // Intentionally blank for now +} + + + +// ============================== REDDIT PAYLOAD PROTOS ============================== // + + +message IngestionRedditMessage { + // Raw message. Needs to be trimmed + optional string message_contents = 6; +} + diff --git a/protos/twitch-gateway/twitch-gateway.proto b/protos/twitch-gateway/twitch-gateway.proto deleted file mode 100644 index 1fbcb886b9..0000000000 --- a/protos/twitch-gateway/twitch-gateway.proto +++ /dev/null @@ -1,17 +0,0 @@ - -package storyteller.twitchgateway; - -// These are raw messages from Twitch that we send to the PubSub enricher. -message TwitchMessage { - optional int64 user_id = 1; - optional string username = 2; - optional bool is_mod = 3; - optional bool is_subscribed = 4; - - // Name of the channel, prefixed with hash. eg. `#vocodes`. - optional string channel = 5; - - // Actual message. Doesn't need to be trimmed. - optional string message_contents = 6; -} - diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index dabb227b78..8126623b49 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -3,17 +3,20 @@ syntax = 'proto3'; package storyteller.unreal; // The payloads delivered to unreal engine over the pubsub bus. +// These are the "enriched" events. message UnrealEventPayloadV1 { // If people Retweet us or mention us, we should show that in the engine! enum SourceType { ST_DO_NOT_USE = 0; - TWITCH = 1; - TWITTER = 2; - DISCORD = 3; - YOUTUBE = 4; - REDDIT = 5; - TIKTOK = 6; + ST_ADMIN_PANEL = 1; + ST_VOCODES_WEBSITE = 2; + ST_TWITCH = 3; + ST_TWITTER = 4; + ST_DISCORD = 5; + ST_YOUTUBE = 6; + ST_REDDIT = 7; + ST_TIKTOK = 8; } enum PayloadType { @@ -47,11 +50,11 @@ message UnrealEventPayloadV1 { // Type of the payload. PayloadType payload_type = 2; - // A binary-encoded proto whose type is based on PayloadType - bytes payload_data = 3; - // A binary-encoded proto whose type is based on SourceType - bytes source_data = 4; + bytes source_data = 3; + + // A binary-encoded proto whose type is based on PayloadType + bytes payload_data = 4; // An optional debug message. string debug_message = 5; @@ -69,7 +72,7 @@ message TwitchMetadata { bool user_is_subscribed = 4; // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. - string ch_channel = 5; + string channel = 5; } From 491b51939b37466d168d8cb9f2fc77fdb8c036fc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 17:23:49 -0400 Subject: [PATCH 13/18] omitted payload data --- protos/pubsub-events/pubsub-events.proto | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto index 12a51d53d0..e33162f01d 100644 --- a/protos/pubsub-events/pubsub-events.proto +++ b/protos/pubsub-events/pubsub-events.proto @@ -56,6 +56,15 @@ message PubsubEventPayloadV1 { // Source of the event. IngestionSourceType ingestion_source_type = 1; IngestionPayloadType ingestion_payload_type = 2; + + // A binary-encoded proto whose type is based on IngestionSourceType + bytes ingestion_source_data = 3; + + // A binary-encoded proto whose type is based on IngestionPayloadType + bytes ingestion_payload_data = 4; + + // An optional debug message. + string debug_message = 5; } // ============================== SOURCE PROTOS ============================== // From a1e931b5e89b8ba0a8f851f7d65628d6501cfe2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 17:31:38 -0400 Subject: [PATCH 14/18] optional types --- protos/pubsub-events/pubsub-events.proto | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto index e33162f01d..62f69ff08d 100644 --- a/protos/pubsub-events/pubsub-events.proto +++ b/protos/pubsub-events/pubsub-events.proto @@ -54,17 +54,17 @@ message PubsubEventPayloadV1 { } // Source of the event. - IngestionSourceType ingestion_source_type = 1; - IngestionPayloadType ingestion_payload_type = 2; + optional IngestionSourceType ingestion_source_type = 1; + optional IngestionPayloadType ingestion_payload_type = 2; // A binary-encoded proto whose type is based on IngestionSourceType - bytes ingestion_source_data = 3; + optional bytes ingestion_source_data = 3; // A binary-encoded proto whose type is based on IngestionPayloadType - bytes ingestion_payload_data = 4; + optional bytes ingestion_payload_data = 4; // An optional debug message. - string debug_message = 5; + optional string debug_message = 5; } // ============================== SOURCE PROTOS ============================== // @@ -72,43 +72,43 @@ message PubsubEventPayloadV1 { message IngestionTwitchMetadata { // Optional. Fields describing the user that sent the message. - int64 user_id = 1; - string username = 2; - bool user_is_mod = 3; - bool user_is_subscribed = 4; + optional int64 user_id = 1; + optional string username = 2; + optional bool user_is_mod = 3; + optional bool user_is_subscribed = 4; // Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. - string channel = 5; + optional string channel = 5; } message IngestionTwitterMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + optional string username = 1; } message IngestionDiscordMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + optional string username = 1; } message IngestionYoutubeMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + optional string username = 1; } message RedditMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + optional string username = 1; } message IngestionTikTokMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + optional string username = 1; } From faf4d85ac62186624d2c4921e42fec889f152e7a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 May 2021 19:00:14 -0400 Subject: [PATCH 15/18] fix name collision --- protos/pubsub-events/pubsub-events.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto index 62f69ff08d..7ca727490c 100644 --- a/protos/pubsub-events/pubsub-events.proto +++ b/protos/pubsub-events/pubsub-events.proto @@ -100,7 +100,7 @@ message IngestionYoutubeMetadata { } -message RedditMetadata { +message IngestionRedditMetadata { // Optional. Fields describing the user that sent the message. optional string username = 1; } From c61a069a6ffcd6fdc15684e3086c48373a670fd0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 8 May 2021 04:18:05 -0400 Subject: [PATCH 16/18] fix proto field types --- protos/unreal/unreal.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 8126623b49..328f42fe5c 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -147,7 +147,7 @@ message LevelWarpRequest { // Spawn a creature message SpawnCreatureRequest { // Required. Name of the creature. - int64 name_slug = 1; + string name_slug = 1; } @@ -156,7 +156,7 @@ message VocodesTtsRequest { // Required. Speaker to use. string voice_slug = 1; // Required. Text to speak. - double text = 2; + string text = 2; } From 921435d1772641e1af384b41f004d905df0f69fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 03:51:02 -0400 Subject: [PATCH 17/18] more twitter fields --- protos/pubsub-events/pubsub-events.proto | 7 ++++++- protos/unreal/unreal.proto | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto index 7ca727490c..4ea0cc542a 100644 --- a/protos/pubsub-events/pubsub-events.proto +++ b/protos/pubsub-events/pubsub-events.proto @@ -84,7 +84,10 @@ message IngestionTwitchMetadata { message IngestionTwitterMetadata { // Optional. Fields describing the user that sent the message. - optional string username = 1; + optional int64 user_id = 1; + optional string username = 2; + optional string display_name = 3; + optional string avatar_url = 4; } @@ -145,9 +148,11 @@ message IngestionTwitterFollow { } message IngestionTwitterRetweet { // Intentionally blank for now + optional string original_text = 1; } message IngestionTwitterMention { // Intentionally blank for now + optional string text = 1; } diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 328f42fe5c..20088def74 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -78,7 +78,10 @@ message TwitchMetadata { message TwitterMetadata { // Optional. Fields describing the user that sent the message. - string username = 1; + int64 user_id = 1; + string username = 2; + string display_name = 3; + string avatar_url = 4; } @@ -168,11 +171,11 @@ message TwitterFollow { } message TwitterRetweet { - // Intentionally blank + string original_text = 1; } message TwitterMention { - // Intentionally blank + string text = 1; } From 67b750d956c9e62051e67acab850a3cbc4398ad9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 04:10:11 -0400 Subject: [PATCH 18/18] Don't use Twitter user_id just yet due to signedness --- protos/pubsub-events/pubsub-events.proto | 5 ++++- protos/unreal/unreal.proto | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/protos/pubsub-events/pubsub-events.proto b/protos/pubsub-events/pubsub-events.proto index 4ea0cc542a..d1af518f18 100644 --- a/protos/pubsub-events/pubsub-events.proto +++ b/protos/pubsub-events/pubsub-events.proto @@ -84,7 +84,10 @@ message IngestionTwitchMetadata { message IngestionTwitterMetadata { // Optional. Fields describing the user that sent the message. - optional int64 user_id = 1; + + // TODO: This is a u64. We need to be careful with math to assure non-overflow. + //optional int64 user_id = 1; + optional string username = 2; optional string display_name = 3; optional string avatar_url = 4; diff --git a/protos/unreal/unreal.proto b/protos/unreal/unreal.proto index 20088def74..1564f412c9 100644 --- a/protos/unreal/unreal.proto +++ b/protos/unreal/unreal.proto @@ -78,7 +78,10 @@ message TwitchMetadata { message TwitterMetadata { // Optional. Fields describing the user that sent the message. - int64 user_id = 1; + + // TODO: This is a u64. We need to be careful with math to assure non-overflow. + //int64 user_id = 1; + string username = 2; string display_name = 3; string avatar_url = 4;