diff --git a/build.rs b/build.rs index 1d84ef2184..fadb422993 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,12 @@ extern crate anyhow; // TODO: No! extern crate prost_build; +use std::env; + fn main() -> anyhow::Result<()> { + // NB: Control prost output dir with env var + env::set_var("OUT_DIR", "src/proto_codegen"); + prost_build::compile_protos( &[ "protos/storyteller-protos/protos/pubsub-events/pubsub-events.proto" diff --git a/protos/storyteller-protos b/protos/storyteller-protos index a1e931b5e8..faf4d85ac6 160000 --- a/protos/storyteller-protos +++ b/protos/storyteller-protos @@ -1 +1 @@ -Subproject commit a1e931b5e89b8ba0a8f851f7d65628d6501cfe2f +Subproject commit faf4d85ac62186624d2c4921e42fec889f152e7a diff --git a/src/proto_codegen/storyteller.pubsub.rs b/src/proto_codegen/storyteller.pubsub.rs new file mode 100644 index 0000000000..098f6b8b08 --- /dev/null +++ b/src/proto_codegen/storyteller.pubsub.rs @@ -0,0 +1,181 @@ +/// Ingestion platforms are "dumb" and shouldn't have much logic in them. +/// (Granted, we might eventually grow to send them back messages.) +/// +/// The "enricher" is the smarts. It does denylisting, rate limiting, +/// service calls, transforms, etc. +/// +/// What if we need to scale? Several ideas: Sharding (redis db indices), +/// brokers with read-once semantics (our own intelligence - UUIDs with +/// transactional semantics in MySQL). +/// +/// Events coming from multiple sources +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PubsubEventPayloadV1 { + /// Source of the event. + #[prost(enumeration="pubsub_event_payload_v1::IngestionSourceType", optional, tag="1")] + pub ingestion_source_type: ::core::option::Option, + #[prost(enumeration="pubsub_event_payload_v1::IngestionPayloadType", optional, tag="2")] + pub ingestion_payload_type: ::core::option::Option, + /// A binary-encoded proto whose type is based on IngestionSourceType + #[prost(bytes="vec", optional, tag="3")] + pub ingestion_source_data: ::core::option::Option<::prost::alloc::vec::Vec>, + /// A binary-encoded proto whose type is based on IngestionPayloadType + #[prost(bytes="vec", optional, tag="4")] + pub ingestion_payload_data: ::core::option::Option<::prost::alloc::vec::Vec>, + /// An optional debug message. + #[prost(string, optional, tag="5")] + pub debug_message: ::core::option::Option<::prost::alloc::string::String>, +} +/// Nested message and enum types in `PubsubEventPayloadV1`. +pub mod pubsub_event_payload_v1 { + /// TODO: This should all be merged in the future, but I need to settle on proto2 vs proto3 + /// And I also need to figure out package names, etc. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum IngestionSourceType { + IstDoNotUse = 0, + IstAdminPanel = 1, + IstVocodesWebsite = 2, + IstTwitch = 3, + IstTwitter = 4, + IstDiscord = 5, + IstYoutube = 6, + IstReddit = 7, + IstTiktok = 8, + } + /// These will differ from the "enriched" types. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum IngestionPayloadType { + IptDoNotUse = 0, + /// Twitch-specific events. + /// + /// This can encode lots of things. + TwitchMessage = 100, + TwitchSubscribed = 101, + /// Twitter-specific events. + TwitterFollow = 200, + TwitterRetweet = 201, + TwitterMention = 202, + /// Discord-specific events. + DiscordMessage = 300, + DiscordJoined = 301, + /// YouTube-specific events. + YoutubeSubscribed = 400, + /// Reddit-specific events. + RedditMessage = 500, + } +} +// ============================== SOURCE PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(int64, optional, tag="1")] + pub user_id: ::core::option::Option, + #[prost(string, optional, tag="2")] + pub username: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bool, optional, tag="3")] + pub user_is_mod: ::core::option::Option, + #[prost(bool, optional, tag="4")] + pub user_is_subscribed: ::core::option::Option, + /// Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. + #[prost(string, optional, tag="5")] + pub channel: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionYoutubeMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionRedditMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTikTokMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +// ============================== TWITCH PAYLOAD PROTOS ============================== // + +/// These are raw messages from Twitch that we send to the PubSub enricher. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchMessage { + /// TODO: DEPRECATE THESE + #[prost(int64, optional, tag="1")] + pub user_id: ::core::option::Option, + #[prost(string, optional, tag="2")] + pub username: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bool, optional, tag="3")] + pub is_mod: ::core::option::Option, + #[prost(bool, optional, tag="4")] + pub is_subscribed: ::core::option::Option, + /// TODO: DEPRECATE THESE + /// Name of the channel, prefixed with hash. eg. `#vocodes`. + #[prost(string, optional, tag="5")] + pub channel: ::core::option::Option<::prost::alloc::string::String>, + /// Actual message. Doesn't need to be trimmed. + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchSubscribed { +} +// ============================== TWITTER PAYLOAD PROTOS ============================== // + +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterFollow { +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterRetweet { +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterMention { +} +// ============================== DISCORD PAYLOAD PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordMessage { + /// Raw message. Needs to be trimmed + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordJoined { +} +// ============================== YOUTUBE PAYLOAD PROTOS ============================== // + +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionYoutubeSubscribed { +} +// ============================== REDDIT PAYLOAD PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionRedditMessage { + /// Raw message. Needs to be trimmed + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} diff --git a/src/protos.rs b/src/protos.rs index 03a9ff7610..e46d12eab0 100644 --- a/src/protos.rs +++ b/src/protos.rs @@ -1,4 +1,5 @@ // Include the protos pub mod protos { - include!(concat!(env!("OUT_DIR"), "/storyteller.pubsub.rs")); + //include!(concat!(env!("OUT_DIR"), "/storyteller.pubsub.rs")); + include!("proto_codegen/storyteller.pubsub.rs"); } \ No newline at end of file