mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-19 08:05:20 +00:00
Killing the pubsub thread works now!
This commit is contained in:
1 parent
35824b295e
commit
36d61a999c
4 files changed
+110
-15
No files matched your search
Generated
+28
-3
@@ -1019,12 +1019,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.5.2"
|
||||
version = "4.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e"
|
||||
checksum = "50b727aacc797f9fc28e355d21f34709ac4fc9adecfe470ad07b8f4464f53062"
|
||||
dependencies = [
|
||||
"bytes 1.0.1",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite 0.2.6",
|
||||
"tokio 1.13.0",
|
||||
"tokio-util 0.6.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3005,7 +3009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a32cb439c4e89c1e6415e5b3b23d9d8cc6dc1bf5a8cade19adbd5418de803be"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"combine 4.5.2",
|
||||
"combine 4.6.3",
|
||||
"dtoa",
|
||||
"itoa",
|
||||
"percent-encoding 2.1.0",
|
||||
@@ -3013,6 +3017,26 @@ dependencies = [
|
||||
"url 2.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.21.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a80b5f38d7f5a020856a0e16e40a9cfabf88ae8f0e4c2dcd8a3114c1e470852"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.0.1",
|
||||
"combine 4.6.3",
|
||||
"dtoa",
|
||||
"futures-util",
|
||||
"itoa",
|
||||
"percent-encoding 2.1.0",
|
||||
"pin-project-lite 0.2.6",
|
||||
"sha1",
|
||||
"tokio 1.13.0",
|
||||
"tokio-util 0.6.9",
|
||||
"url 2.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis_common"
|
||||
version = "0.0.1"
|
||||
@@ -4762,6 +4786,7 @@ dependencies = [
|
||||
"pulldown-cmark",
|
||||
"r2d2_redis",
|
||||
"rand 0.8.4",
|
||||
"redis 0.21.5",
|
||||
"redis_common",
|
||||
"regex",
|
||||
"reqwest",
|
||||
|
||||
@@ -63,6 +63,7 @@ sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "off
|
||||
# Redis Database
|
||||
limitation = "0.1.1" # Redis rate limiter
|
||||
r2d2_redis = "0.14.0"
|
||||
redis_async = { package = "redis", version = "0.21.5", features = ["tokio-comp"] }
|
||||
|
||||
# Twitch
|
||||
# NB "unsupported" pubsub features may disappear from the Twitch API at any time. It's an unofficial API.
|
||||
|
||||
@@ -33,6 +33,7 @@ use twitch_common::twitch_user_id::TwitchUserId;
|
||||
use crate::endpoints_ws::helpers::publish_active_browser_info::publish_active_browser_info;
|
||||
use crate::endpoints_ws::threads::redis_pubsub_event_listener_thread::RedisPubsubEventListenerThread;
|
||||
use crate::endpoints_ws::threads::tts_inference_job_token_queue::TtsInferenceJobTokenQueue;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
// TODO: Redis calls are synchronous (but fast), but is there any way to make them async?
|
||||
|
||||
@@ -129,7 +130,10 @@ pub async fn obs_gateway_websocket_handler(
|
||||
|
||||
info!("Starting user Redis PubSub thread...");
|
||||
|
||||
server_state.multithreading.redis_pubsub_runtime.spawn(thread.start_thread());
|
||||
let mut jh = server_state.multithreading.redis_pubsub_runtime.spawn(thread.start_thread());
|
||||
|
||||
|
||||
server_state.multithreading.redis_pubsub_runtime.spawn(kill(jh));
|
||||
|
||||
let websocket = ObsGatewayWebSocket::new(
|
||||
tts_job_token_queue,
|
||||
@@ -146,6 +150,15 @@ pub async fn obs_gateway_websocket_handler(
|
||||
})
|
||||
}
|
||||
|
||||
// NB: Well, abort() doesn't stop the thread :(
|
||||
async fn kill (handle: JoinHandle<()>) {
|
||||
error!("Waiting to kill pubsub thread...");
|
||||
sleep(Duration::from_secs(120));
|
||||
error!("Killing pubsub thread...");
|
||||
handle.abort();
|
||||
error!("Killed pubsub thread.");
|
||||
}
|
||||
|
||||
/// Websocket behavior
|
||||
struct ObsGatewayWebSocket {
|
||||
twitch_user_id: TwitchUserId,
|
||||
|
||||
+67
-11
@@ -1,17 +1,25 @@
|
||||
use std::sync::Arc;
|
||||
use container_common::anyhow_result::AnyhowResult;
|
||||
use futures_util::StreamExt;
|
||||
use crate::endpoints_ws::threads::tts_inference_job_token_queue::TtsInferenceJobTokenQueue;
|
||||
use futures::Future;
|
||||
use futures::future::Ready;
|
||||
use log::error;
|
||||
use log::info;
|
||||
use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2;
|
||||
use r2d2_redis::redis::Commands;
|
||||
use container_common::anyhow_result::AnyhowResult;
|
||||
use log::info;
|
||||
use crate::endpoints_ws::threads::tts_inference_job_token_queue::TtsInferenceJobTokenQueue;
|
||||
use twitch_common::twitch_user_id::TwitchUserId;
|
||||
use r2d2_redis::redis::{Commands, PubSub, Msg};
|
||||
use redis_common::redis_keys::RedisKeys;
|
||||
use redis_async::Client as AsyncClient;
|
||||
use std::sync::Arc;
|
||||
use std::thread::sleep;
|
||||
use time::Duration;
|
||||
use twitch_common::twitch_user_id::TwitchUserId;
|
||||
|
||||
pub struct RedisPubsubEventListenerThread {
|
||||
twitch_user_id: TwitchUserId,
|
||||
/// Redis PubSub connection exclusive to *this* thread.
|
||||
redis: Arc<r2d2::Pool<RedisConnectionManager>>,
|
||||
async_redis: AsyncClient,
|
||||
token_queue: TtsInferenceJobTokenQueue,
|
||||
}
|
||||
|
||||
@@ -28,8 +36,12 @@ impl RedisPubsubEventListenerThread {
|
||||
let redis = Arc::new(r2d2::Pool::builder()
|
||||
.build(redis_manager)?);
|
||||
|
||||
|
||||
let async_redis = AsyncClient::open(redis_pubsub_connection_string)?;
|
||||
|
||||
Ok(Self {
|
||||
twitch_user_id: twitch_user_id.clone(),
|
||||
async_redis,
|
||||
redis,
|
||||
token_queue,
|
||||
})
|
||||
@@ -46,18 +58,62 @@ impl RedisPubsubEventListenerThread {
|
||||
let mut pubsub_pool = self.redis.get().unwrap();
|
||||
let mut pubsub = pubsub_pool.as_pubsub();
|
||||
|
||||
let mut async_redis_conn = self.async_redis.get_async_connection().await.unwrap();
|
||||
let mut async_pubsub = async_redis_conn.into_pubsub();
|
||||
|
||||
let pubsub_key = RedisKeys::twitch_tts_job_topic(self.twitch_user_id.get_str());
|
||||
pubsub.subscribe(&pubsub_key).unwrap();
|
||||
|
||||
let mut pubsub_stream = async_pubsub.on_message();
|
||||
|
||||
loop {
|
||||
info!("Twitch user Redis PubSub thread loop iter");
|
||||
|
||||
// TODO: This is blocking. We need a way to exit the thread.
|
||||
let message = pubsub.get_message().unwrap();
|
||||
let tts_job_token : String = message.get_payload().unwrap();
|
||||
|
||||
info!("Job token received from Redis: {}", tts_job_token);
|
||||
self.token_queue.enqueue_token(&tts_job_token).unwrap();
|
||||
// NB: We can't have calls to read the Twitch websocket client block forever, and they
|
||||
// would do exactly that if not for this code. This is adapted from the very good example
|
||||
// in the `tokio-tungstenite` repo, which also contains good recipes for splitting sockets
|
||||
// into two unidirectional streams:
|
||||
// https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs
|
||||
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
|
||||
tokio::select! {
|
||||
//maybe_message = pubsub.get_message().unwrap() => {
|
||||
//maybe_message = Self::blocking_get_message(&mut pubsub) => {
|
||||
maybe_message = pubsub_stream.next() => {
|
||||
//let tts_job_token : String = maybe_message.unwrap().get_payload().unwrap();
|
||||
if let Some(message) = maybe_message {
|
||||
let tts_job_token : String = message.get_payload().unwrap();
|
||||
|
||||
info!("Job token received from Redis: {}", tts_job_token);
|
||||
self.token_queue.enqueue_token(&tts_job_token).unwrap();
|
||||
|
||||
} else {
|
||||
info!("NO MESSAGE");
|
||||
sleep(std::time::Duration::from_secs(1));
|
||||
|
||||
}
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
sleep(std::time::Duration::from_secs(1));
|
||||
error!(".");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// // TODO: This is blocking. We need a way to exit the thread.
|
||||
// let message = pubsub.get_message().unwrap();
|
||||
// let tts_job_token : String = message.get_payload().unwrap();
|
||||
//
|
||||
// info!("Job token received from Redis: {}", tts_job_token);
|
||||
// self.token_queue.enqueue_token(&tts_job_token).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Futures. Ugh
|
||||
fn blocking_get_message(pubsub: &mut PubSub) -> Ready<Result<Msg,()>> {
|
||||
let message = pubsub.get_message().unwrap();
|
||||
let fut = futures::future::ok::<Msg,()>(message);
|
||||
fut
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user