//! `FrameFetcher` against a real server with a fake `FrameSource`: cache, deadline, dedupe, //! parallel connections, and a peer that goes away. use std::path::PathBuf; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use serde_json::Value; use craft_ipc::bulk::{FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; use craft_ipc::fetch::{FetchOptions, Fetched, FrameFetcher, FrameKey, QUEUE_MAX}; use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; use craft_ipc::{InstanceRecord, LinkError, RuntimeDir}; const W: u32 = 4; const H: u32 = 3; const STRIDE: u32 = 16; struct Tmp(PathBuf); impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } struct NoSink; impl ControlSink for NoSink { fn submit(&self, _method: String, _params: Value) -> Option> { None } } struct NoActivate; impl Activate for NoActivate { fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { Activation::No } } /// The first pixel byte is the frame number asked for, so a wrong frame cannot match by accident. struct Fake { epoch: AtomicU64, delay_ms: AtomicU64, renders: AtomicUsize, times: Mutex>, fail_time: Mutex>, } impl FrameSource for Fake { fn open(&self, _spec: &FrameSpec) -> Result { Ok(FrameLayout { w: W, h: H, stride: STRIDE, len: u64::from(STRIDE * H) }) } fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { self.renders.fetch_add(1, Ordering::SeqCst); self.times.lock().unwrap().push(request.time_s); let delay = self.delay_ms.load(Ordering::SeqCst); if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); } if *self.fail_time.lock().unwrap() == Some(request.time_s) { return Err(LinkError::NotReady("comp is busy".into())); } let mut pixels = vec![0u8; (STRIDE * H) as usize]; pixels[0] = request.time_s as u8; Ok(RenderedFrame { epoch: self.epoch.load(Ordering::SeqCst).max(request.epoch), w: W, h: H, stride: STRIDE, format: spec.format, render_ms: 1.0, pixels, }) } fn close(&self, _spec: &FrameSpec) {} } struct Rig { _tmp: Tmp, server: Option, record: InstanceRecord, fake: Arc, } fn boot() -> Rig { // Clock ticks need not be distinct across parallel tests. static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let path = std::env::temp_dir().join(format!("cl-fch-{}-{serial}", std::process::id())); let _ = std::fs::remove_dir_all(&path); let fake = Arc::new(Fake { epoch: AtomicU64::new(1), delay_ms: AtomicU64::new(0), renders: AtomicUsize::new(0), times: Mutex::new(Vec::new()), fail_time: Mutex::new(None), }); let tickets = Arc::new(FrameTickets::new(Arc::clone(&fake) as Arc)); let server = Server::start(ServerConfig { app: format!("ec-{serial}"), app_version: "0.8.3".into(), instance: None, runtime: Some(RuntimeDir::at(&path).unwrap()), tcp_port: None, sink: Arc::new(NoSink), activate: Arc::new(NoActivate), frames: tickets as Arc, }) .unwrap(); let record = server.record().clone(); Rig { _tmp: Tmp(path), server: Some(server), record, fake } } fn connect(rig: &Rig, connections: usize) -> FrameFetcher { let options = FetchOptions { connections, io_timeout: Duration::from_secs(2), ..FetchOptions::default() }; FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap() } const fn key(frame: i64) -> FrameKey { FrameKey { frame, scale_milli: 1000 } } const WAIT: Duration = Duration::from_secs(2); #[test] fn fetch_returns_the_frame_and_the_second_ask_is_a_cache_hit() { let rig = boot(); let fetcher = connect(&rig, 1); let layout = fetcher.layout(); assert_eq!((layout.w, layout.h, layout.stride), (W, H, STRIDE)); let Fetched::Frame(first) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; assert_eq!(first.pixels[0], 7); assert_eq!(first.epoch, 1); let Fetched::Frame(second) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; assert!(Arc::ptr_eq(&first, &second), "a hit returns the cached frame"); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); let stats = fetcher.stats(); assert_eq!((stats.hits, stats.renders), (1, 1)); } #[test] fn a_late_frame_misses_the_deadline_then_lands_in_the_cache() { let rig = boot(); rig.fake.delay_ms.store(300, Ordering::SeqCst); let fetcher = connect(&rig, 1); let t0 = Instant::now(); assert!(matches!(fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(40)), Fetched::TimedOut)); assert!(t0.elapsed() < Duration::from_millis(200), "the call must not wait for the render: {:?}", t0.elapsed()); assert!(fetcher.cached(key(3), 1).is_none(), "not there yet"); let deadline = Instant::now() + Duration::from_secs(3); while fetcher.cached(key(3), 1).is_none() { assert!(Instant::now() < deadline, "the late frame never reached the cache"); std::thread::sleep(Duration::from_millis(10)); } let Fetched::Frame(frame) = fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(10)) else { panic!("no frame") }; assert_eq!(frame.pixels[0], 3); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1, "the timed-out ask and the retry share one render"); assert_eq!(fetcher.stats().timeouts, 1); } #[test] fn a_newer_epoch_asked_for_renders_again_and_an_older_cached_one_does_not_count() { let rig = boot(); let fetcher = connect(&rig, 1); let Fetched::Frame(old) = fetcher.fetch(key(1), 1.0, 1, WAIT) else { panic!("no frame") }; assert_eq!(old.epoch, 1); rig.fake.epoch.store(5, Ordering::SeqCst); // Epoch 1 is still good for a caller that wants epoch 1. assert!(fetcher.cached(key(1), 1).is_some()); assert!(fetcher.cached(key(1), 2).is_none(), "an epoch 1 frame cannot answer a request for epoch 2"); let Fetched::Frame(fresh) = fetcher.fetch(key(1), 1.0, 2, WAIT) else { panic!("no frame") }; assert_eq!(fresh.epoch, 5, "the source may answer with a newer epoch"); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2); // The newer frame answers an older request too. assert!(fetcher.cached(key(1), 1).is_some_and(|f| f.epoch == 5)); } #[test] fn two_asks_for_one_key_render_once() { let rig = boot(); rig.fake.delay_ms.store(120, Ordering::SeqCst); let fetcher = Arc::new(connect(&rig, 2)); let a = { let fetcher = Arc::clone(&fetcher); std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) }; let b = { let fetcher = Arc::clone(&fetcher); std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) }; assert!(a.join().unwrap() && b.join().unwrap()); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); } #[test] fn four_connections_render_four_frames_at_once() { let rig = boot(); rig.fake.delay_ms.store(200, Ordering::SeqCst); let fetcher = Arc::new(connect(&rig, 4)); let t0 = Instant::now(); let handles: Vec<_> = (0..4) .map(|n| { let fetcher = Arc::clone(&fetcher); std::thread::spawn(move || matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))) }) .collect(); for handle in handles { assert!(handle.join().unwrap()); } let took = t0.elapsed(); assert!(took < Duration::from_millis(600), "four 200 ms renders on four connections took {took:?}"); } #[test] fn a_frame_error_fails_that_frame_and_the_fetcher_keeps_working() { let rig = boot(); *rig.fake.fail_time.lock().unwrap() = Some(4.0); let fetcher = connect(&rig, 1); match fetcher.fetch(key(4), 4.0, 1, WAIT) { Fetched::Failed(message) => assert!(message.contains("comp is busy"), "{message}"), other => panic!("expected Failed, got {other:?}"), } assert!(!fetcher.is_down()); assert!(matches!(fetcher.fetch(key(5), 5.0, 1, WAIT), Fetched::Frame(_))); } #[test] fn a_peer_that_goes_away_makes_the_fetcher_down_and_asks_return_at_once() { let mut rig = boot(); let fetcher = connect(&rig, 2); assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); rig.server.take().unwrap().shutdown().unwrap(); let deadline = Instant::now() + Duration::from_secs(5); while !fetcher.is_down() { // A miss makes the workers touch the dead socket. let _ = fetcher.fetch(key(100), 100.0, 1, Duration::from_millis(50)); assert!(Instant::now() < deadline, "the fetcher never noticed the peer is gone"); } let t0 = Instant::now(); assert!(matches!(fetcher.fetch(key(200), 200.0, 1, WAIT), Fetched::Down)); assert!(t0.elapsed() < Duration::from_millis(100)); // Frames already in the cache still answer. assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); } #[test] fn the_cache_stays_inside_its_byte_budget() { let rig = boot(); let options = FetchOptions { connections: 1, cache_bytes: usize::try_from(STRIDE * H).unwrap() * 3, ..FetchOptions::default() }; let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); for n in 0..6 { assert!(matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))); } let kept = (0..6).filter(|n| fetcher.cached(key(*n), 1).is_some()).count(); assert_eq!(kept, 3, "a budget of three frames keeps three"); assert!(fetcher.cached(key(5), 1).is_some(), "the newest frame stays"); } #[test] fn queued_asks_nobody_waits_for_are_dropped_when_old() { let rig = boot(); rig.fake.delay_ms.store(150, Ordering::SeqCst); let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); // Five asks with no patience: frame 0 starts at once, frames 1 to 4 wait behind it. for n in 0..5 { assert!(matches!(fetcher.fetch(key(n), n as f64, 1, Duration::from_millis(1)), Fetched::TimedOut)); } std::thread::sleep(Duration::from_millis(900)); let renders = rig.fake.renders.load(Ordering::SeqCst); assert!(renders < 5, "frames that waited longer than max_age must be skipped, got {renders} renders"); } #[test] fn a_time_that_is_not_finite_fails_the_ask_and_keeps_the_workers() { let rig = boot(); let fetcher = connect(&rig, 1); for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { match fetcher.fetch(key(1), bad, 1, WAIT) { Fetched::Failed(message) => assert!(message.contains("finite"), "{message}"), other => panic!("expected Failed for {bad}, got {other:?}"), } } assert!(!fetcher.is_down(), "a bad time must not end the worker"); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 0, "nothing was queued"); assert!(matches!(fetcher.fetch(key(2), 2.0, 1, WAIT), Fetched::Frame(_))); } #[test] fn an_ask_with_a_waiting_caller_is_not_dropped_for_age() { let rig = boot(); rig.fake.delay_ms.store(300, Ordering::SeqCst); let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); // Frame 0 keeps the only worker busy for 300 ms. Frame 1 waits in the queue past max_age, // and its caller is still there when the worker reaches it. assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); let waiter = { let fetcher = Arc::clone(&fetcher); std::thread::spawn(move || fetcher.fetch(key(1), 1.0, 1, WAIT)) }; // A second caller for the same key joins the queued job and shares its render. std::thread::sleep(Duration::from_millis(20)); assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); assert!(matches!(waiter.join().unwrap(), Fetched::Frame(_))); let stats = fetcher.stats(); assert_eq!(stats.skipped, 0, "a job a caller waits for is not skipped"); assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2, "each frame renders once"); } #[test] fn the_queue_holds_at_most_queue_max_asks() { let rig = boot(); rig.fake.delay_ms.store(500, Ordering::SeqCst); let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); // Frame 0 holds the only worker. The asks after it only queue. assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); std::thread::sleep(Duration::from_millis(100)); let extra = 40; for n in 1..=i64::try_from(QUEUE_MAX + extra).unwrap() { assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); } assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap(), "the oldest asks past the cap are dropped"); } #[test] fn a_caller_waiting_on_a_dropped_ask_gets_failed_and_does_not_queue_it_again() { let rig = boot(); rig.fake.delay_ms.store(1500, Ordering::SeqCst); let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); // Frame 0 holds the only worker. Frame 1 queues behind it with a caller that waits. assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); std::thread::sleep(Duration::from_millis(100)); let (done, finished) = std::sync::mpsc::channel(); let waiter = { let fetcher = Arc::clone(&fetcher); std::thread::spawn(move || { let started = Instant::now(); let _ = done.send((fetcher.fetch(key(1), 1.0, 1, Duration::from_secs(20)), started.elapsed())); }) }; std::thread::sleep(Duration::from_millis(100)); // Fill the queue past its cap: frame 1 is the oldest, so it goes first. let extra = 5; for n in 2..=i64::try_from(QUEUE_MAX + extra).unwrap() { assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); } let (outcome, waited) = finished.recv_timeout(Duration::from_secs(1)).expect("the dropped caller must return at once"); assert!(matches!(&outcome, Fetched::Failed(message) if message.contains("dropped")), "{outcome:?}"); assert!(waited < Duration::from_secs(5), "{waited:?}"); waiter.join().unwrap(); std::thread::sleep(Duration::from_millis(100)); // Frame 1 plus the others make QUEUE_MAX + extra asks for QUEUE_MAX places. A caller that queued // frame 1 again would evict more. assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap()); // A new ask for the dropped frame is a new ask, not a repeat of the failure. assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); }