diff --git a/src/graphics_imgui.rs b/src/graphics_imgui.rs index fb818fc518..3850f28d9f 100644 --- a/src/graphics_imgui.rs +++ b/src/graphics_imgui.rs @@ -8,14 +8,13 @@ use imgui::Image; use core_types::RgbaF32; use gui::enhanced_window::EnhancedWindow; use gui::mouse_camera_arcball::MouseCameraArcball; -use kinect::sensor_control::CaptureProvider; use point_cloud::pixel_structs::BgraPixel; use point_cloud::point_cloud_visualiser::{ColorizationStrategy, PointCloudVisualizer, PointCloudVisualizerError}; use point_cloud::viewer_image::ViewerImage; use webcam::WebcamWriter; -use kinect::multi_device_capturer::MultiDeviceCaptureProvider; +use kinect::capture::device_capturer::CaptureProvider; -pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4a_calibration_t, enable_webcam: bool) { +pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4a_calibration_t, enable_webcam: bool) { let mut webcam_writer = None; if enable_webcam { diff --git a/src/kinect/capture/device_capturer.rs b/src/kinect/capture/device_capturer.rs new file mode 100644 index 0000000000..748b14f8e2 --- /dev/null +++ b/src/kinect/capture/device_capturer.rs @@ -0,0 +1,13 @@ +use point_cloud::debug::capture_proxy::CaptureProxy; + +pub trait CaptureProvider { + + /** Return the number of cameras. */ + fn get_num_cameras(&self) -> usize; + + /** + * Take the latest captures, if available, through interior mutability. + * This leaves the mutex holding an empty vec. + */ + fn get_captures(&self) -> Option>; +} diff --git a/src/kinect/capture/fake_device_capturer.rs b/src/kinect/capture/fake_device_capturer.rs new file mode 100644 index 0000000000..1889fa7e73 --- /dev/null +++ b/src/kinect/capture/fake_device_capturer.rs @@ -0,0 +1,48 @@ +use anyhow::Result as AnyhowResult; +use kinect::capture::device_capturer::CaptureProvider; +use point_cloud::debug::capture_proxy::CaptureProxy; +use point_cloud::debug::image_proxy::ImageProxy; +use kinect::k4a_sys_wrapper::ImageFormat; + +pub struct FakeDeviceCaptureProvider { + num_cameras: usize, + captures: Vec, +} + +impl FakeDeviceCaptureProvider { + pub fn new() -> AnyhowResult { + // TODO: Dimensions are wrong + let color = ImageProxy::from_file("output/color_src_0", 0, 0, 0, ImageFormat::ColorBgra32)?; + let depth = ImageProxy::from_file("output/depth_src_0", 3840, 2160, 7680, ImageFormat::Depth16)?; + //let depth = ImageProxy::from_file("output/depth_src_0", 640, 576, 1280, ImageFormat::Depth16)?; + let frame_1 = CaptureProxy::from_image_proxy_pair(color, depth); + + // TODO: Dimensions are wrong + let color = ImageProxy::from_file("output/color_src_1", 0, 0, 0, ImageFormat::ColorBgra32)?; + let depth = ImageProxy::from_file("output/depth_src_1", 3840, 2160, 7680, ImageFormat::Depth16)?; + //let depth = ImageProxy::from_file("output/depth_src_1", 640, 576, 1280, ImageFormat::Depth16)?; + let frame_2 = CaptureProxy::from_image_proxy_pair(color, depth); + + let mut captures = Vec::new(); + captures.push(frame_1); + captures.push(frame_2); + + Ok(Self { + num_cameras: 2, + captures, + }) + } +} + +impl CaptureProvider for FakeDeviceCaptureProvider { + fn get_num_cameras(&self) -> usize { + self.num_cameras + } + + fn get_captures(&self) -> Option> { + let capture_clones = self.captures.iter() + .map(|capture| capture.clone()) + .collect(); + Some(capture_clones) + } +} diff --git a/src/kinect/capture/mod.rs b/src/kinect/capture/mod.rs new file mode 100644 index 0000000000..9c8a1fc4c1 --- /dev/null +++ b/src/kinect/capture/mod.rs @@ -0,0 +1,3 @@ +pub mod device_capturer; +pub mod fake_device_capturer; +pub mod multi_device_capturer; diff --git a/src/kinect/multi_device_capturer.rs b/src/kinect/capture/multi_device_capturer.rs similarity index 95% rename from src/kinect/multi_device_capturer.rs rename to src/kinect/capture/multi_device_capturer.rs index 399a641829..e30fdbf4e4 100644 --- a/src/kinect/multi_device_capturer.rs +++ b/src/kinect/capture/multi_device_capturer.rs @@ -1,6 +1,8 @@ use kinect::k4a_sys_wrapper::{Device, KinectError, Capture, CaptureError, GetCaptureError}; use std::sync::Arc; use std::sync::Mutex; +use kinect::capture::device_capturer::CaptureProvider; +use point_cloud::debug::capture_proxy::CaptureProxy; // Allowing at least 160 microseconds between depth cameras should ensure they do not interfere with one another. const MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC : libc::int32_t = 160; @@ -123,21 +125,6 @@ impl MultiDeviceCaptureProvider { } } - /** Return the number of cameras. */ - pub fn get_num_cameras(&self) -> usize { - self.num_cameras - } - - /** - * Take the latest captures, if available, through interior mutability. - * This leaves the mutex holding an empty vec. - */ - pub fn get_captures(&self) -> Option> { - self.captures.lock() - .ok() - .map(|mut v| (*v).split_off(0)) - } - /** * Consume captures and replace whatever we currently hold. */ @@ -151,6 +138,27 @@ impl MultiDeviceCaptureProvider { } } +impl CaptureProvider for MultiDeviceCaptureProvider { + + /** Return the number of cameras. */ + fn get_num_cameras(&self) -> usize { + self.num_cameras + } + + /** + * Take the latest captures, if available, through interior mutability. + * This leaves the mutex holding an empty vec. + */ + fn get_captures(&self) -> Option> { + self.captures.lock() + .ok() + .map(|mut v| (*v).split_off(0)) + .map(|mut captures| captures.into_iter() + .map(|capture| CaptureProxy::consume_k4a_capture(capture)) + .collect()) + } +} + pub fn start_capture_thread(capturer: MultiDeviceCapturer) { let mut capture_provider = capturer.get_sync_capture_provider(); diff --git a/src/kinect/k4a_sys_wrapper.rs b/src/kinect/k4a_sys_wrapper.rs index f904e69fb9..c09f79e038 100644 --- a/src/kinect/k4a_sys_wrapper.rs +++ b/src/kinect/k4a_sys_wrapper.rs @@ -525,6 +525,23 @@ pub enum ImageFormat { UnknownFormatError, // FIXME: Just return Result? } +impl ImageFormat { + pub fn to_k4a(&self) -> k4a_sys::k4a_image_format_t { + match self { + ImageFormat::ColorMjpg => 0, + ImageFormat::ColorNv12 => 1, + ImageFormat::ColorYuy2 => 2, + ImageFormat::ColorBgra32 => 3, + ImageFormat::Depth16 => 4, + ImageFormat::Ir16 => 5, + ImageFormat::Custom8 => 6, + ImageFormat::Custom16 => 7, + ImageFormat::Custom => 8, + ImageFormat::UnknownFormatError => 255, // TODO: Better handling? + } + } +} + impl From for ImageFormat { fn from(format: k4a_sys::k4a_image_format_t) -> Self { match format { diff --git a/src/kinect/mod.rs b/src/kinect/mod.rs index 91899392c5..8fb401d12a 100644 --- a/src/kinect/mod.rs +++ b/src/kinect/mod.rs @@ -1,3 +1,3 @@ +pub mod capture; pub mod k4a_sys_wrapper; -pub mod multi_device_capturer; pub mod sensor_control; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index b6a840e7a6..7db9558275 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,8 @@ use opencv::prelude::*; use kinect::k4a_sys_wrapper::Device; use kinect::sensor_control::{capture_thread, CaptureProvider}; -use kinect::multi_device_capturer::{MultiDeviceCapturer, start_capture_thread}; +use kinect::capture::multi_device_capturer::{MultiDeviceCapturer, start_capture_thread}; +use kinect::capture::fake_device_capturer::FakeDeviceCaptureProvider; pub mod core_types; pub mod files; @@ -69,5 +70,7 @@ pub fn main() { //thread::spawn(move || capture_thread(capture_provider, Some(primary_device), false)); thread::spawn(move || start_capture_thread(multi_device)); + //let capture_provider = Arc::new(FakeDeviceCaptureProvider::new().unwrap()); + graphics_imgui::run(capture_provider, calibration, ENABLE_WEBCAM); } diff --git a/src/point_cloud/debug/image_proxy.rs b/src/point_cloud/debug/image_proxy.rs index ad8e51aa39..b58035d00c 100644 --- a/src/point_cloud/debug/image_proxy.rs +++ b/src/point_cloud/debug/image_proxy.rs @@ -1,8 +1,11 @@ use anyhow::Result as AnyhowResult; +use k4a_sys; +use kinect::k4a_sys_wrapper; use std::fs::File; use std::fs; use std::io::Read; -use kinect::k4a_sys_wrapper; +use kinect::k4a_sys_wrapper::ImageFormat; +use std::ptr::null_mut; /// Store either raw bytes, or wrap a k4a::Image /// This is meant to be plumbed through the system instead of a k4a::Image (depth or color image) @@ -18,14 +21,23 @@ enum UnderlyingStorage { Bytes { bytes: Vec, width: usize, - height: usize + height: usize, + stride_bytes: i32, + format: ImageFormat, }, /// From the Kinect camera K4aImage(k4a_sys_wrapper::Image), } impl ImageProxy { - pub fn from_file(filename: &str, width: usize, height: usize) -> AnyhowResult { + + pub fn from_file( + filename: &str, + width: usize, + height: usize, + stride_bytes: i32, + format: ImageFormat) -> AnyhowResult + { let mut file = File::open(filename)?; let metadata = fs::metadata(&filename)?; @@ -36,6 +48,8 @@ impl ImageProxy { bytes: buffer, width, height, + stride_bytes, + format, }; Ok(Self { @@ -105,11 +119,33 @@ impl ImageProxy { pub fn get_handle(&self) -> k4a_sys::k4a_image_t { match &self.storage { - UnderlyingStorage::Bytes { .. } => { - println!("NO K4A_IMAGE_T!"); - panic!("no k4a_image_t") + UnderlyingStorage::Bytes { bytes, width, height, stride_bytes, format } => { + // TODO: We're not handing over ownership of the buffer! + // The k4a_image cannot be used after this ImageProxy dies! + let mut handle = null_mut(); + + //println!("fake.get_handle(): {:?}, {}, {}x{}", format, stride_bytes, width, height); + + unsafe { + k4a_sys::k4a_image_create_from_buffer( + format.to_k4a(), + *width as i32, + *height as i32, + *stride_bytes, + bytes.as_ptr() as *mut u8, // TODO: Technically we won't write. + bytes.len(), + None, + null_mut(), + &mut handle, + ); + } + + handle + }, + UnderlyingStorage::K4aImage(image) => { + //println!("real.get_handle(): {:?}, {}, {}x{}", image.get_format(), image.get_stride_bytes(), image.get_width_pixels(), image.get_height_pixels()); + image.get_handle() }, - UnderlyingStorage::K4aImage(image) => image.get_handle(), } } } diff --git a/src/point_cloud/point_cloud_visualiser.rs b/src/point_cloud/point_cloud_visualiser.rs index 5aeace6965..c92b6b9be3 100644 --- a/src/point_cloud/point_cloud_visualiser.rs +++ b/src/point_cloud/point_cloud_visualiser.rs @@ -211,12 +211,12 @@ impl PointCloudVisualizer { } let mut debug_static_color_frames = Vec::new(); - //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_0", 0, 0).unwrap()); - //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_1", 0, 0).unwrap()); + //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_0", 0, 0, 0, ImageFormat::ColorBgra32).unwrap()); + //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_1", 0, 0, 0, ImageFormat::ColorBgra32).unwrap()); let mut debug_static_depth_frames = Vec::new(); - //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_0", 3840, 2160).unwrap()); - //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_1", 3840, 2160).unwrap()); + //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_0", 3840, 2160, 7680, ImageFormat::Depth16).unwrap()); + //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_1", 3840, 2160, 7680, ImageFormat::Depth16).unwrap()); let mut visualizer = Self { num_cameras, @@ -258,7 +258,7 @@ impl PointCloudVisualizer { /// /// /// - pub fn update_texture(&mut self, texture: &ViewerImage, mut captures: Vec) -> Result<()> { + pub fn update_texture(&mut self, texture: &ViewerImage, mut captures: Vec) -> Result<()> { self.update_texture_id(texture.texture_id(), captures) } @@ -266,11 +266,7 @@ impl PointCloudVisualizer { /// Take the latest capture, calculate updates to the xyz texture, then render the point cloud to /// the output texture. /// - pub fn update_texture_id(&mut self, texture_id: GLuint, mut captures: Vec) -> Result<()> { - let captures = captures.into_iter() - .map(|capture| CaptureProxy::consume_k4a_capture(capture)) - .collect(); - + pub fn update_texture_id(&mut self, texture_id: GLuint, mut captures: Vec) -> Result<()> { // Update the point cloud renderer with the latest point data self.update_point_clouds(captures)?;