From 68ea871c9f1e344dc8bbc286533236fa5cfc7a9f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 Aug 2020 20:35:31 -0400 Subject: [PATCH] move up the stack --- src/graphics_imgui.rs | 10 ++++--- src/point_cloud/debug/camera_image_bytes.rs | 20 +++++++++---- src/point_cloud/point_cloud_renderer.rs | 33 ++++----------------- src/point_cloud/point_cloud_visualiser.rs | 30 ++++++++++++++----- 4 files changed, 49 insertions(+), 44 deletions(-) diff --git a/src/graphics_imgui.rs b/src/graphics_imgui.rs index 0df1f78752..fb818fc518 100644 --- a/src/graphics_imgui.rs +++ b/src/graphics_imgui.rs @@ -138,14 +138,16 @@ pub fn run(capture_provider: Arc, calibration_data: const window_height : f32 = 760.0; if imgui_visualizer_xyz_texture_0.is_none() { - if visualizer.xyz_textures.get(0).unwrap().id() != 0 { // TODO: TEMP SUPPORT MULTI-CAMERA - imgui_visualizer_xyz_texture_0 = Some(TextureId::from(visualizer.xyz_textures.get(0).unwrap().id() as usize)); // TODO: TEMP SUPPORT MULTI-CAMERA + // TODO: TEMPORARY MESS TO SUPPORT MULTI-CAMERA + if visualizer.xyz_textures.get(0).unwrap().id() != 0 { + imgui_visualizer_xyz_texture_0 = Some(TextureId::from(visualizer.xyz_textures.get(0).unwrap().id() as usize)); } } if imgui_visualizer_xyz_texture_1.is_none() { - if visualizer.xyz_textures.get(1).unwrap().id() != 0 { // TODO: TEMP SUPPORT MULTI-CAMERA - imgui_visualizer_xyz_texture_1 = Some(TextureId::from(visualizer.xyz_textures.get(1).unwrap().id() as usize)); // TODO: TEMP SUPPORT MULTI-CAMERA + // TODO: TEMPORARY MESS TO SUPPORT MULTI-CAMERA + if visualizer.xyz_textures.get(1).unwrap().id() != 0 { + imgui_visualizer_xyz_texture_1 = Some(TextureId::from(visualizer.xyz_textures.get(1).unwrap().id() as usize)); } } diff --git a/src/point_cloud/debug/camera_image_bytes.rs b/src/point_cloud/debug/camera_image_bytes.rs index aae8008be8..3f33c8d428 100644 --- a/src/point_cloud/debug/camera_image_bytes.rs +++ b/src/point_cloud/debug/camera_image_bytes.rs @@ -5,20 +5,23 @@ use std::io::Read; use kinect::k4a_sys_wrapper; /// Store either raw bytes, or wrap a k4a::Image +/// This is meant to be plumbed through the system instead of a k4a::Image (depth or color image) +pub struct CameraImageBytes { + storage: UnderlyingStorage, +} + +/// The underlying data store enum UnderlyingStorage { + /// Our own serialization Bytes { bytes: Vec, width: usize, height: usize }, + /// From the Kinect camera K4aImage(k4a_sys_wrapper::Image), } -/// Loaded images from the filesystem -pub struct CameraImageBytes { - storage: UnderlyingStorage, -} - impl CameraImageBytes { pub fn from_file(filename: &str, width: usize, height: usize) -> AnyhowResult { let mut file = File::open(filename)?; @@ -48,6 +51,13 @@ impl CameraImageBytes { } } + pub fn is_k4a(&self) -> bool { + match &self.storage { + UnderlyingStorage::Bytes { .. } => false, + UnderlyingStorage::K4aImage(_) => true, + } + } + pub fn len(&self) -> usize { match &self.storage { UnderlyingStorage::Bytes { bytes, .. } => bytes.len(), diff --git a/src/point_cloud/point_cloud_renderer.rs b/src/point_cloud/point_cloud_renderer.rs index f10ff1b7c3..5e3c2ffb76 100644 --- a/src/point_cloud/point_cloud_renderer.rs +++ b/src/point_cloud/point_cloud_renderer.rs @@ -109,10 +109,6 @@ pub struct PointCloudRenderer { vertex_color_buffer_objects: Vec, vertex_attrib_locations: Vec, - - /// For debugging - /// If present, use these for color images instead of camera captures. - debug_static_color_frames: Vec, } const fn translation_matrix_4x4(x: f32, y: f32, z: f32) -> [f32; 16] { @@ -268,10 +264,6 @@ impl PointCloudRenderer { [0.0, 0.0, -0.2002, 0.0], ]; - let mut debug_static_color_frames = Vec::new(); - debug_static_color_frames.push(CameraImageBytes::from_file("output/color_src_0", 0, 0).unwrap()); - debug_static_color_frames.push(CameraImageBytes::from_file("output/color_src_1", 0, 0).unwrap()); - Self { num_cameras, arcball_camera: arcball, @@ -292,7 +284,6 @@ impl PointCloudRenderer { vertex_array_objects, vertex_color_buffer_objects, vertex_attrib_locations, - debug_static_color_frames, } } @@ -319,9 +310,10 @@ impl PointCloudRenderer { /// /// /// - pub fn update_point_clouds(&mut self, color_images: &Vec, - point_cloud_textures: &Vec) -> Result<()> - { + pub fn update_point_clouds(&mut self, + color_images: &Vec, + point_cloud_textures: &Vec + ) -> Result<()> { let time = SystemTime::now(); let time_since_epoch = time.duration_since(UNIX_EPOCH).unwrap(); let seconds = time_since_epoch.as_secs(); @@ -346,22 +338,7 @@ impl PointCloudRenderer { gl::BindBuffer(gl::ARRAY_BUFFER, vertex_color_buffer_object.id()); } - // Hack to hold onto the K4A ColorImageBytes wrapper. - let mut color_image_storage : Option = None; - - let color_image_bytes = if let Some(debug_color_image) = self.debug_static_color_frames.get(i) { - debug_color_image - } else { - let color_image = color_images.get(i).unwrap(); - let color_image_bytes = CameraImageBytes::from_k4a_image(&color_image); - - //let filename = format!("output/color_src_{}", i); - //write_to_file_from_byte_ptr(&filename, color_image_bytes.as_ptr(), color_image_bytes.len()).unwrap(); - - color_image_storage = Some(color_image_bytes); - color_image_storage.as_ref().unwrap() - }; - + let color_image_bytes = color_images.get(i).unwrap(); let color_image_size_bytes = color_image_bytes.len() as i32; if *vertex_array_size_bytes != color_image_size_bytes { diff --git a/src/point_cloud/point_cloud_visualiser.rs b/src/point_cloud/point_cloud_visualiser.rs index 5004af1bd5..7bdbee038f 100644 --- a/src/point_cloud/point_cloud_visualiser.rs +++ b/src/point_cloud/point_cloud_visualiser.rs @@ -136,7 +136,8 @@ pub struct PointCloudVisualizer { clear_color: RgbaF32, /// For debugging - /// If present, use these for depth images instead of camera captures. + /// If present, use these instead of camera captures. + debug_static_color_frames: Vec, debug_static_depth_frames: Vec, } @@ -208,6 +209,10 @@ impl PointCloudVisualizer { depth_xy_tables.push(depth_xy_table); } + let mut debug_static_color_frames = Vec::new(); + debug_static_color_frames.push(CameraImageBytes::from_file("output/color_src_0", 0, 0).unwrap()); + debug_static_color_frames.push(CameraImageBytes::from_file("output/color_src_1", 0, 0).unwrap()); + let mut debug_static_depth_frames = Vec::new(); debug_static_depth_frames.push(CameraImageBytes::from_file("output/depth_src_0", 3840, 2160).unwrap()); debug_static_depth_frames.push(CameraImageBytes::from_file("output/depth_src_1", 3840, 2160).unwrap()); @@ -233,6 +238,7 @@ impl PointCloudVisualizer { xyz_textures, depth_value_range: expected_value_range, clear_color, + debug_static_color_frames, debug_static_depth_frames, }; @@ -339,14 +345,24 @@ impl PointCloudVisualizer { self.update_point_clouds_for_camera(camera_index, capture)?; } - // TODO: wrt k4a::Image.clone(), k4a should use ref counting. (I did this one other place a few commits ago. - // Hope it's not ref leaking.) - let colorizations: Vec = self.point_cloud_colorizations.iter() - .map(|img| img.as_ref().map(|img| img.clone()).unwrap()) - .collect(); + let mut color_captures = Vec::new(); + + let color_frames = if self.debug_static_color_frames.len() > 0 { + &self.debug_static_color_frames + } else { + // TODO: wrt k4a::Image.clone(), k4a should use ref counting. (I did this one other place a few commits ago. + // Hope it's not ref leaking.) + let colorizations: Vec = self.point_cloud_colorizations.iter() + .map(|img| img.as_ref().map(|img| img.clone()).unwrap()) // Potential ref leak #1 + .map(|image| CameraImageBytes::from_k4a_image(&image)) // Potential ref leak #2 + .collect(); + + color_captures = colorizations; + &color_captures + }; self.point_cloud_renderer.update_point_clouds( - &colorizations, + &color_frames, &self.xyz_textures ).map_err(|err| PointCloudVisualizerError::PointCloudRendererError(err)) }