better k4a image refcount handling, lots of logging, ...

src/graphics_gl.rs

@@ -120,7 +120,8 @@ extern "system" fn glDebugOutput(
   userParam: *mut c_void)
 {
   let message = unsafe { CStr::from_ptr(message) };
-  println!(">>> GL DEBUG = {:?}", message);
+  println!(">>> [GL DEBUG] id: {:?} message: {:?} source: {:?} type: {:?} severity: {:?}",
+    id, message, source, gltype, severity);
 }
 
 pub fn run(capture_provider: Arc<CaptureProvider>, calibration_data: k4a_sys::k4a_calibration_t) {
@@ -136,7 +137,10 @@ pub fn run(capture_provider: Arc<CaptureProvider>, calibration_data: k4a_sys::k4
   // Load the OpenGL function pointers
   gl::load_with(|symbol| gl_window.get_proc_address(symbol));
 
-  let mut visualizer = PointCloudVisualizer::new(false, calibration_data);
+  let mut visualizer = PointCloudVisualizer::new(
+    true,
+    calibration_data
+  );
 
   // TODO - constructed in PointCloudWindow.
   /*
@@ -148,9 +152,11 @@ pub fn run(capture_provider: Arc<CaptureProvider>, calibration_data: k4a_sys::k4
   */
 
   unsafe {
+    // Set debugging. This can be expensive, so we should be able to trigger it with a flag.
     gl::Enable(gl::DEBUG_OUTPUT);
     gl::Enable(gl::DEBUG_OUTPUT_SYNCHRONOUS);
     gl::DebugMessageCallback(Some(glDebugOutput), null());
+    // Opt into everything.
     gl::DebugMessageControl(gl::DONT_CARE, gl::DONT_CARE, gl::DONT_CARE, 0, null(), gl::TRUE);
   }
 
@@ -279,7 +285,7 @@ pub fn run(capture_provider: Arc<CaptureProvider>, calibration_data: k4a_sys::k4
 
   event_loop.run(move |event, window, control_flow| {
     //*control_flow = ControlFlow::Wait;
-    println!("\n\n------- event loop ------");
+    //println!("\n\n------- event loop ------");
     match event {
       Event::LoopDestroyed => return,
       Event::WindowEvent { event, .. } => match event {

src/k4a_sys_wrapper.rs

@@ -378,6 +378,48 @@ impl Image {
     };
     format.into()
   }
+
+  /// Returns the underlying opaque handle *without* an additional refcount.
+  /// Do not deallocate it.
+  pub fn get_handle(&self) -> k4a_sys::k4a_image_t {
+    self.0
+  }
+}
+
+/// Remove a libk4a image refcount on every drop.
+/// When the refcount drops to zero, the image goes away.
+impl Drop for Image {
+  fn drop(&mut self) {
+    unsafe {
+      k4a_sys::k4a_image_release(self.0);
+    }
+  }
+}
+
+/// Handles are refcounted by libk4a. The final reference is destroyed
+impl Clone for Image {
+  fn clone(&self) -> Self {
+    // We must increment the refcount.
+    let handle = self.get_handle();
+    unsafe {
+      k4a_sys::k4a_image_reference(handle);
+    }
+    Self {
+      0: handle,
+    }
+  }
+
+  fn clone_from(&mut self, source: &Self) {
+    let handle = source.get_handle();
+    unsafe {
+      k4a_sys::k4a_image_reference(handle);
+    }
+    unsafe {
+      // drop refcount, potentially releasing if reached zero
+      k4a_sys::k4a_image_release(self.0);
+    }
+    self.0 = handle;
+  }
 }
 
 #[derive(Debug,Clone,Copy)]
@@ -425,14 +467,6 @@ impl Drop for Device {
   }
 }
 
-impl Drop for Image {
-  fn drop(&mut self) {
-    unsafe {
-      k4a_sys::k4a_image_release(self.0);
-    }
-  }
-}
-
 /// Copied from k4a-sys
 pub struct DeviceConfiguration (k4a_sys::k4a_device_configuration_t);
 
@@ -484,6 +518,12 @@ impl Transformation {
       },
     }
   }
+
+  /// Returns the underlying opaque handle *without* an additional refcount.
+  /// Do not deallocate it.
+  pub fn get_handle(&self) -> k4a_sys::k4a_transformation_t {
+    self.transformation
+  }
 }
 
 impl Drop for Transformation {

src/point_cloud/point_cloud_compute_shader.rs

@@ -144,6 +144,10 @@ impl PointCloudComputeShader {
       depth_image_id = gl::GetUniformLocation(program_id, DEPTH_IMAGE_PTR);
     }
 
+    println!("Uniform: dest_tex_id location = {:?}", dest_tex_id);
+    println!("Uniform: xy_table_id location = {:?}", xy_table_id);
+    println!("Uniform: depth_image_id location = {:?}", depth_image_id);
+
     PointCloudComputeShader {
       program_id,
       shader_id,

src/point_cloud/point_cloud_renderer_shader.rs

@@ -92,7 +92,8 @@ void main()
     ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize.x, gl_VertexID / pointCloudSize.x);
     vec3 vertexPosition = imageLoad(pointCloudTexture, currentDepthPixelCoordinates).xyz;
 
-    gl_Position = projection * view * vec4(vertexPosition, 1);
+    //gl_Position = projection * view * vec4(vertexPosition, 1);
+    gl_Position = vec4(vertexPosition, 1);
 
     vertexColor = inColor;
 
@@ -265,12 +266,21 @@ impl PointCloudRendererShader {
     let mut point_cloud_texture_index = 0;
 
     unsafe {
+      // This function returns -1 if name does not correspond to an active uniform variable in
+      // program, if name starts with the reserved prefix "gl_", or if name is associated with an
+      // atomic counter or a named uniform block.
+      // FIXME: THe 'view' and 'projection' uniforms are not binding for some reason.
       view_index = gl::GetUniformLocation(program_id, VIEW_PTR);
       projection_index = gl::GetUniformLocation(program_id, PROJECTION_PTR);
       enable_shading_index = gl::GetUniformLocation(program_id, ENABLE_SHADING_PTR);
       point_cloud_texture_index = gl::GetUniformLocation(program_id, POINT_CLOUD_PTR);
     }
 
+    println!("Uniform view location - view: {:?}", view_index);
+    println!("Uniform projection location - view: {:?}", projection_index);
+    println!("Uniform enable shading location - view: {:?}", enable_shading_index);
+    println!("Uniform point cloud texture location - view: {:?}", point_cloud_texture_index);
+
     Self {
       program_id,
       vertex_shader_id,
@@ -299,22 +309,25 @@ impl PointCloudRendererShader {
   pub fn update_point_clouds(&mut self, color_image: &k4a_sys_wrapper::Image,
                              point_cloud_texture: &Texture) -> Result<()>
   {
-    println!("Renderer.update_point_clouds()");
+    println!("==== Renderer.update_point_clouds() [buffering data]");
 
     unsafe {
+      println!("-> gl::BindVertexArray(): {:?}", self.vertex_array_object.id());
       gl::BindVertexArray(self.vertex_array_object.id());
 
       // Vertex Colors
+      println!("-> gl::BindBuffer(): {:?}", self.vertex_color_buffer_object.id());
       gl::BindBuffer(gl::ARRAY_BUFFER, self.vertex_color_buffer_object.id());
     }
 
     let color_image_size_bytes = color_image.get_size() as i32;
 
-    if self.vertex_array_size_bytes != color_image_size_bytes {
+    if self.vertex_array_size_bytes != color_image_size_bytes /* TODO TEMP ENABLE W/ +1 */ {
       println!("Updating vertex array size bytes: {}", color_image_size_bytes);
       self.vertex_array_size_bytes = color_image_size_bytes;
 
       unsafe {
+        println!("-> gl::BufferData(...)");
         gl::BufferData(
           gl::ARRAY_BUFFER,
           self.vertex_array_size_bytes as isize,
@@ -328,6 +341,9 @@ impl PointCloudRendererShader {
 
     let vertex_mapped_buffer = unsafe {
       // GLubyte *vertexMappedBuffer = reinterpret_cast<GLubyte *>(
+      println!("-> gl::MapBufferRange(...) [puts the buffer into client addr space] size = {}",
+        color_image_size_bytes);
+
       gl::MapBufferRange(
         gl::ARRAY_BUFFER,
         0,
@@ -343,11 +359,16 @@ impl PointCloudRendererShader {
       return Err(PointCloudRendererError::UnknownError);
     }
 
+    println!("color image: handle = {:?} size = {}x{}",
+      color_image.0,
+      color_image.get_width_pixels(),
+      color_image.get_height_pixels());
+
     let mut color_src = color_image.get_buffer();
 
     unsafe {
       println!(
-        "Color image bytes: {:?} {:?} {:?} {:?} {:?} {:?}",
+        "color image bytes: {:?} {:?} {:?} {:?} {:?} {:?}",
         *color_src.offset(0),
         *color_src.offset(100),
         *color_src.offset(1000),
@@ -362,16 +383,22 @@ impl PointCloudRendererShader {
       //std::copy(colorSrc, colorSrc + colorImageSizeBytes, vertexMappedBuffer);
       std::ptr::copy_nonoverlapping::<u8>(color_src, vertex_mapped_buffer as *mut u8,
         color_image_size_bytes as usize);
+      std::ptr::copy::<u8>(color_src, vertex_mapped_buffer as *mut u8,
+        color_image_size_bytes as usize);
 
+      println!("-> gl::UnmapBuffer()");
       gl::UnmapBuffer(gl::ARRAY_BUFFER)
     };
 
     if result == gl::FALSE {
+      println!("Failure copying data / unmapping buffer");
       return Err(PointCloudRendererError::UnknownError);
     }
 
     unsafe {
+      println!("-> gl::EnableVertexAttribArray(0)");
       gl::EnableVertexAttribArray(0);
+      println!("-> gl::VertexAttribPointer(...)");
       gl::VertexAttribPointer(
         0,
         gl::BGRA as i32,
@@ -380,14 +407,18 @@ impl PointCloudRendererShader {
         get_stride::<f32>(0),
         get_pointer_offset::<f32>(0),
       );
+      println!("-> gl::UseProgram(): {:?}", self.program_id);
       gl::UseProgram(self.program_id);
     }
 
     // Uniforms
     // Bind our point cloud texture
     unsafe {
+      println!("-> gl::ActiveTexture(TEXTURE0)");
       gl::ActiveTexture(gl::TEXTURE0);
+      println!("-> gl::BindTexture(): {:?}", point_cloud_texture.id());
       gl::BindTexture(gl::TEXTURE_2D, point_cloud_texture.id());
+      println!("-> gl::BindImageTexture(...)");
       gl::BindImageTexture(
         0,
         point_cloud_texture.id(),
@@ -397,8 +428,10 @@ impl PointCloudRendererShader {
         gl::READ_ONLY,
         POINT_CLOUD_TEXTURE_FORMAT,
       );
+      println!("-> gl::Uniform1i(): {:?}", self.point_cloud_texture_index);
       gl::Uniform1i(self.point_cloud_texture_index, 0);
 
+      println!("-> gl::BindVertexArray(0)");
       gl::BindVertexArray(0);
     }
 
@@ -407,13 +440,19 @@ impl PointCloudRendererShader {
   }
 
   pub fn render(&self) -> Result<()> {
+    println!("==== Renderer.render() [the draw call]");
     unsafe {
+      println!("-> gl::Enable(DEPTH_TEST)");
       gl::Enable(gl::DEPTH_TEST);
+      println!("-> gl::Enable(BLEND)");
       gl::Enable(gl::BLEND);
+      println!("-> gl::BlendFunc(...)");
       gl::BlendFunc(gl::SRC_ALPHA, gl::ONE_MINUS_SRC_ALPHA);
 
+      println!("-> gl::PointSize(): {:?}", self.point_size as f32);
       gl::PointSize(self.point_size as f32);
 
+      println!("-> gl::UseProgram(): {:?}", self.program_id);
       gl::UseProgram(self.program_id);
 
       // TODO: view and projection matrices
@@ -423,24 +462,30 @@ impl PointCloudRendererShader {
 
       // Update render settings in shader
       let enable_shading = if self.enable_shading { 1 } else { 0 };
-      gl::Uniform1i(self.enable_shading_index, enable_shading);
 
       // TODO: Not sure what is setting this false.
       let enable_shading = 1;
       println!("Enable shading: {} -> {}", self.enable_shading, enable_shading);
 
+
+      println!("-> gl::Uniform1i(): {:?}", self.enable_shading_index);
+      gl::Uniform1i(self.enable_shading_index, enable_shading);
+
       // glDrawArrays(GL_POINTS, 0, m_vertexArraySizeBytes / static_cast<GLsizei>(sizeof(BgraPixel)));
       let size = self.vertex_array_size_bytes / size_of::<BgraPixel>() as i32;
 
       // Render point cloud
+      println!("-> gl::BindVertexArray(): {:?}", self.vertex_array_object.id());
       gl::BindVertexArray(self.vertex_array_object.id());
-      println!("gl::DrawArrays - points in the point cloud; size = {}", size);
+      println!("gl::DrawArrays - points in the point cloud; size = {} (indices to be rendered)", size);
+      println!("-> gl::DrawArrays(...)");
       gl::DrawArrays(
         gl::POINTS,
         0,
         size,
       );
 
+      println!("-> gl::BindVertexArray(0)");
       gl::BindVertexArray(0);
 
       gl_get_error()

src/point_cloud/point_cloud_visualiser.rs

@@ -118,6 +118,7 @@ impl Drop for CleanupGuard {
   fn drop(&mut self) {
     println!("Running CleanupGuard");
     unsafe {
+      println!("-> gl::BindFramebuffer(0) [cleanup]");
       gl::BindFramebuffer(gl::FRAMEBUFFER, 0);
     }
   }
@@ -167,6 +168,8 @@ impl PointCloudVisualizer {
       0
     ).expect("should allocate");*/
 
+    let color_strategy = ColorizationStrategy::Color;
+
     let mut visualizer = Self {
       width: width as u16,
       height: height as u16,
@@ -177,7 +180,7 @@ impl PointCloudVisualizer {
       depth_buffer,
       color_xy_table,
       depth_xy_table,
-      colorization_strategy: ColorizationStrategy::Simple,
+      colorization_strategy: color_strategy,
       calibration_data: calibration_data.clone(),
       transformation: Transformation::from_calibration(&calibration_data),
       last_capture: None,
@@ -186,7 +189,7 @@ impl PointCloudVisualizer {
       xyz_texture: Texture::new(),
     };
 
-    visualizer.set_colorization_strategy(ColorizationStrategy::Simple).expect("Should work");
+    visualizer.set_colorization_strategy(color_strategy).expect("Should work");
 
     visualizer
   }
@@ -196,13 +199,16 @@ impl PointCloudVisualizer {
   }
 
   pub fn update_texture(&mut self, texture: &ViewerImage, capture: Capture) -> Result<()> {
+    println!("==== Visualizer.update_texture()");
 
     // Update the point cloud renderer with the latest point data
     self.update_point_clouds(capture)?;
 
     // Set up rendering to a texture
     unsafe {
+      println!("-> gl::BindRenderbuffer(): {:?}", self.depth_buffer.id());
       gl::BindRenderbuffer(gl::RENDERBUFFER, self.depth_buffer.id());
+      println!("-> gl::BindFramebuffer(): {:?}", self.frame_buffer.id());
       gl::BindFramebuffer(gl::FRAMEBUFFER, self.frame_buffer.id());
     }
 
@@ -210,25 +216,31 @@ impl PointCloudVisualizer {
     let cleanup_guard = CleanupGuard {};
 
     unsafe {
+      println!("-> gl::FramebufferRenderbuffer(...)");
       gl::FramebufferRenderbuffer(gl::FRAMEBUFFER, gl::DEPTH_ATTACHMENT, gl::RENDERBUFFER, self.depth_buffer.id());
       // TODO:
       //  glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, static_cast<GLuint>(**texture), 0);
       //  gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture_buffer, 0);
-      //gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture, 0);
+      gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture.texture_id(), 0);
 
-      println!("gl::DrawBuffers");
+      println!("-> gl::DrawBuffers(COLOR_ATTACHMENT0)");
       gl::DrawBuffers(1, &gl::COLOR_ATTACHMENT0);
 
+      println!("-> gl::CheckFramebufferStatus()");
       let frame_buffer_status = gl::CheckFramebufferStatus(gl::FRAMEBUFFER);
 
       if frame_buffer_status != gl::FRAMEBUFFER_COMPLETE {
         return Err(PointCloudVisualizerError::FramebufferError);
       }
 
+      println!("-> gl::Viewport() {}x{}", self.width, self.height);
       gl::Viewport(0, 0, self.width as i32, self.height as i32);
 
+      println!("-> gl::Enable(DEPTH_TEST");
       gl::Enable(gl::DEPTH_TEST);
+      println!("-> gl::ClearColor()");
       gl::ClearColor(0.0, 0.0, 0.0, 0.0);
+      println!("-> gl::Clear(COLOR,DEPTH)");
       gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);
     }
 
@@ -238,6 +250,7 @@ impl PointCloudVisualizer {
     let render_status = self.point_cloud_renderer.render();
 
     unsafe {
+      println!("-> gl::BindRenderbuffer(RENDERBUFFER, 0)");
       gl::BindRenderbuffer(gl::RENDERBUFFER, 0);
     }
 
@@ -246,9 +259,13 @@ impl PointCloudVisualizer {
   }
 
   fn update_point_clouds(&mut self, capture: Capture) -> Result<()> {
-    let depth_image = match capture.get_depth_image() {
+    println!("==== Visualizer.update_point_clouds() [from kinect capture]");
+    println!("colorization strategy = {:?}", self.colorization_strategy);
+
+    let mut depth_image = match capture.get_depth_image() {
       Ok(img) => img,
       Err(e) => {
+        println!("<< Missing Depth Image >> ");
         // Capture doesn't have depth info. Drop the capture.
         return Err(PointCloudVisualizerError::MissingDepthImage);
       },
@@ -258,24 +275,27 @@ impl PointCloudVisualizer {
 
     if self.enable_color_point_cloud {
       if maybe_color_image.is_err() {
+        println!("<< Missing Color Image >> ");
         return Err(PointCloudVisualizerError::MissingColorImage);
       }
       if self.colorization_strategy == ColorizationStrategy::Color {
-        // TODO
-        //  m_transformation.depth_image_to_color_camera(depthImage, &m_transformedDepthImage);
-        //  depthImage = m_transformedDepthImage;
-        /*
-            try
-            {
-                m_transformation.depth_image_to_color_camera(depthImage, &m_transformedDepthImage);
-                depthImage = m_transformedDepthImage;
-            }
-            catch (const k4a::error &)
-            {
-                return PointCloudVisualizationResult::DepthToColorTransformationFailed;
-            }
-        */
-        return Err(PointCloudVisualizerError::ColorSupportNotYetImplemented);
+
+        /*let mut transformed_depth_image = self.transformed_depth_image.take()
+            .expect("Must be present");*/
+
+        if let Some(transformed_depth_image) = self.transformed_depth_image.as_ref() {
+          println!("Take transformed depth image...");
+          unsafe {
+            // TODO: Totally missed error handling here.
+            k4a_sys::k4a_transformation_depth_image_to_color_camera(
+              self.transformation.get_handle(),
+              depth_image.get_handle(),
+              transformed_depth_image.get_handle(),
+            );
+
+            depth_image = transformed_depth_image.clone();
+          }
+        }
       }
     }