From 2df1ba36cc2b0650fb2e9818b5db19eb5e1bd44b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 19 Apr 2020 05:04:52 -0400 Subject: [PATCH] better k4a image refcount handling, lots of logging, ... --- src/graphics_gl.rs | 12 +++- src/k4a_sys_wrapper.rs | 56 ++++++++++++++--- src/point_cloud/point_cloud_compute_shader.rs | 4 ++ .../point_cloud_renderer_shader.rs | 57 ++++++++++++++++-- src/point_cloud/point_cloud_visualiser.rs | 60 ++++++++++++------- 5 files changed, 152 insertions(+), 37 deletions(-) diff --git a/src/graphics_gl.rs b/src/graphics_gl.rs index 50d2bc5162..5541ac8c2e 100644 --- a/src/graphics_gl.rs +++ b/src/graphics_gl.rs @@ -120,7 +120,8 @@ extern "system" fn glDebugOutput( userParam: *mut c_void) { let message = unsafe { CStr::from_ptr(message) }; - println!(">>> GL DEBUG = {:?}", message); + println!(">>> [GL DEBUG] id: {:?} message: {:?} source: {:?} type: {:?} severity: {:?}", + id, message, source, gltype, severity); } pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4a_calibration_t) { @@ -136,7 +137,10 @@ pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4 // Load the OpenGL function pointers gl::load_with(|symbol| gl_window.get_proc_address(symbol)); - let mut visualizer = PointCloudVisualizer::new(false, calibration_data); + let mut visualizer = PointCloudVisualizer::new( + true, + calibration_data + ); // TODO - constructed in PointCloudWindow. /* @@ -148,9 +152,11 @@ pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4 */ unsafe { + // Set debugging. This can be expensive, so we should be able to trigger it with a flag. gl::Enable(gl::DEBUG_OUTPUT); gl::Enable(gl::DEBUG_OUTPUT_SYNCHRONOUS); gl::DebugMessageCallback(Some(glDebugOutput), null()); + // Opt into everything. gl::DebugMessageControl(gl::DONT_CARE, gl::DONT_CARE, gl::DONT_CARE, 0, null(), gl::TRUE); } @@ -279,7 +285,7 @@ pub fn run(capture_provider: Arc, calibration_data: k4a_sys::k4 event_loop.run(move |event, window, control_flow| { //*control_flow = ControlFlow::Wait; - println!("\n\n------- event loop ------"); + //println!("\n\n------- event loop ------"); match event { Event::LoopDestroyed => return, Event::WindowEvent { event, .. } => match event { diff --git a/src/k4a_sys_wrapper.rs b/src/k4a_sys_wrapper.rs index 8cc0f9c9f7..cd5ea7f31f 100644 --- a/src/k4a_sys_wrapper.rs +++ b/src/k4a_sys_wrapper.rs @@ -378,6 +378,48 @@ impl Image { }; format.into() } + + /// Returns the underlying opaque handle *without* an additional refcount. + /// Do not deallocate it. + pub fn get_handle(&self) -> k4a_sys::k4a_image_t { + self.0 + } +} + +/// Remove a libk4a image refcount on every drop. +/// When the refcount drops to zero, the image goes away. +impl Drop for Image { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_image_release(self.0); + } + } +} + +/// Handles are refcounted by libk4a. The final reference is destroyed +impl Clone for Image { + fn clone(&self) -> Self { + // We must increment the refcount. + let handle = self.get_handle(); + unsafe { + k4a_sys::k4a_image_reference(handle); + } + Self { + 0: handle, + } + } + + fn clone_from(&mut self, source: &Self) { + let handle = source.get_handle(); + unsafe { + k4a_sys::k4a_image_reference(handle); + } + unsafe { + // drop refcount, potentially releasing if reached zero + k4a_sys::k4a_image_release(self.0); + } + self.0 = handle; + } } #[derive(Debug,Clone,Copy)] @@ -425,14 +467,6 @@ impl Drop for Device { } } -impl Drop for Image { - fn drop(&mut self) { - unsafe { - k4a_sys::k4a_image_release(self.0); - } - } -} - /// Copied from k4a-sys pub struct DeviceConfiguration (k4a_sys::k4a_device_configuration_t); @@ -484,6 +518,12 @@ impl Transformation { }, } } + + /// Returns the underlying opaque handle *without* an additional refcount. + /// Do not deallocate it. + pub fn get_handle(&self) -> k4a_sys::k4a_transformation_t { + self.transformation + } } impl Drop for Transformation { diff --git a/src/point_cloud/point_cloud_compute_shader.rs b/src/point_cloud/point_cloud_compute_shader.rs index 5ef4022ccc..8d36738f21 100644 --- a/src/point_cloud/point_cloud_compute_shader.rs +++ b/src/point_cloud/point_cloud_compute_shader.rs @@ -144,6 +144,10 @@ impl PointCloudComputeShader { depth_image_id = gl::GetUniformLocation(program_id, DEPTH_IMAGE_PTR); } + println!("Uniform: dest_tex_id location = {:?}", dest_tex_id); + println!("Uniform: xy_table_id location = {:?}", xy_table_id); + println!("Uniform: depth_image_id location = {:?}", depth_image_id); + PointCloudComputeShader { program_id, shader_id, diff --git a/src/point_cloud/point_cloud_renderer_shader.rs b/src/point_cloud/point_cloud_renderer_shader.rs index 42c9a8df6b..a22bb3529b 100644 --- a/src/point_cloud/point_cloud_renderer_shader.rs +++ b/src/point_cloud/point_cloud_renderer_shader.rs @@ -92,7 +92,8 @@ void main() ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize.x, gl_VertexID / pointCloudSize.x); vec3 vertexPosition = imageLoad(pointCloudTexture, currentDepthPixelCoordinates).xyz; - gl_Position = projection * view * vec4(vertexPosition, 1); + //gl_Position = projection * view * vec4(vertexPosition, 1); + gl_Position = vec4(vertexPosition, 1); vertexColor = inColor; @@ -265,12 +266,21 @@ impl PointCloudRendererShader { let mut point_cloud_texture_index = 0; unsafe { + // This function returns -1 if name does not correspond to an active uniform variable in + // program, if name starts with the reserved prefix "gl_", or if name is associated with an + // atomic counter or a named uniform block. + // FIXME: THe 'view' and 'projection' uniforms are not binding for some reason. view_index = gl::GetUniformLocation(program_id, VIEW_PTR); projection_index = gl::GetUniformLocation(program_id, PROJECTION_PTR); enable_shading_index = gl::GetUniformLocation(program_id, ENABLE_SHADING_PTR); point_cloud_texture_index = gl::GetUniformLocation(program_id, POINT_CLOUD_PTR); } + println!("Uniform view location - view: {:?}", view_index); + println!("Uniform projection location - view: {:?}", projection_index); + println!("Uniform enable shading location - view: {:?}", enable_shading_index); + println!("Uniform point cloud texture location - view: {:?}", point_cloud_texture_index); + Self { program_id, vertex_shader_id, @@ -299,22 +309,25 @@ impl PointCloudRendererShader { pub fn update_point_clouds(&mut self, color_image: &k4a_sys_wrapper::Image, point_cloud_texture: &Texture) -> Result<()> { - println!("Renderer.update_point_clouds()"); + println!("==== Renderer.update_point_clouds() [buffering data]"); unsafe { + println!("-> gl::BindVertexArray(): {:?}", self.vertex_array_object.id()); gl::BindVertexArray(self.vertex_array_object.id()); // Vertex Colors + println!("-> gl::BindBuffer(): {:?}", self.vertex_color_buffer_object.id()); gl::BindBuffer(gl::ARRAY_BUFFER, self.vertex_color_buffer_object.id()); } let color_image_size_bytes = color_image.get_size() as i32; - if self.vertex_array_size_bytes != color_image_size_bytes { + if self.vertex_array_size_bytes != color_image_size_bytes /* TODO TEMP ENABLE W/ +1 */ { println!("Updating vertex array size bytes: {}", color_image_size_bytes); self.vertex_array_size_bytes = color_image_size_bytes; unsafe { + println!("-> gl::BufferData(...)"); gl::BufferData( gl::ARRAY_BUFFER, self.vertex_array_size_bytes as isize, @@ -328,6 +341,9 @@ impl PointCloudRendererShader { let vertex_mapped_buffer = unsafe { // GLubyte *vertexMappedBuffer = reinterpret_cast( + println!("-> gl::MapBufferRange(...) [puts the buffer into client addr space] size = {}", + color_image_size_bytes); + gl::MapBufferRange( gl::ARRAY_BUFFER, 0, @@ -343,11 +359,16 @@ impl PointCloudRendererShader { return Err(PointCloudRendererError::UnknownError); } + println!("color image: handle = {:?} size = {}x{}", + color_image.0, + color_image.get_width_pixels(), + color_image.get_height_pixels()); + let mut color_src = color_image.get_buffer(); unsafe { println!( - "Color image bytes: {:?} {:?} {:?} {:?} {:?} {:?}", + "color image bytes: {:?} {:?} {:?} {:?} {:?} {:?}", *color_src.offset(0), *color_src.offset(100), *color_src.offset(1000), @@ -362,16 +383,22 @@ impl PointCloudRendererShader { //std::copy(colorSrc, colorSrc + colorImageSizeBytes, vertexMappedBuffer); std::ptr::copy_nonoverlapping::(color_src, vertex_mapped_buffer as *mut u8, color_image_size_bytes as usize); + std::ptr::copy::(color_src, vertex_mapped_buffer as *mut u8, + color_image_size_bytes as usize); + println!("-> gl::UnmapBuffer()"); gl::UnmapBuffer(gl::ARRAY_BUFFER) }; if result == gl::FALSE { + println!("Failure copying data / unmapping buffer"); return Err(PointCloudRendererError::UnknownError); } unsafe { + println!("-> gl::EnableVertexAttribArray(0)"); gl::EnableVertexAttribArray(0); + println!("-> gl::VertexAttribPointer(...)"); gl::VertexAttribPointer( 0, gl::BGRA as i32, @@ -380,14 +407,18 @@ impl PointCloudRendererShader { get_stride::(0), get_pointer_offset::(0), ); + println!("-> gl::UseProgram(): {:?}", self.program_id); gl::UseProgram(self.program_id); } // Uniforms // Bind our point cloud texture unsafe { + println!("-> gl::ActiveTexture(TEXTURE0)"); gl::ActiveTexture(gl::TEXTURE0); + println!("-> gl::BindTexture(): {:?}", point_cloud_texture.id()); gl::BindTexture(gl::TEXTURE_2D, point_cloud_texture.id()); + println!("-> gl::BindImageTexture(...)"); gl::BindImageTexture( 0, point_cloud_texture.id(), @@ -397,8 +428,10 @@ impl PointCloudRendererShader { gl::READ_ONLY, POINT_CLOUD_TEXTURE_FORMAT, ); + println!("-> gl::Uniform1i(): {:?}", self.point_cloud_texture_index); gl::Uniform1i(self.point_cloud_texture_index, 0); + println!("-> gl::BindVertexArray(0)"); gl::BindVertexArray(0); } @@ -407,13 +440,19 @@ impl PointCloudRendererShader { } pub fn render(&self) -> Result<()> { + println!("==== Renderer.render() [the draw call]"); unsafe { + println!("-> gl::Enable(DEPTH_TEST)"); gl::Enable(gl::DEPTH_TEST); + println!("-> gl::Enable(BLEND)"); gl::Enable(gl::BLEND); + println!("-> gl::BlendFunc(...)"); gl::BlendFunc(gl::SRC_ALPHA, gl::ONE_MINUS_SRC_ALPHA); + println!("-> gl::PointSize(): {:?}", self.point_size as f32); gl::PointSize(self.point_size as f32); + println!("-> gl::UseProgram(): {:?}", self.program_id); gl::UseProgram(self.program_id); // TODO: view and projection matrices @@ -423,24 +462,30 @@ impl PointCloudRendererShader { // Update render settings in shader let enable_shading = if self.enable_shading { 1 } else { 0 }; - gl::Uniform1i(self.enable_shading_index, enable_shading); // TODO: Not sure what is setting this false. let enable_shading = 1; println!("Enable shading: {} -> {}", self.enable_shading, enable_shading); + + println!("-> gl::Uniform1i(): {:?}", self.enable_shading_index); + gl::Uniform1i(self.enable_shading_index, enable_shading); + // glDrawArrays(GL_POINTS, 0, m_vertexArraySizeBytes / static_cast(sizeof(BgraPixel))); let size = self.vertex_array_size_bytes / size_of::() as i32; // Render point cloud + println!("-> gl::BindVertexArray(): {:?}", self.vertex_array_object.id()); gl::BindVertexArray(self.vertex_array_object.id()); - println!("gl::DrawArrays - points in the point cloud; size = {}", size); + println!("gl::DrawArrays - points in the point cloud; size = {} (indices to be rendered)", size); + println!("-> gl::DrawArrays(...)"); gl::DrawArrays( gl::POINTS, 0, size, ); + println!("-> gl::BindVertexArray(0)"); gl::BindVertexArray(0); gl_get_error() diff --git a/src/point_cloud/point_cloud_visualiser.rs b/src/point_cloud/point_cloud_visualiser.rs index cfbeb5353e..409e46c162 100644 --- a/src/point_cloud/point_cloud_visualiser.rs +++ b/src/point_cloud/point_cloud_visualiser.rs @@ -118,6 +118,7 @@ impl Drop for CleanupGuard { fn drop(&mut self) { println!("Running CleanupGuard"); unsafe { + println!("-> gl::BindFramebuffer(0) [cleanup]"); gl::BindFramebuffer(gl::FRAMEBUFFER, 0); } } @@ -167,6 +168,8 @@ impl PointCloudVisualizer { 0 ).expect("should allocate");*/ + let color_strategy = ColorizationStrategy::Color; + let mut visualizer = Self { width: width as u16, height: height as u16, @@ -177,7 +180,7 @@ impl PointCloudVisualizer { depth_buffer, color_xy_table, depth_xy_table, - colorization_strategy: ColorizationStrategy::Simple, + colorization_strategy: color_strategy, calibration_data: calibration_data.clone(), transformation: Transformation::from_calibration(&calibration_data), last_capture: None, @@ -186,7 +189,7 @@ impl PointCloudVisualizer { xyz_texture: Texture::new(), }; - visualizer.set_colorization_strategy(ColorizationStrategy::Simple).expect("Should work"); + visualizer.set_colorization_strategy(color_strategy).expect("Should work"); visualizer } @@ -196,13 +199,16 @@ impl PointCloudVisualizer { } pub fn update_texture(&mut self, texture: &ViewerImage, capture: Capture) -> Result<()> { + println!("==== Visualizer.update_texture()"); // Update the point cloud renderer with the latest point data self.update_point_clouds(capture)?; // Set up rendering to a texture unsafe { + println!("-> gl::BindRenderbuffer(): {:?}", self.depth_buffer.id()); gl::BindRenderbuffer(gl::RENDERBUFFER, self.depth_buffer.id()); + println!("-> gl::BindFramebuffer(): {:?}", self.frame_buffer.id()); gl::BindFramebuffer(gl::FRAMEBUFFER, self.frame_buffer.id()); } @@ -210,25 +216,31 @@ impl PointCloudVisualizer { let cleanup_guard = CleanupGuard {}; unsafe { + println!("-> gl::FramebufferRenderbuffer(...)"); gl::FramebufferRenderbuffer(gl::FRAMEBUFFER, gl::DEPTH_ATTACHMENT, gl::RENDERBUFFER, self.depth_buffer.id()); // TODO: // glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, static_cast(**texture), 0); // gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture_buffer, 0); - //gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture, 0); + gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture.texture_id(), 0); - println!("gl::DrawBuffers"); + println!("-> gl::DrawBuffers(COLOR_ATTACHMENT0)"); gl::DrawBuffers(1, &gl::COLOR_ATTACHMENT0); + println!("-> gl::CheckFramebufferStatus()"); let frame_buffer_status = gl::CheckFramebufferStatus(gl::FRAMEBUFFER); if frame_buffer_status != gl::FRAMEBUFFER_COMPLETE { return Err(PointCloudVisualizerError::FramebufferError); } + println!("-> gl::Viewport() {}x{}", self.width, self.height); gl::Viewport(0, 0, self.width as i32, self.height as i32); + println!("-> gl::Enable(DEPTH_TEST"); gl::Enable(gl::DEPTH_TEST); + println!("-> gl::ClearColor()"); gl::ClearColor(0.0, 0.0, 0.0, 0.0); + println!("-> gl::Clear(COLOR,DEPTH)"); gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT); } @@ -238,6 +250,7 @@ impl PointCloudVisualizer { let render_status = self.point_cloud_renderer.render(); unsafe { + println!("-> gl::BindRenderbuffer(RENDERBUFFER, 0)"); gl::BindRenderbuffer(gl::RENDERBUFFER, 0); } @@ -246,9 +259,13 @@ impl PointCloudVisualizer { } fn update_point_clouds(&mut self, capture: Capture) -> Result<()> { - let depth_image = match capture.get_depth_image() { + println!("==== Visualizer.update_point_clouds() [from kinect capture]"); + println!("colorization strategy = {:?}", self.colorization_strategy); + + let mut depth_image = match capture.get_depth_image() { Ok(img) => img, Err(e) => { + println!("<< Missing Depth Image >> "); // Capture doesn't have depth info. Drop the capture. return Err(PointCloudVisualizerError::MissingDepthImage); }, @@ -258,24 +275,27 @@ impl PointCloudVisualizer { if self.enable_color_point_cloud { if maybe_color_image.is_err() { + println!("<< Missing Color Image >> "); return Err(PointCloudVisualizerError::MissingColorImage); } if self.colorization_strategy == ColorizationStrategy::Color { - // TODO - // m_transformation.depth_image_to_color_camera(depthImage, &m_transformedDepthImage); - // depthImage = m_transformedDepthImage; - /* - try - { - m_transformation.depth_image_to_color_camera(depthImage, &m_transformedDepthImage); - depthImage = m_transformedDepthImage; - } - catch (const k4a::error &) - { - return PointCloudVisualizationResult::DepthToColorTransformationFailed; - } - */ - return Err(PointCloudVisualizerError::ColorSupportNotYetImplemented); + + /*let mut transformed_depth_image = self.transformed_depth_image.take() + .expect("Must be present");*/ + + if let Some(transformed_depth_image) = self.transformed_depth_image.as_ref() { + println!("Take transformed depth image..."); + unsafe { + // TODO: Totally missed error handling here. + k4a_sys::k4a_transformation_depth_image_to_color_camera( + self.transformation.get_handle(), + depth_image.get_handle(), + transformed_depth_image.get_handle(), + ); + + depth_image = transformed_depth_image.clone(); + } + } } }