From 78da7175951a112288f69ea6e43152ddade77636 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 18 Apr 2020 05:39:49 -0400 Subject: [PATCH] visualizer, ctor, update_texture, wip/untested --- src/graphics_gl.rs | 6 +- src/k4a_sys_wrapper.rs | 41 ++++ src/main.rs | 2 +- src/opengl_wrapper.rs | 102 +++++++++ .../compile_shader.rs | 0 src/{shaders => point_cloud}/mod.rs | 1 + .../point_cloud_compute_shader.rs | 4 +- .../point_cloud_renderer_shader.rs | 6 +- src/point_cloud/point_cloud_visualiser.rs | 212 ++++++++++++++++++ 9 files changed, 365 insertions(+), 9 deletions(-) rename src/{shaders => point_cloud}/compile_shader.rs (100%) rename src/{shaders => point_cloud}/mod.rs (75%) rename src/{shaders => point_cloud}/point_cloud_compute_shader.rs (99%) rename src/{shaders => point_cloud}/point_cloud_renderer_shader.rs (98%) create mode 100644 src/point_cloud/point_cloud_visualiser.rs diff --git a/src/graphics_gl.rs b/src/graphics_gl.rs index 2e26a4d0ba..89787dde69 100644 --- a/src/graphics_gl.rs +++ b/src/graphics_gl.rs @@ -20,9 +20,9 @@ use std::ptr::null; use glium::framebuffer::ColorAttachment::Texture; use std::sync::Arc; use sensor_control::CaptureProvider; -use shaders; -use shaders::point_cloud_compute_shader::PointCloudComputeShader; -use shaders::point_cloud_renderer_shader::PointCloudRendererShader; +use point_cloud; +use point_cloud::point_cloud_compute_shader::PointCloudComputeShader; +use point_cloud::point_cloud_renderer_shader::PointCloudRendererShader; //use shader::Shader; diff --git a/src/k4a_sys_wrapper.rs b/src/k4a_sys_wrapper.rs index 84fa1d1755..25798ec633 100644 --- a/src/k4a_sys_wrapper.rs +++ b/src/k4a_sys_wrapper.rs @@ -377,3 +377,44 @@ impl DeviceConfiguration { }) } } + +#[derive(Clone,Debug)] +pub struct Resolution { + pub width: i32, + pub height: i32, +} + +#[derive(Clone,Debug)] +pub struct Transformation { + transformation: k4a_sys::k4a_transformation_t, + pub color_resolution: Resolution, + pub depth_resolution: Resolution, +} + +impl Transformation { + /// Creates a transformation associated with a calibration + pub fn from_calibration(calibration: &k4a_sys::k4a_calibration_t) -> Self { + let transformation = unsafe { + k4a_sys::k4a_transformation_create(calibration) + }; + Self { + transformation, + color_resolution: Resolution { + width: calibration.color_camera_calibration.resolution_width, + height: calibration.color_camera_calibration.resolution_height, + }, + depth_resolution: Resolution { + width: calibration.depth_camera_calibration.resolution_width, + height: calibration.depth_camera_calibration.resolution_height, + }, + } + } +} + +impl Drop for Transformation { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_transformation_destroy(self.transformation); + } + } +} diff --git a/src/main.rs b/src/main.rs index 93a7995549..9c3a55b981 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,7 +59,7 @@ pub mod old_graphics_libraries; pub mod k4a_sys_wrapper; pub mod old_k4a_wrapper; pub mod sensor_control; -pub mod shaders; +pub mod point_cloud; pub fn main() { let capture_provider = Arc::new(CaptureProvider::new()); diff --git a/src/opengl_wrapper.rs b/src/opengl_wrapper.rs index 3223f09b17..cf26fe9291 100644 --- a/src/opengl_wrapper.rs +++ b/src/opengl_wrapper.rs @@ -57,6 +57,108 @@ impl Buffer { } } +/// A slightly more convenient OpenGL Framebuffer type +pub struct Framebuffer { + id: GLuint, +} + +impl Framebuffer { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + pub fn gen(&mut self) { + unsafe { + gl::GenFramebuffers(1, &mut self.id); + } + } + + pub fn delete(&mut self) { + unsafe { + gl::DeleteFramebuffers(1, &mut self.id); + } + } +} + +/// A slightly more convenient OpenGL Renderbuffer type +pub struct Renderbuffer { + id: GLuint, +} + +impl Renderbuffer { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + fn gen(&mut self) { + unsafe { + gl::GenRenderbuffers(1, &mut self.id); + } + } + + fn delete(&mut self) { + unsafe { + gl::DeleteRenderbuffers(1, &mut self.id); + } + } +} + /// A slightly more convenient OpenGL Texture type pub struct Texture { id: GLuint, diff --git a/src/shaders/compile_shader.rs b/src/point_cloud/compile_shader.rs similarity index 100% rename from src/shaders/compile_shader.rs rename to src/point_cloud/compile_shader.rs diff --git a/src/shaders/mod.rs b/src/point_cloud/mod.rs similarity index 75% rename from src/shaders/mod.rs rename to src/point_cloud/mod.rs index 5a09c965d8..e0b1910ad2 100644 --- a/src/shaders/mod.rs +++ b/src/point_cloud/mod.rs @@ -1,3 +1,4 @@ pub mod point_cloud_compute_shader; pub mod point_cloud_renderer_shader; +pub mod point_cloud_visualiser; pub mod compile_shader; diff --git a/src/shaders/point_cloud_compute_shader.rs b/src/point_cloud/point_cloud_compute_shader.rs similarity index 99% rename from src/shaders/point_cloud_compute_shader.rs rename to src/point_cloud/point_cloud_compute_shader.rs index 891f84d5fe..6a5e67e539 100644 --- a/src/shaders/point_cloud_compute_shader.rs +++ b/src/point_cloud/point_cloud_compute_shader.rs @@ -1,4 +1,4 @@ -//! This is a port of Microsoft's libk4a `tools/k4aviewer/graphics/shaders/gpudepthtopointcloudconverter.h`. +//! This is a port of Microsoft's libk4a `gpudepthtopointcloudconverter.h`. //! This code turns depth images into point clouds. use std::ffi::CString; @@ -18,7 +18,7 @@ use k4a_sys_wrapper::Image; use k4a_sys_wrapper::ImageFormat; use opengl_wrapper::Buffer; use opengl_wrapper::Texture; -use shaders::compile_shader::compile_shader; +use point_cloud::compile_shader::compile_shader; pub type Result = std::result::Result; diff --git a/src/shaders/point_cloud_renderer_shader.rs b/src/point_cloud/point_cloud_renderer_shader.rs similarity index 98% rename from src/shaders/point_cloud_renderer_shader.rs rename to src/point_cloud/point_cloud_renderer_shader.rs index 2c9e296ed9..72a47125f4 100644 --- a/src/shaders/point_cloud_renderer_shader.rs +++ b/src/point_cloud/point_cloud_renderer_shader.rs @@ -1,4 +1,4 @@ -//! This is a port of Microsoft's libk4a `tools/k4aviewer/window/point_cloud/4_k4apointcloudrenderer.cpp`. +//! This is a port of Microsoft's libk4a `k4apointcloudrenderer.cpp`. //! This provides the visual output. use std::ffi::CString; @@ -16,8 +16,8 @@ use libc; use opengl_wrapper::{Buffer, VertexArray, gl_get_error, OpenGlError}; use opengl_wrapper::Texture; -use shaders::compile_shader::compile_shader; -use shaders::point_cloud_compute_shader::POINT_CLOUD_TEXTURE_FORMAT; +use point_cloud::compile_shader::compile_shader; +use point_cloud::point_cloud_compute_shader::POINT_CLOUD_TEXTURE_FORMAT; use graphics_gl::{get_stride, get_pointer_offset}; pub type Result = std::result::Result; diff --git a/src/point_cloud/point_cloud_visualiser.rs b/src/point_cloud/point_cloud_visualiser.rs new file mode 100644 index 0000000000..3576a93846 --- /dev/null +++ b/src/point_cloud/point_cloud_visualiser.rs @@ -0,0 +1,212 @@ +//! Based off Microsoft's MIT-licensed k4a library code and examples, specifically +//! `k4apointcloudvisualizer.h` + +use point_cloud::point_cloud_renderer_shader::{PointCloudRendererShader, PointCloudRendererError}; +use k4a_sys_wrapper::{Capture, Transformation}; +use k4a_sys_wrapper::Image; +use opengl_wrapper::{Texture, Renderbuffer, Framebuffer}; +use point_cloud::point_cloud_compute_shader::PointCloudComputeShader; +use std::fmt::{Error, Formatter}; +use opengl_wrapper::OpenGlError; + +pub type Result = std::result::Result; + + +#[derive(Clone, Debug)] +pub enum PointCloudVisualizerError { + OpenGlError(OpenGlError), + PointCloudRendererError(PointCloudRendererError), + FramebufferError, + UnknownError, +} + +impl std::fmt::Display for PointCloudVisualizerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let description = match self { + PointCloudVisualizerError::OpenGlError(inner) => { + format!("Visualizer OpenGL error: {}", inner) + }, + PointCloudVisualizerError::PointCloudRendererError(inner) => { + format!("Visualizer PointCloudRenderer error: {}", inner) + }, + PointCloudVisualizerError::FramebufferError => "Visualizer Framebuffer Error".into(), + PointCloudVisualizerError::UnknownError => "Visualizer Unknown Error".into(), + }; + + write!(f, "{}", description) + } +} + +impl std::error::Error for PointCloudVisualizerError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + +pub struct PointCloudVisualizer { + // TODO: std::pair m_expectedValueRange; + // TODO: ImageDimensions m_dimensions; + // TODO: ViewControl m_viewControl; + // TODO: ColorizationStrategy m_colorizationStrategy; + // TODO: linmath::mat4x4 m_projection{}; + // TODO: linmath::mat4x4 m_view{}; + + width: u16, + height: u16, + + enable_color_point_cloud: bool, + + point_cloud_renderer: PointCloudRendererShader, + point_cloud_converter: PointCloudComputeShader, + + calibration_data: k4a_sys::k4a_calibration_t, + transformation: Transformation, // TODO: WAT k4a_sys::k4a_transformation_t + + frame_buffer: Framebuffer, + depth_buffer: Renderbuffer, + + last_capture: Option, + + /// Buffer that holds the depth image transformed to the color coordinate space. + /// Used in color mode only. + transformed_depth_image: Option, + + /// In color mode, this is just a shallow copy of the latest color image. + /// In depth mode, this is a buffer that holds the colorization of the depth image. + point_cloud_colorization: Option, + + /// Holds the XYZ point cloud as a texture. + /// Format is XYZA, where A (the alpha channel) is unused. + xyz_texture: Texture, + + color_xy_table: Image, + depth_xy_table: Image, +} + +impl PointCloudVisualizer { + pub fn new(enable_color_point_cloud: bool, + calibration_data: k4a_sys::k4a_calibration_t) -> Self + { + // TODO: + let width = 800; + let height= 800; + + let depth_buffer = Renderbuffer::new_initialized(); + + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, depth_buffer.id()); + gl::RenderbufferStorage(gl::RENDERBUFFER, gl::DEPTH_COMPONENT, width, height); + } + + /*typedef enum + { + K4A_CALIBRATION_TYPE_UNKNOWN = -1, /**< Calibration type is unknown */ + K4A_CALIBRATION_TYPE_DEPTH, /**< Depth sensor */ + K4A_CALIBRATION_TYPE_COLOR, /**< Color sensor */ + K4A_CALIBRATION_TYPE_GYRO, /**< Gyroscope sensor */ + K4A_CALIBRATION_TYPE_ACCEL, /**< Accelerometer sensor */ + K4A_CALIBRATION_TYPE_NUM, /**< Number of types excluding unknown type*/ + } k4a_calibration_type_t;*/ + + // TODO: generate color xytable only on `enable_color_point_cloud`. + let color_xy_table = PointCloudComputeShader::generate_xy_table( + calibration_data.clone(), + 1, //k4a_sys::K4A_CALIBRATION_TYPE_COLOR, + ).unwrap(); + + let depth_xy_table = PointCloudComputeShader::generate_xy_table( + calibration_data.clone(), + 0, // k4a_sys::K4A_CALIBRATION_TYPE_DEPTH, + ).unwrap(); + + // TODO: C++ does further construction within the CTOR by calling: + // SetColorizationStrategy(m_colorizationStrategy); + + Self { + width: width as u16, + height: height as u16, + enable_color_point_cloud, + point_cloud_renderer: PointCloudRendererShader::new(), + point_cloud_converter: PointCloudComputeShader::new(), + frame_buffer: Framebuffer::new_initialized(), + depth_buffer, + color_xy_table, + depth_xy_table, + calibration_data: calibration_data.clone(), + transformation: Transformation::from_calibration(&calibration_data), + last_capture: None, + transformed_depth_image: None, + point_cloud_colorization: None, + xyz_texture: Texture::new(), + } + } + + pub fn set_point_size(&mut self, point_size: u8) { + self.point_cloud_renderer.set_point_size(point_size); + } + + // TODO + pub fn set_colorization_strategy(&mut self) { + unimplemented!(); + } + + pub fn update_texture(&mut self) -> Result<()> { + + /* TODO // Update the point cloud renderer with the latest point data + // + PointCloudVisualizationResult result = UpdatePointClouds(capture); + if (result != PointCloudVisualizationResult::Success) + { + return result; + }*/ + + + // Set up rendering to a texture + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, self.depth_buffer.id()); + gl::BindFramebuffer(gl::FRAMEBUFFER, self.frame_buffer.id()); + } + + // TODO + // CleanupGuard frameBufferBindingGuard([]() { glBindFramebuffer(GL_FRAMEBUFFER, 0); }); + + unsafe { + gl::FramebufferRenderbuffer(gl::FRAMEBUFFER, gl::DEPTH_ATTACHMENT, gl::RENDERBUFFER, self.depth_buffer.id()); + // TODO: + // glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, static_cast(**texture), 0); + // gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture_buffer, 0); + + gl::DrawBuffers(1, &gl::COLOR_ATTACHMENT0); + + let frame_buffer_status = gl::CheckFramebufferStatus(gl::FRAMEBUFFER); + + if frame_buffer_status != gl::FRAMEBUFFER_COMPLETE { + return Err(PointCloudVisualizerError::FramebufferError); + } + + gl::Viewport(0, 0, self.width as i32, self.height as i32); + + gl::Enable(gl::DEPTH_TEST); + gl::ClearColor(0.0, 0.0, 0.0, 0.0); + gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT); + } + + // TODO: View matrix maths + self.point_cloud_renderer.update_view_projection(); + + let render_status = self.point_cloud_renderer.render(); + + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, 0); + } + + render_status + .map_err(|err| PointCloudVisualizerError::PointCloudRendererError(err)) + } + + fn update_point_clouds(&mut self) -> Result<()> { + + unimplemented!(); + } +}