diff --git a/Cargo.lock b/Cargo.lock index 537305f91c..5002805552 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,6 +351,14 @@ name = "downcast-rs" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "draw_state" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "either" version = "1.5.3" @@ -422,6 +430,26 @@ dependencies = [ "mint 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "gfx" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "draw_state 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gfx_core 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gfx_core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "draw_state 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "gif" version = "0.10.3" @@ -1221,6 +1249,7 @@ version = "0.0.0" dependencies = [ "enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "genmesh 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gfx 0.18.2 (registry+https://github.com/rust-lang/crates.io-index)", "glium 0.27.0 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.23.2 (registry+https://github.com/rust-lang/crates.io-index)", "k4a-sys 0.2.0", @@ -1518,6 +1547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum dispatch 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" "checksum dlib 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "77e51249a9d823a4cb79e3eca6dcd756153e8ed0157b6c04775d04bf1b13b76a" "checksum downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" +"checksum draw_state 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "33cf9537e2d06891448799b96d5a8c8083e0e90522a7fdabe6ebf4f41d79d651" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" @@ -1528,6 +1558,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum genmesh 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a5fd00927c1c06051d7da1a933ee87cb9edc799c0657be5449f10cf658f35883" +"checksum gfx 0.18.2 (registry+https://github.com/rust-lang/crates.io-index)" = "01de46f9508a5c259aef105f0bff760ceddca832ea9c87ce03d1923e22ee155b" +"checksum gfx_core 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "75fbddaef2e12b4995900539d7209d947b988a3d87ee8737484d049b526e5441" "checksum gif 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "471d90201b3b223f3451cd4ad53e34295f16a1df17b1edf3736d47761c3981af" "checksum gl_generator 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ca98bbde17256e02d17336a6bdb5a50f7d0ccacee502e191d3e3d0ec2f96f84a" "checksum gl_generator 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" diff --git a/Cargo.toml b/Cargo.toml index b8c0dea2a7..5349ed4697 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,8 @@ path = "src/main.rs" [dependencies] enum_primitive = "0.1.1" # c wrapper ergonomics genmesh = "0.6.0" # for glium example -glium="0.27.0" +glium="0.27.0" # high-level OpenGL abstraction (*not* vanilla OpenGL) +gfx = "0.18.2" # another high level OpenGL / multi-API abstraction? image = "0.23.2" k4a-sys = { path = "/home/bt/dev/3rd/k4a-sys" } # k4a-sys = "0.2.0" libc = "0.2.68" diff --git a/src/graphics_glium.rs b/src/graphics_glium.rs new file mode 100644 index 0000000000..b45ae13b03 --- /dev/null +++ b/src/graphics_glium.rs @@ -0,0 +1,269 @@ +use std::borrow::BorrowMut; +use std::ffi::{c_void, CStr, CString}; +use std::io::Cursor; +use std::os::raw::c_char; +use std::ptr; +use std::slice; +use std::sync::{Arc, PoisonError, RwLockReadGuard}; +use std::sync::Mutex; +use std::sync::RwLock; +use std::thread; +use std::time::Duration; + +use glium::{Display, glutin, Surface}; +use glium::glutin::event::{Event, StartCause}; +use glium::glutin::event_loop::{ControlFlow, EventLoop}; +use glium::Texture2d; +use glium::texture::RawImage2d; +use glium::vertex::VertexBufferAny; +use image::{DynamicImage, ImageFormat}; +use image::flat::{FlatSamples, SampleLayout}; +use image::GenericImage; +use image::ImageBuffer; +use image::ImageError; +use image::Rgb; +use image::Rgba; +use image::RgbaImage; +use image::RgbImage; +use libc::size_t; +use opencv::core; +use opencv::highgui; +use opencv::imgproc; +use opencv::prelude::*; + +use conversion::TextureData2d; +use handwritten_wrapper::*; +use handwritten_wrapper_test; +use k4a_sys_wrapper::Device; +use k4a_sys_wrapper::device_get_installed_count; +use k4a_sys_wrapper::Image; +use sensor_control::{capture_thread, grab_single_frame}; + +pub struct TextureContainer { + pub texture: Texture2d +} + +pub fn run_glium() { + let frame : Arc>> = Arc::new(Mutex::new(None)); + let frame2 = frame.clone(); + + thread::spawn(move || { + capture_thread(frame2); + }); + + let event_loop = glutin::event_loop::EventLoop::new(); + let wb = glutin::window::WindowBuilder::new(); + let cb = glutin::ContextBuilder::new(); + let display = glium::Display::new(wb, cb, &event_loop).unwrap(); + + let imageA = image::load(Cursor::new(&include_bytes!("../sneslogo.png")[..]), + image::ImageFormat::Png).unwrap().to_rgba(); + + let imageB = image::load(Cursor::new(&include_bytes!("../n64logo.png")[..]), + image::ImageFormat::Png).unwrap().to_rgba(); + + let imageA_dimensions = imageA.dimensions(); + let imageB_dimensions = imageB.dimensions(); + + let imageA = glium::texture::RawImage2d::from_raw_rgba_reversed(&imageA.into_raw(), imageA_dimensions); + let textureA = glium::texture::Texture2d::new(&display, imageA).unwrap(); + + let imageB = glium::texture::RawImage2d::from_raw_rgba_reversed(&imageB.into_raw(), imageB_dimensions); + let textureB = glium::texture::Texture2d::new(&display, imageB).unwrap(); + + /*let frame_dimensions = frame_image.dimensions(); + let frame_image = glium::texture::RawImage2d::from_raw_rgba_reversed(&frame_image.into_raw(), frame_dimensions); + let textureFrame = glium::texture::Texture2d::new(&display, frame_image).unwrap();*/ + + #[derive(Copy, Clone)] + struct Vertex { + position: [f32; 2], + tex_coords: [f32; 2], + } + + implement_vertex!(Vertex, position, tex_coords); + + let vertex1 = Vertex { position: [-0.9, -0.5], tex_coords: [0.0, 0.0] }; + let vertex2 = Vertex { position: [ 0.0, 0.7], tex_coords: [0.0, 1.0] }; + let vertex3 = Vertex { position: [ 0.5, -0.25], tex_coords: [1.0, 1.0] }; + let vertex4 = Vertex { position: [ 0.75, -0.50], tex_coords: [1.0, 0.0] }; + let shape = vec![vertex1, vertex2, vertex3, vertex4]; + + let vertex_buffer = glium::VertexBuffer::new(&display, &shape).unwrap(); + let indices = glium::index::NoIndices(glium::index::PrimitiveType::TrianglesList); + + let vertex_shader_src = r#" + #version 140 + in vec2 position; + in vec2 tex_coords; + out vec2 v_tex_coords; + uniform mat4 matrix; + void main() { + v_tex_coords = tex_coords; + gl_Position = matrix * vec4(position, 0.0, 1.0); + } + "#; + + let fragment_shader_src = r#" + #version 140 + in vec2 v_tex_coords; + out vec4 color; + uniform sampler2D tex; + void main() { + color = texture(tex, v_tex_coords); + } + "#; + + let program = glium::Program::from_source(&display, vertex_shader_src, fragment_shader_src, None).unwrap(); + + let mut t = -0.5; + let mut switchTexture = false; + let mut useTexture = 0; // 0 for 'A', 1 for 'B' + + let mut texture_data_2d : Option = None; + let mut texture_container : Option = None; + + event_loop.run(move |event, _, control_flow| { + let next_frame_time = std::time::Instant::now() + + std::time::Duration::from_nanos(16_666_667); + *control_flow = glutin::event_loop::ControlFlow::WaitUntil(next_frame_time); + + match event { + glutin::event::Event::WindowEvent { event, .. } => match event { + glutin::event::WindowEvent::CloseRequested => { + *control_flow = glutin::event_loop::ControlFlow::Exit; + return; + }, + _ => return, + }, + glutin::event::Event::NewEvents(cause) => match cause { + glutin::event::StartCause::ResumeTimeReached { .. } => (), + glutin::event::StartCause::Init => (), + _ => return, + }, + _ => return, + } + + // we update `t` + t += 0.02; + if t > 0.5 { + t = -0.5; + switchTexture = true; + } + + if switchTexture { + useTexture = match useTexture { + 0 => 1, + _ => 0, + }; + switchTexture = false; + } + + let mut target = display.draw(); + target.clear_color(0.0, 0.0, 1.0, 1.0); + + match frame.lock() { + Ok(mut lock) => { + let tex_data_2d = lock.take(); + + match tex_data_2d { + None => {}, + Some(tex_data) => { + let texture = glium::texture::Texture2d::new(&display, tex_data.raw_image).unwrap(); + + texture_container = Some(TextureContainer{ + texture, + }) + }, + } + }, + Err(_) => {}, + } + + let texture_to_use: &Texture2d = texture_container + .as_ref() + .map(|c| &c.texture) + .unwrap_or(&textureB); + + let uniforms = uniform! { + matrix: [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [ t , 0.0, 0.0, 1.0f32], + ], + tex: texture_to_use, + }; + + target.draw(&vertex_buffer, &indices, &program, &uniforms, + &Default::default()).unwrap(); + target.finish().unwrap(); + }); +} + +pub fn run() { + let installed_devices = device_get_installed_count(); + println!("Installed devices: {}", installed_devices); + + let window_name = "output"; + highgui::named_window(window_name, 1).unwrap(); + + { + let device = Device::open(0).unwrap(); + println!("Device: {:?}", device); + let serial_number = device.get_serial_number().unwrap(); + println!("Device: {:?}", serial_number); + + println!("Starting cameras..."); + device.start_cameras().unwrap(); + + + loop { + let capture = device.get_capture(1000).ok().unwrap(); + let image = capture.get_depth_image().ok().unwrap(); + + let opencv_image = depth_to_opencv(&image).ok().unwrap(); + + highgui::imshow(window_name, &opencv_image).unwrap(); + if highgui::wait_key(10).unwrap() > 0 { + break; + } + } + + device.stop_cameras(); + } + + handwritten_wrapper_test::test_integration(); +} + +/// Copied from k4a-sys +pub fn color_to_opencv(mut image: Image) -> opencv::Result { + let with_alpha = unsafe { + let stride = image.get_stride_bytes(); + Mat::new_rows_cols_with_data( + image.get_height_pixels() as i32, + image.get_width_pixels() as i32, + core::CV_8UC4, + &mut *(image.get_buffer() as *mut c_void), + stride, + )? + }; + let mut no_alpha = Mat::default()?; + imgproc::cvt_color(&with_alpha, &mut no_alpha, imgproc::COLOR_BGRA2BGR, 0)?; + return Ok(no_alpha); +} + +/// Copied from k4a-sys +pub fn depth_to_opencv(image: &Image) -> opencv::Result { + unsafe { + let stride = image.get_stride_bytes(); + let mat = Mat::new_rows_cols_with_data( + image.get_height_pixels() as i32, + image.get_width_pixels() as i32, + core::CV_16U, + &mut *(image.get_buffer() as *mut c_void), + stride, + ); + mat + } +} diff --git a/src/main.rs b/src/main.rs index e413706676..9ce990adf6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,15 +16,16 @@ use std::os::raw::c_char; use std::ptr; use std::slice; use std::sync::{Arc, PoisonError, RwLockReadGuard}; +use std::sync::Mutex; use std::sync::RwLock; use std::thread; use std::time::Duration; -use std::sync::Mutex; use glium::{Display, glutin, Surface}; use glium::glutin::event::{Event, StartCause}; use glium::glutin::event_loop::{ControlFlow, EventLoop}; use glium::Texture2d; +use glium::texture::RawImage2d; use glium::vertex::VertexBufferAny; use image::{DynamicImage, ImageFormat}; use image::flat::{FlatSamples, SampleLayout}; @@ -41,245 +42,21 @@ use opencv::highgui; use opencv::imgproc; use opencv::prelude::*; +use conversion::TextureData2d; +use graphics_glium::run_glium; use handwritten_wrapper::*; use k4a_sys_wrapper::Device; use k4a_sys_wrapper::device_get_installed_count; use k4a_sys_wrapper::Image; use sensor_control::{capture_thread, grab_single_frame}; -use glium::texture::RawImage2d; -use conversion::TextureData2d; pub mod conversion; +pub mod graphics_glium; pub mod handwritten_wrapper; pub mod handwritten_wrapper_test; pub mod k4a_sys_wrapper; pub mod sensor_control; -pub struct TextureContainer { - pub texture: Texture2d -} - pub fn main() { - let frame : Arc>> = Arc::new(Mutex::new(None)); - let frame2 = frame.clone(); - - thread::spawn(move || { - capture_thread(frame2); - }); - - let event_loop = glutin::event_loop::EventLoop::new(); - let wb = glutin::window::WindowBuilder::new(); - let cb = glutin::ContextBuilder::new(); - let display = glium::Display::new(wb, cb, &event_loop).unwrap(); - - let imageA = image::load(Cursor::new(&include_bytes!("../sneslogo.png")[..]), - image::ImageFormat::Png).unwrap().to_rgba(); - - let imageB = image::load(Cursor::new(&include_bytes!("../n64logo.png")[..]), - image::ImageFormat::Png).unwrap().to_rgba(); - - let imageA_dimensions = imageA.dimensions(); - let imageB_dimensions = imageB.dimensions(); - - let imageA = glium::texture::RawImage2d::from_raw_rgba_reversed(&imageA.into_raw(), imageA_dimensions); - let textureA = glium::texture::Texture2d::new(&display, imageA).unwrap(); - - let imageB = glium::texture::RawImage2d::from_raw_rgba_reversed(&imageB.into_raw(), imageB_dimensions); - let textureB = glium::texture::Texture2d::new(&display, imageB).unwrap(); - - /*let frame_dimensions = frame_image.dimensions(); - let frame_image = glium::texture::RawImage2d::from_raw_rgba_reversed(&frame_image.into_raw(), frame_dimensions); - let textureFrame = glium::texture::Texture2d::new(&display, frame_image).unwrap();*/ - - #[derive(Copy, Clone)] - struct Vertex { - position: [f32; 2], - tex_coords: [f32; 2], - } - - implement_vertex!(Vertex, position, tex_coords); - - let vertex1 = Vertex { position: [-0.9, -0.5], tex_coords: [0.0, 0.0] }; - let vertex2 = Vertex { position: [ 0.0, 0.7], tex_coords: [0.0, 1.0] }; - let vertex3 = Vertex { position: [ 0.5, -0.25], tex_coords: [1.0, 1.0] }; - let vertex4 = Vertex { position: [ 0.75, -0.50], tex_coords: [1.0, 0.0] }; - let shape = vec![vertex1, vertex2, vertex3, vertex4]; - - let vertex_buffer = glium::VertexBuffer::new(&display, &shape).unwrap(); - let indices = glium::index::NoIndices(glium::index::PrimitiveType::TrianglesList); - - let vertex_shader_src = r#" - #version 140 - in vec2 position; - in vec2 tex_coords; - out vec2 v_tex_coords; - uniform mat4 matrix; - void main() { - v_tex_coords = tex_coords; - gl_Position = matrix * vec4(position, 0.0, 1.0); - } - "#; - - let fragment_shader_src = r#" - #version 140 - in vec2 v_tex_coords; - out vec4 color; - uniform sampler2D tex; - void main() { - color = texture(tex, v_tex_coords); - } - "#; - - let program = glium::Program::from_source(&display, vertex_shader_src, fragment_shader_src, None).unwrap(); - - let mut t = -0.5; - let mut switchTexture = false; - let mut useTexture = 0; // 0 for 'A', 1 for 'B' - - let mut texture_data_2d : Option = None; - let mut texture_container : Option = None; - - event_loop.run(move |event, _, control_flow| { - let next_frame_time = std::time::Instant::now() + - std::time::Duration::from_nanos(16_666_667); - *control_flow = glutin::event_loop::ControlFlow::WaitUntil(next_frame_time); - - match event { - glutin::event::Event::WindowEvent { event, .. } => match event { - glutin::event::WindowEvent::CloseRequested => { - *control_flow = glutin::event_loop::ControlFlow::Exit; - return; - }, - _ => return, - }, - glutin::event::Event::NewEvents(cause) => match cause { - glutin::event::StartCause::ResumeTimeReached { .. } => (), - glutin::event::StartCause::Init => (), - _ => return, - }, - _ => return, - } - - // we update `t` - t += 0.02; - if t > 0.5 { - t = -0.5; - switchTexture = true; - } - - if switchTexture { - useTexture = match useTexture { - 0 => 1, - _ => 0, - }; - switchTexture = false; - } - - let mut target = display.draw(); - target.clear_color(0.0, 0.0, 1.0, 1.0); - - match frame.lock() { - Ok(mut lock) => { - let tex_data_2d = lock.take(); - - match tex_data_2d { - None => {}, - Some(tex_data) => { - let texture = glium::texture::Texture2d::new(&display, tex_data.raw_image).unwrap(); - - texture_container = Some(TextureContainer{ - texture, - }) - }, - } - }, - Err(_) => {}, - } - - let texture_to_use: &Texture2d = texture_container - .as_ref() - .map(|c| &c.texture) - .unwrap_or(&textureB); - - let uniforms = uniform! { - matrix: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - [ t , 0.0, 0.0, 1.0f32], - ], - tex: texture_to_use, - }; - - target.draw(&vertex_buffer, &indices, &program, &uniforms, - &Default::default()).unwrap(); - target.finish().unwrap(); - }); -} - -pub fn run() { - let installed_devices = device_get_installed_count(); - println!("Installed devices: {}", installed_devices); - - let window_name = "output"; - highgui::named_window(window_name, 1).unwrap(); - - { - let device = Device::open(0).unwrap(); - println!("Device: {:?}", device); - let serial_number = device.get_serial_number().unwrap(); - println!("Device: {:?}", serial_number); - - println!("Starting cameras..."); - device.start_cameras().unwrap(); - - - loop { - let capture = device.get_capture(1000).ok().unwrap(); - let image = capture.get_depth_image().ok().unwrap(); - - let opencv_image = depth_to_opencv(&image).ok().unwrap(); - - highgui::imshow(window_name, &opencv_image).unwrap(); - if highgui::wait_key(10).unwrap() > 0 { - break; - } - } - - device.stop_cameras(); - } - - handwritten_wrapper_test::test_integration(); -} - -/// Copied from k4a-sys -pub fn color_to_opencv(mut image: Image) -> opencv::Result { - let with_alpha = unsafe { - let stride = image.get_stride_bytes(); - Mat::new_rows_cols_with_data( - image.get_height_pixels() as i32, - image.get_width_pixels() as i32, - core::CV_8UC4, - &mut *(image.get_buffer() as *mut c_void), - stride, - )? - }; - let mut no_alpha = Mat::default()?; - imgproc::cvt_color(&with_alpha, &mut no_alpha, imgproc::COLOR_BGRA2BGR, 0)?; - return Ok(no_alpha); -} - -/// Copied from k4a-sys -pub fn depth_to_opencv(image: &Image) -> opencv::Result { - unsafe { - let stride = image.get_stride_bytes(); - let mat = Mat::new_rows_cols_with_data( - image.get_height_pixels() as i32, - image.get_width_pixels() as i32, - core::CV_16U, - &mut *(image.get_buffer() as *mut c_void), - stride, - ); - mat - } + run_glium(); }