diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a6eaf47f5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +.idea/ +target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..168fbe12c --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2420 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "andrew" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7f09f89872c2b6b29e319377b1fbe91c6f5947df19a25596e121cf19a7b35e" +dependencies = [ + "bitflags", + "line_drawing", + "rusttype 0.7.9", + "walkdir", + "xdg", + "xml-rs 0.8.3", +] + +[[package]] +name = "android_glue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000444226fcff248f2bc4c7625be32c63caccfecc2723a2b9f78a7487a49c407" + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arcball" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e3433ec37e64e887a11f4f1a98599780ad766070f33c68bda46f8196359aae" +dependencies = [ + "cgmath", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide 0.4.3", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "bindgen" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c85344eb535a31b62f0af37be84441ba9e7f0f4111eb0530f43d15e513fe57" +dependencies = [ + "bitflags", + "cexpr", + "cfg-if 0.1.10", + "clang-sys", + "clap 2.33.3", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2 1.0.24", + "quote 1.0.8", + "regex", + "rustc-hash", + "shlex", + "which", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + +[[package]] +name = "bytemuck" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a4bad0c5981acc24bc09e532f35160f952e35422603f0563cd7a73c2c2e65a0" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "calloop" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa2097be53a00de9e8fc349fea6d76221f398f5c4fa550d420669906962d160" +dependencies = [ + "mio", + "mio-extras", + "nix", +] + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" + +[[package]] +name = "cexpr" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + +[[package]] +name = "cgmath" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283944cdecc44bf0b8dd010ec9af888d3b4f142844fdbe026c20ef68148d6fe7" +dependencies = [ + "approx", + "num-traits", + "rand 0.6.5", +] + +[[package]] +name = "clang-sys" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +dependencies = [ + "glob", + "libc", + "libloading 0.5.2", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim 0.10.0", + "termcolor", + "textwrap 0.12.1", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2 1.0.24", + "quote 1.0.8", + "syn", +] + +[[package]] +name = "clipboard" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a904646c0340239dcf7c51677b33928bf24fdf424b79a57909c0109075b2e7" +dependencies = [ + "clipboard-win", + "objc", + "objc-foundation", + "objc_id", + "x11-clipboard", +] + +[[package]] +name = "clipboard-win" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a093d6fed558e5fe24c3dfc85a68bb68f1c824f440d3ba5aca189e2998786b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "cloudcam" +version = "0.0.0" +dependencies = [ + "anyhow", + "arcball", + "byteorder", + "bytes", + "cgmath", + "clap 3.0.0-beta.2", + "clipboard", + "env_logger", + "gl", + "glfw", + "glfw-sys", + "glium", + "gltf", + "image", + "imgui", + "imgui-glium-renderer", + "imgui-opengl-renderer", + "imgui-sdl2", + "imgui-winit-support", + "k4a-sys-temp", + "kinect", + "libc", + "log", + "nalgebra", + "nalgebra-glm", + "rand 0.7.3", + "sdl2", + "tobj", + "winit", + "zmq", +] + +[[package]] +name = "cmake" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855" +dependencies = [ + "cc", +] + +[[package]] +name = "cocoa" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29f7768b2d1be17b96158e3285951d366b40211320fb30826a76cb7a0da6400" +dependencies = [ + "bitflags", + "block", + "core-foundation 0.6.4", + "core-graphics 0.17.3", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "const_fn" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" + +[[package]] +name = "core-foundation" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" +dependencies = [ + "core-foundation-sys 0.6.2", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "core-graphics" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56790968ab1c8a1202a102e6de05fc6e1ec87da99e4e93e9a7d13efbfc1e95a9" +dependencies = [ + "bitflags", + "core-foundation 0.6.4", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +dependencies = [ + "bitflags", + "core-foundation 0.7.0", + "foreign-types", + "libc", +] + +[[package]] +name = "core-video-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "core-graphics 0.19.2", + "libc", + "objc", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset 0.6.1", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dlib" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f15d1e3268f140f68d390637d5e76d849782d971ae7063e0da69fe9709a76" +dependencies = [ + "libloading 0.6.7", +] + +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "error-chain" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "generic-array" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed1e761351b56f54eb9dcd0cfaca9fd0daecf93918e1cfc01c8a3d26ee7adcd" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gif" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02efba560f227847cb41463a7395c514d127d4f74fff12ef0137fff1b84b96c4" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "gl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94edab108827d67608095e269cf862e60d920f144a5026d3dbcfd8b877fb404" +dependencies = [ + "gl_generator 0.14.0", +] + +[[package]] +name = "gl_generator" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a795170cbd85b5a7baa58d6d7525cae6a03e486859860c220f7ebbbdd379d0a" +dependencies = [ + "khronos_api 2.2.0", + "log", + "xml-rs 0.7.0", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api 3.1.0", + "log", + "xml-rs 0.8.3", +] + +[[package]] +name = "glfw" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ed2a51f1e1c5722b7a3724fc146cbfe27297584a4fbb2797ce487f62bb0320" +dependencies = [ + "bitflags", + "glfw-sys", + "libc", + "log", + "objc", + "raw-window-handle", + "semver", + "winapi 0.3.9", +] + +[[package]] +name = "glfw-sys" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b567b13b593ec58ea912b2658ee6230ffe20a069fa5b771800acc69bb3a157e" +dependencies = [ + "cmake", +] + +[[package]] +name = "glium" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a46e88b2c64cfa63d54ed54dc2a0d25136470eb1721e10be49d857ab583f8663" +dependencies = [ + "backtrace", + "fnv", + "gl_generator 0.14.0", + "glutin", + "lazy_static", + "memoffset 0.5.6", + "smallvec", + "takeable-option", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "gltf" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6fb0d1d772daf10ea74528c3aeb12215f6d5b820adf2ecfc93a6578d6779c3c" +dependencies = [ + "base64", + "byteorder", + "gltf-json", + "image", + "lazy_static", +] + +[[package]] +name = "gltf-derive" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6636de7bf52227363554f1ca2d9cd180fc666129ddd0933097e1f227dfa7293" +dependencies = [ + "inflections", + "proc-macro2 1.0.24", + "quote 1.0.8", + "syn", +] + +[[package]] +name = "gltf-json" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3fc3deb81e6fa04bf808f6be7c3983229552a95b77f687ad96af00f6d3e7d6c" +dependencies = [ + "gltf-derive", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "glutin" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf22d4e90c55d9be9f2ad52410e7a2c0d7e9c99d93a13df73a672e7ef4e8c7f7" +dependencies = [ + "android_glue", + "cgl", + "cocoa", + "core-foundation 0.6.4", + "core-graphics 0.17.3", + "glutin_egl_sys", + "glutin_emscripten_sys", + "glutin_gles2_sys", + "glutin_glx_sys", + "glutin_wgl_sys", + "lazy_static", + "libloading 0.5.2", + "log", + "objc", + "osmesa-sys", + "parking_lot", + "wayland-client", + "winapi 0.3.9", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2abb6aa55523480c4adc5a56bbaa249992e2dddb2fc63dc96e04a3355364c211" +dependencies = [ + "gl_generator 0.14.0", + "winapi 0.3.9", +] + +[[package]] +name = "glutin_emscripten_sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80de4146df76e8a6c32b03007bc764ff3249dcaeb4f675d68a06caf1bac363f1" + +[[package]] +name = "glutin_gles2_sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094e708b730a7c8a1954f4f8a31880af00eb8a1c5b5bf85d28a0a3c6d69103" +dependencies = [ + "gl_generator 0.14.0", + "objc", +] + +[[package]] +name = "glutin_glx_sys" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e393c8fc02b807459410429150e9c4faffdb312d59b8c038566173c81991351" +dependencies = [ + "gl_generator 0.14.0", + "x11-dl", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da5951a1569dbab865c6f2a863efafff193a93caf05538d193e9e3816d21696" +dependencies = [ + "gl_generator 0.14.0", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "image" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational 0.3.2", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + +[[package]] +name = "imgui" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2d28cee2982d4dab4dd954c23be339ab1f98be4ff5a13ecd8fbfb47cdfa23a" +dependencies = [ + "bitflags", + "glium", + "imgui-sys", + "lazy_static", + "parking_lot", +] + +[[package]] +name = "imgui-glium-renderer" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50773138db17dbedfa04feb3f49a2b220365938746882d349e06cb0a4c187e9" +dependencies = [ + "glium", + "imgui", +] + +[[package]] +name = "imgui-opengl-renderer" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0567f7e5fb8cbf74a8469040ae3cebed0d75823d6d991803dbdd81661a5d99" +dependencies = [ + "gl_generator 0.9.0", + "imgui", +] + +[[package]] +name = "imgui-sdl2" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e021b0ecd228c598f6ea39ba1aca7550e02cfa035c7e849ff393d082227bf70" +dependencies = [ + "imgui", + "sdl2", +] + +[[package]] +name = "imgui-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a8693e0f66ecd2ec10e2f0ed4dc4519071495f350e6d8ddaf3e70351229257" +dependencies = [ + "cc", +] + +[[package]] +name = "imgui-winit-support" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5685c7d17aee9555377f9c4f7b6fac96899a7015f6b41950c613a3df034aff08" +dependencies = [ + "imgui", + "winit", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "jpeg-decoder" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" +dependencies = [ + "byteorder", + "rayon", +] + +[[package]] +name = "k4a-sys-temp" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eec91eee878aa31ccb81b902102102f29d5b649640a57d4ebb7ce4ce7e891ff" +dependencies = [ + "bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "khronos_api" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037ab472c33f67b5fbd3e9163a2645319e5356fcd355efa6d4eb7fff4bbcb554" + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "kinect" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f185ce99cfd26fb71672096f68709eabc8a20d20f42fece762fe7dd104b4fc40" +dependencies = [ + "k4a-sys-temp", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" + +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +dependencies = [ + "cc", + "winapi 0.3.9", +] + +[[package]] +name = "libloading" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" +dependencies = [ + "cfg-if 1.0.0", + "winapi 0.3.9", +] + +[[package]] +name = "libm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" + +[[package]] +name = "line_drawing" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc7ad3d82c845bdb5dde34ffdcc7a5fb4d2996e1e1ee0f19c33bc80e15196b9" +dependencies = [ + "num-traits", +] + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + +[[package]] +name = "matrixmultiply" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916806ba0031cd542105d916a97c8572e1fa6dd79c9c51e7eb43a09ec2dd84c1" +dependencies = [ + "rawpointer", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "metadeps" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b122901b3a675fac8cecf68dcb2f0d3036193bc861d1ac0e1c337f7d5254c2" +dependencies = [ + "error-chain", + "pkg-config", + "toml", +] + +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg 1.0.1", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-extras" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" +dependencies = [ + "lazycell", + "log", + "mio", + "slab", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "nalgebra" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6147c3d50b4f3cdabfe2ecc94a0191fd3d6ad58aefd9664cf396285883486" +dependencies = [ + "approx", + "generic-array", + "matrixmultiply", + "num-complex", + "num-rational 0.2.4", + "num-traits", + "rand 0.7.3", + "rand_distr", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-glm" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0689d473f0cbf5d763219dc9faf5d3c3136d20583f533317cc9463f85400ca1" +dependencies = [ + "approx", + "nalgebra", + "num-traits", + "simba", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check 0.1.5", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" + +[[package]] +name = "osmesa-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88cfece6e95d2e717e0872a7f53a8684712ad13822a7979bc760b9c77ec0013b" +dependencies = [ + "shared_library", +] + +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.24", + "quote 1.0.8", + "syn", + "version_check 0.9.2", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.8", + "version_check 0.9.2", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid 0.2.1", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2 1.0.24", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96977acbdd3a6576fb1d27391900035bf3863d4a16422973a409b488cf29ffb2" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "raw-window-handle" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a441a7a6c80ad6473bd4b74ec1c9a4c951794285bf941c2126f607c72e48211" +dependencies = [ + "libc", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg 1.0.1", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rusttype" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310942406a39981bed7e12b09182a221a29e0990f3e7e0c971f131922ed135d5" +dependencies = [ + "rusttype 0.8.3", +] + +[[package]] +name = "rusttype" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f61411055101f7b60ecf1041d87fb74205fb20b0c7a723f07ef39174cf6b4c0" +dependencies = [ + "approx", + "ordered-float", + "stb_truetype", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sdl2" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f74124048ea86b5cd50236b2443f6f57cf4625a8e8818009b4e50dbb8729a43" +dependencies = [ + "bitflags", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e1deb61ff274d29fb985017d4611d4004b113676eaa9c06754194caf82094e" +dependencies = [ + "cfg-if 0.1.10", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bdd36f49e35b61d49efd8aa7fc068fd295961fd2286d0b2ee9a4c7a14e99cc3" + +[[package]] +name = "serde_derive" +version = "1.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552954ce79a059ddd5fd68c271592374bd15cab2274970380c000118aeffe1cd" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.8", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shared_library" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + +[[package]] +name = "simba" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb931b1367faadea6b1ab1c306a860ec17aaa5fa39f367d0c744e69d971a1fb2" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "smithay-client-toolkit" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421c8dc7acf5cb205b88160f8b4cc2c5cfabe210e43b2f80f009f4c1ef910f1d" +dependencies = [ + "andrew", + "bitflags", + "dlib", + "lazy_static", + "memmap", + "nix", + "wayland-client", + "wayland-protocols", +] + +[[package]] +name = "stb_truetype" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f77b6b07e862c66a9f3e62a07588fee67cd90a9135a2b942409f195507b4fb51" +dependencies = [ + "byteorder", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.8", + "unicode-xid 0.2.1", +] + +[[package]] +name = "takeable-option" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ae8932fcfea38b7d3883ae2ab357b0d57a02caaa18ebb4f5ece08beaec4aa0" + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread_local" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.3", + "weezl", +] + +[[package]] +name = "tobj" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b9d0bafde95a2f8f50dd3c10bc127b462efa8c3c0b60dfa276b7e032100e21d" +dependencies = [ + "log", +] + +[[package]] +name = "toml" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736b60249cb25337bc196faa43ee12c705e426f3d55c214d73a4e7be06f92cb4" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wayland-client" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1080ebe0efabcf12aef2132152f616038f2d7dcbbccf7b2d8c5270fe14bcda" +dependencies = [ + "bitflags", + "calloop", + "downcast-rs", + "libc", + "mio", + "nix", + "wayland-commons", + "wayland-scanner", + "wayland-sys", +] + +[[package]] +name = "wayland-commons" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb66b0d1a27c39bbce712b6372131c6e25149f03ffb0cd017cf8f7de8d66dbdb" +dependencies = [ + "nix", + "wayland-sys", +] + +[[package]] +name = "wayland-protocols" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc286643656742777d55dc8e70d144fa4699e426ca8e9d4ef454f4bf15ffcf9" +dependencies = [ + "bitflags", + "wayland-client", + "wayland-commons", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93b02247366f395b9258054f964fe293ddd019c3237afba9be2ccbe9e1651c3d" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "xml-rs 0.8.3", +] + +[[package]] +name = "wayland-sys" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d94e89a86e6d6d7c7c9b19ebf48a03afaac4af6bc22ae570e9a24124b75358f4" +dependencies = [ + "dlib", + "lazy_static", +] + +[[package]] +name = "weezl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" + +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a5c1a5ef76ac31cc97ad29489acdbed2178f3fc12ca00ee6cb11d60adb5a3a" +dependencies = [ + "android_glue", + "bitflags", + "cocoa", + "core-foundation 0.6.4", + "core-graphics 0.17.3", + "core-video-sys", + "dispatch", + "instant", + "lazy_static", + "libc", + "log", + "mio", + "mio-extras", + "objc", + "parking_lot", + "percent-encoding", + "raw-window-handle", + "smithay-client-toolkit", + "wayland-client", + "winapi 0.3.9", + "x11-dl", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "x11-clipboard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bd49c06c9eb5d98e6ba6536cf64ac9f7ee3a009b2f53996d405b3944f6bcea" +dependencies = [ + "xcb", +] + +[[package]] +name = "x11-dl" +version = "2.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf981e3a5b3301209754218f962052d4d9ee97e478f4d26d4a6eced34c1fef8" +dependencies = [ + "lazy_static", + "libc", + "maybe-uninit", + "pkg-config", +] + +[[package]] +name = "xcb" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e917a3f24142e9ff8be2414e36c649d47d6cc2ba81f16201cdef96e533e02de" +dependencies = [ + "libc", + "log", +] + +[[package]] +name = "xdg" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" + +[[package]] +name = "xml-rs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1cb601d29fe2c2ac60a2b2e5e293994d87a1f6fa9687a31a15270f909be9c2" +dependencies = [ + "bitflags", +] + +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "zeromq-src" +version = "0.1.10+4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9133d366817fcffe22e4356043ba187ae122ec5db63d7ce73d1e6a18efa2f1" +dependencies = [ + "cmake", +] + +[[package]] +name = "zmq" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad98a7a617d608cd9e1127147f630d24af07c7cd95ba1533246d96cbdd76c66" +dependencies = [ + "bitflags", + "libc", + "log", + "zmq-sys", +] + +[[package]] +name = "zmq-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d33a2c51dde24d5b451a2ed4b488266df221a5eaee2ee519933dc46b9a9b3648" +dependencies = [ + "libc", + "metadeps", + "zeromq-src", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..9b9debcc1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "cloudcam" +version = "0.0.0" +edition = "2018" +authors = [ "Brandon Thomas ", + "Brandon Thomas " ] + +[[bin]] +name = "cloudcam_visualizer" +path = "src/main_opencv.rs" + +[[bin]] +name = "cloudcam_points" +path = "src/main_points.rs" + +[[bin]] +name = "cloudcam_zeromq" +path = "src/main_zeromq.rs" + +[dependencies] +anyhow = "1.0.32" +arcball = "1.0.0" # camera +byteorder = "1.4.2" +bytes = "1.0.1" +cgmath = "0.17.0" # for arcball, but probably good for matrices, vecs, etc. +clap = "3.0.0-beta.1" +clipboard = "0.5" # to support imgui... +env_logger = "0.7.1" +gl = "0.14.0" # this looks like bare bones OpenGL +glfw = "0.37.0" # GL framework - windows, contexts, devices +glfw-sys = "3.3.2" # Does this have linear algebra? +glium = { version = "0.26.0", default-features = true } # high-level OpenGL abstraction (*not* vanilla OpenGL) +gltf = "0.15.2" # glTF loader (more modern OBJ) +image = "0.23.8" # all the image formats +imgui = "0.3.0" # OpenGL windowing toolkit (experimental bindings) +imgui-glium-renderer = "0.3.0" +imgui-opengl-renderer = "0.7.0" +imgui-sdl2 = "0.9.0" # for imgui-opengl-renderer +imgui-winit-support = "0.3.1" +k4a-sys-temp = "0.2.3" +kinect = "0.0.5" +#kinect = { path = "../kinect.rs" } +libc = "0.2.68" +log = "0.4.11" +nalgebra = "0.21.1" # linear algebra and matrices, fantastic docs +nalgebra-glm = "0.7.0" # computer graphics-friendly nalgebra, inspired by C++'s glm +rand = "0.7.3" +sdl2 = "0.33" # for imgui-opengl-renderer +tobj = { version = "1.0", features = ["log"] }# Another obj loader (using older version to match `tobj_viewer` binary/crate) +winit = "=0.21.0" # cross-platform window management, because imgui +zmq = { version = "0.9", features = ['vendored'] } \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..e07180449 --- /dev/null +++ b/README.md @@ -0,0 +1,179 @@ +Cloudcam (WIP) +============== + +This is a Rust server that polls Kinect camera(s) and sends +streaming point clouds to Unreal Engine or some other consumer. + +Right now the implementation is incredibly naive, but in the +future we may be able to implement compression, keyframes, and +other advanced features. + +This was forked from my original "rust-experiments" repo, so it +has a lot of dead code that needs to be culled. Once this is ready +it will simply be code that interfaces with Kinect and talks to +consumers. + +The binary of interest is `main_zeromq.rs` (`cloudcam_zeromq`). + +Installation / Requirements +--------------------------- +Install the following packages on Linux: + +* clang +* cmake +* libsdl2-dev +* libxcb-xfixes0-dev +* GLFW build requirements: + * libpthread (eg. libpthread-stubs0-dev) + * libx11-dev + * libxcursor-dev + * libxi-dev + * libxinerama-dev + * libxrandr-dev + +Supporting devices over USB in Linux +------------------------------------ +Sometimes the cameras just won't work. We need to increase USB memory and +copy a device config file. + +Modify this: + +``` +sudo sh -c 'echo 2047 > /sys/module/usbcore/parameters/usbfs_memory_mb' +``` + +Grub bootloader modifications: + +``` +sudo vi /etc/default/grub +``` + +Then modify this directive to add the following: + +``` +GRUB_CMDLINE_LINUX_DEFAULT="[KEEP EXISTING PARTS THE SAME] usbcore.usbfs_memory_mb=2047" +``` + +Then + +``` +sudo update-grub +``` + +(Reboot) + +If `sudo k4aviewer` works but non-sudo doesn't, + +``` +git clone git@github.com:microsoft/Azure-Kinect-Sensor-SDK.git +cd Azure-Kinect-Sensor-SDK/ +sudo cp scripts/99-k4a.rules /etc/udev/rules.d/ +``` + +Then disconnect and reconnect the USB device. It should work now. + +Ubuntu 18.04 Installation +------------------------- + +Configure the Microsoft PPA and install the packages: + +``` +# Configure PPA +curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - +sudo apt-add-repository https://packages.microsoft.com/ubuntu/18.04/prod +sudo apt-get update + +#Install packages: +sudo apt install k4a-tools +sudo apt install libk4a1.1-dev +``` + + +Ubuntu 20.04 LTS Support +------------------------ + +Azure Kinect isn't officially supported on 20.04 + +mpdroid's comment on October 17, 2020 seems to indicate that there is a workaround: + +https://feedback.azure.com/forums/920053-azure-kinect-dk/suggestions/40368301-support-for-ubuntu-20-04-lts?page=1&per\_page=20 + +``` +These are the steps I followed to install k4a-tools, libk4a and libk4abt on Ubuntu 20.04. The general steps are as outlined in +https://docs.microsoft.com/en-us/azure/Kinect-dk/sensor-sdk-download, with a couple of hacks to make things work on 20.04. +- use of 18.04 repo, even though OS is 20.04 +- installed lower versions of tools and libraries (as latest versions of sensor and body tracker don't seem to be compatible on 20.04) + +$ curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - +$ sudo apt-add-repository https://packages.microsoft.com/ubuntu/18.04/prod +$ curl -sSL https://packages.microsoft.com/config/ubuntu/18.04/prod.list | sudo tee /etc/apt/sources.list.d/microsoft-prod.list +$ curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - +$ sudo apt-get update +$ sudo apt install libk4a1.3-dev +$ sudo apt install libk4abt1.0-dev +$ sudo apt install k4a-tools=1.3.0 + +- Verify sensor library by launching camera viewer + +$ k4aviewer + +- Clone and build [Azure Kinect Samples](https://github.com/microsoft/Azure-Kinect-Samples). +- Run `simple_3d_viewer` to verify body tracker works. +``` + +There will be complaints: + +``` +# E: The repository 'https://packages.microsoft.com/ubuntu/18.04/prod focal Release' does not have a Release file. +# N: Updating from such a repository can't be done securely, and is therefore disabled by default. +``` + +Just ignore them. + +Webcam +------ + +Update: + +This seems to work with some combination of reloading the driver (`./webcam/reload_driver.sh`), running VLC +(maybe it sets the video buffer mode?), killing firefox, and starting random browsers and going to meet. +Eventually it seems to work. + +Docs and software: + +- software: https://github.com/webcamoid/akvcam +- wiki: https://github.com/webcamoid/akvcam/wiki/Usage-and-examples + +The configuration I use is as follows (this may have changed with the new ini file): + +- /dev/video0 is the input +- /dev/video1 is the output + +Check to see if the driver is okay: + +``` +v4l2-compliance -d /dev/video0 -f +``` + +Check properties: + +``` +v4l2-ctl -d /dev/video0 +``` + +Playback: + +``` +ffplay /dev/video1 +``` + +Test writing random noise: + +``` +cat /dev/urandom > /dev/video0 +``` + +This may have the answers for my bugs: + +- https://github.com/webcamoid/akvcam/issues/6 + diff --git a/src/assets/mod.rs b/src/assets/mod.rs new file mode 100644 index 000000000..711910a61 --- /dev/null +++ b/src/assets/mod.rs @@ -0,0 +1,3 @@ +pub mod obj_loader; +pub mod positionable_object; +pub mod renderable_object; \ No newline at end of file diff --git a/src/assets/obj_loader.rs b/src/assets/obj_loader.rs new file mode 100644 index 000000000..deae7a403 --- /dev/null +++ b/src/assets/obj_loader.rs @@ -0,0 +1,114 @@ +use anyhow::Result as AnyhowResult; +use std::path::Path; +use std::time::{Duration, Instant}; +use glium::{self, Display}; +use glium::vertex::VertexBufferAny; +use std::f32; + +#[derive(Copy, Clone)] +pub struct ExtractedVertex { + pub position: [f32; 3], + pub normal: [f32; 3], + pub color_diffuse: [f32; 3], + pub color_specular: [f32; 4], + pub tex_coords: [f32; 2], // [u, v] +} + +/// Returns a vertex buffer that should be rendered as `TrianglesList`. +/// Adapted from https://github.com/Twinklebear/tobj_viewer/blob/master/src/support/mod.rs +pub fn load_wavefront(path: &Path) -> AnyhowResult> +//pub fn load_wavefront(display: &Display, path: &Path) + //-> (VertexBufferAny, f32) +{ + // TODO: This appears to be one of two hooks for the data to be uploaded to OpenGL + implement_vertex!(ExtractedVertex, position, normal, color_diffuse, color_specular, tex_coords); + + let mut min_pos = [f32::INFINITY; 3]; + let mut max_pos = [f32::NEG_INFINITY; 3]; + let mut vertex_data = Vec::new(); + + println!("loading {:?}...", path); + let (models, mats) = tobj::load_obj(path)?; + println!("loaded {:?}...", path); + + // Just upload the first object in the group + for model in &models { + let mesh = &model.mesh; + //println!("Uploading model: {}", model.name); + + for idx in &mesh.indices { + let i = *idx as usize; + + let pos = [ + mesh.positions[3 * i], + mesh.positions[3 * i + 1], + mesh.positions[3 * i + 2], + ]; + + let normal = if !mesh.normals.is_empty() { + [ + mesh.normals[3 * i], + mesh.normals[3 * i + 1], + mesh.normals[3 * i + 2], + ] + } else { + [0.0, 0.0, 0.0] + }; + + let tex_coords = if !mesh.texcoords.is_empty() { + // texcoord = [u, v]; + [ + mesh.texcoords[i * 2], + mesh.texcoords[i * 2 + 1] + ] + } else { + [0.0, 0.0] + }; + + let (color_diffuse, color_specular) = match mesh.material_id { + Some(i) => ( + mats[i].diffuse, + [ + mats[i].specular[0], + mats[i].specular[1], + mats[i].specular[2], + mats[i].shininess, + ], + ), + None => ([0.8, 0.8, 0.8], [0.15, 0.15, 0.15, 15.0]), + }; + + vertex_data.push(ExtractedVertex { + position: pos, + normal: normal, + color_diffuse: color_diffuse, + color_specular: color_specular, + tex_coords: tex_coords, + }); + + // Update our min/max pos so we can figure out the bounding box of the object + // to view it + for i in 0..3 { + min_pos[i] = f32::min(min_pos[i], pos[i]); + max_pos[i] = f32::max(max_pos[i], pos[i]); + } + } + } + + Ok(vertex_data) + + // TODO: This appears to be the second hook: + // Compute scale factor to fit the model with a [-1, 1] bounding box + /*let diagonal_len = 6.0; + let current_len = f32::powf(max_pos[0] - min_pos[0], 2.0) + + f32::powf(max_pos[1] - min_pos[1], 2.0) + + f32::powf(max_pos[2] - min_pos[2], 2.0); + let scale = f32::sqrt(diagonal_len / current_len); + println!("Model scaled by {} to fit", scale); + ( + glium::vertex::VertexBuffer::new(display, &vertex_data) + .unwrap() + .into_vertex_buffer_any(), + scale, + )*/ +} \ No newline at end of file diff --git a/src/assets/positionable_object.rs b/src/assets/positionable_object.rs new file mode 100644 index 000000000..e8582c77c --- /dev/null +++ b/src/assets/positionable_object.rs @@ -0,0 +1,142 @@ +use crate::assets::renderable_object::RenderableObject; +use gl::types::*; +use nalgebra::{Vector4, Rotation3, Translation3, Matrix4, Vector3, RealField}; +use std::ffi::CString; +use std::os::raw::c_char; +use crate::opengl::wrapper::uniform::Uniform; + +#[derive(Copy,Clone,Debug,PartialEq,Eq)] +pub enum ObjectType { + Skybox, + Level, + Character, + Unknown, +} + +pub struct PositionableObject { + pub renderable_object: RenderableObject, + + // Transformation data for the object + + pub scale: Vector3, + pub rotation: Vector4, + pub translation: Vector3, + + //pub rotation_typed: Rotation3, + //pub translation_typed: Translation3, + //pub is_transformed: bool, + pub object_type: ObjectType, +} + +impl PositionableObject { + + pub fn new(renderable_object: RenderableObject) -> Self { + Self { + renderable_object, + scale: Vector3::new(1.0, 1.0, 1.0), + rotation: Vector4::default(), + translation: Vector3::default(), + //rotation_typed: Rotation3::identity(), + //translation_typed: Translation3::identity(), + //is_transformed: true, + object_type: ObjectType::Unknown, + } + } + + pub fn rotate(&mut self, x: f32, y: f32, z: f32) { + self.rotation.x = x; + self.rotation.y = y; + self.rotation.z = z; + } + + pub fn rotate_x(&mut self, x: f32) { + self.rotation.x = x; + } + + pub fn rotate_y(&mut self, y: f32) { + self.rotation.y = y; + } + + pub fn rotate_z(&mut self, z: f32) { + self.rotation.z = z; + } + + /// Add pi to x rotation + pub fn flip_x(&mut self) { + self.rotation.x += f32::pi(); + } + + /// Add pi to y rotation + pub fn flip_y(&mut self) { + self.rotation.y += f32::pi(); + } + + /// Add pi to z rotation + pub fn flip_z(&mut self) { + self.rotation.z += f32::pi(); + } + + pub fn increment_rotation_x(&mut self, x: f32) { + self.rotation.x += x; + } + + pub fn increment_rotation_y(&mut self, y: f32) { + self.rotation.y += y; + } + + pub fn increment_rotation_z(&mut self, z: f32) { + self.rotation.z += z; + } + + pub fn scale_nonuniform(&mut self, x: f32, y: f32, z: f32) { + self.scale.x = x; + self.scale.y = y; + self.scale.z = z; + } + + pub fn scale(&mut self, scale: f32) { + self.scale.x = scale; + self.scale.y = scale; + self.scale.z = scale; + } + + pub fn translate(&mut self, x: f32, y: f32, z: f32) { + self.translation.x = x; + self.translation.y = y; + self.translation.z = z; + } + + pub fn translate_x(&mut self, x: f32) { + self.translation.x = x; + } + + pub fn translate_y(&mut self, y: f32) { + self.translation.y = y; + } + + pub fn translate_z(&mut self, z: f32) { + self.translation.z = z; + } + + pub fn draw(&self, model_transform_id: Uniform) { + let mut transformation = Matrix4::identity(); + let rotation = Matrix4::from_euler_angles(self.rotation.x, self.rotation.y, self.rotation.z); + + transformation.append_nonuniform_scaling_mut(&self.scale); + transformation = transformation * rotation; + transformation.append_translation_mut(&self.translation); + + //println!("matrix: {:?}", transformation); + //println!("translation: {:?}", &self.translation); + + let mat_ptr = transformation.as_ptr(); + + self.renderable_object.vao.bind(); + + unsafe { + gl::UniformMatrix4fv(model_transform_id.id(), 1, gl::FALSE, mat_ptr); + } + + self.renderable_object.draw(true); + } +} diff --git a/src/assets/renderable_object.rs b/src/assets/renderable_object.rs new file mode 100644 index 000000000..bde92866d --- /dev/null +++ b/src/assets/renderable_object.rs @@ -0,0 +1,247 @@ +use anyhow::Result as AnyhowResult; +use anyhow::bail; +use crate::assets::obj_loader::{ExtractedVertex, load_wavefront}; +use crate::opengl::wrapper::buffer::Buffer; +use crate::opengl::wrapper::texture::Texture; +use crate::opengl::wrapper::vertex_array::VertexArray; +use gl::types::*; +use std::ffi::{c_void, CString}; +use std::mem::size_of; +use std::os::raw::c_char; +use std::path::Path; +use std::ptr::null; +use image::{DynamicImage, Pixel, Rgba, RgbaImage, ImageBuffer, RgbImage}; +use crate::opengl::wrapper::uniform::Uniform; +use image::imageops::{flip_horizontal, flip_vertical}; +use crate::opengl::wrapper::attribute::Attribute; + +/// A collection of all the graphics-related data to render: vertices, normals, etc. +pub struct RenderableObject { + pub vao: VertexArray, + pub vertex_buffer: Buffer, + pub normal_buffer: Buffer, + pub color_buffer: Buffer, + pub texture_coords_buffer: Buffer, + pub texture: Texture, + pub num_vertices: usize, + pub buffered: bool, +} + +impl RenderableObject { + pub fn new() -> Self { + unsafe { + let vao = VertexArray::new_initialized(); + vao.bind(); + + let vertex_buffer = Buffer::new_initialized(); + let normal_buffer = Buffer::new_initialized(); + let color_buffer = Buffer::new_initialized(); + let texture_coords_buffer = Buffer::new_initialized(); + let texture = Texture::new_initialized(); + + Self { + vao, + vertex_buffer, + normal_buffer, + color_buffer, + texture_coords_buffer, + texture, + num_vertices: 0, + buffered: false, + } + } + } + + pub fn from_wavefront(path: &Path, shader_id: GLuint) -> AnyhowResult { + let vertices = load_wavefront(&path)?; + + let mut renderable_object = RenderableObject::new(); + renderable_object.load_vertices(&vertices, shader_id); + + renderable_object.load_texture_coordinates(&vertices, shader_id); + + Ok(renderable_object) + } + + pub fn load_vertices(&mut self, vertex_data: &Vec, shader_id: GLuint) { + self.vao.bind(); + self.vertex_buffer.bind_as_array_buffer(); + + let size = vertex_data.len() * 3 * size_of::(); + let mut vertices : Vec = Vec::with_capacity(vertex_data.len() * 3); + + for v in vertex_data.into_iter() { + vertices.extend(&v.position); + } + + self.num_vertices = vertex_data.len(); + + println!("Object vertices loaded: {}", vertices.len()); + + unsafe { + let vertices_ptr = vertices.as_ptr() as *const c_void; + + gl::BufferData( + gl::ARRAY_BUFFER, + size as isize, + vertices_ptr, + gl::STATIC_DRAW, + ); + + let name : CString = CString::new("position").expect("string is correct"); + let name_ptr : *const c_char = name.as_ptr() as *const c_char; + + let loc = gl::GetAttribLocation(shader_id, name_ptr); + println!("Attrib location: {}", loc); + + gl::EnableVertexAttribArray(loc as u32); + + gl::VertexAttribPointer(loc as u32, 3, gl::FLOAT, gl::FALSE, 0, null()); + } + } + + pub fn set_vertex_type(&mut self, vertex_type: &Uniform) { + /*self.vao.bind(); + let loc = vertex_type.id(); + println!("Uniform location: {}", loc); + + unsafe { + //gl::EnableVertexAttribArray(loc as u32); + //gl::VertexAttribIPointer(loc as u32, 1, gl::UNSIGNED_BYTE, 0, null()); + //gl::VertexAttribI1ui(loc as u32, 0); + //gl::Uniform1i(loc, 1); + }*/ + + //vertex_type.set_uint(0); + } + + pub fn load_texture_coordinates(&mut self, vertex_data: &Vec, shader_id: GLuint) { + self.vao.bind(); + self.texture_coords_buffer.bind_as_array_buffer(); + + let size = vertex_data.len() * 2 * size_of::(); + let mut tex_coords : Vec = Vec::with_capacity(vertex_data.len() * 2); + + for v in vertex_data.into_iter() { + tex_coords.extend(&v.tex_coords); + } + + unsafe { + let tex_coords_ptr = tex_coords.as_ptr() as *const c_void; + + gl::BufferData( + gl::ARRAY_BUFFER, + size as isize, + tex_coords_ptr, + gl::STATIC_DRAW, + ); + + let name : CString = CString::new("vTextureCoord").expect("string is correct"); + let name_ptr : *const c_char = name.as_ptr() as *const c_char; + + let loc = gl::GetAttribLocation(shader_id, name_ptr); + println!("Attrib location: {}", loc); + + gl::EnableVertexAttribArray(loc as u32); + + gl::VertexAttribPointer(loc as u32, 2, gl::FLOAT, gl::FALSE, 0, null()); + + // TODO: This is in my old OpenGL program, but not the rust package? + //gl::TexCoordPointer(2, gl::FLOAT, 0, tex_coords_ptr); + } + } + + // TODO: Option for 'rgb'/'rgba' + // TODO: Option to flip texture on any axis + pub fn load_texture(&mut self, filename: &str, texture_uniform: &Uniform) -> AnyhowResult<()> { + let img = image::open(&filename)?; + + let mut rgba_image= img.to_rgba(); + + //let rgb_image : RgbImage = flip_horizontal(&rgb_image); + let rgba_image : RgbaImage = flip_vertical(&rgba_image); + + let width = rgba_image.width(); + let height = rgba_image.height(); + + let img_data = rgba_image.into_raw(); + let pixel_data = img_data.as_ptr() as *const c_void; + + self.texture.bind_as_texture_2d(); + + println!("Loading texture into OpenGL {}x{}...", width, height); + + unsafe { + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::LINEAR as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::LINEAR as i32); + + //gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_S, gl::REPEAT as i32); + //gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_T, gl::REPEAT as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_S, gl::CLAMP_TO_BORDER as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_T, gl::CLAMP_TO_BORDER as i32); + + let border_color : Vec = vec![0.0, 1.0, 0.0, 1.0]; + let border_color_ptr = border_color.as_ptr() as *const f32; + gl::TexParameterfv(gl::TEXTURE_2D, gl::TEXTURE_BORDER_COLOR, border_color_ptr); + + gl::TexImage2D( + gl::TEXTURE_2D, + 0, // image level (mipmap); 0 is base + gl::RGBA as i32, // internal format + width as i32, + height as i32, + 0, // border; "must be 0" + gl::RGBA, // format of pixels + gl::UNSIGNED_BYTE, // pixel data type + pixel_data, + ); + + /* + // Checkerboard texture. + let pixels : Vec = vec![ + 0.0, 0.0, 0.0, 1.0, + 1.0, 1.0, 1.0, 1.0, + 0.0, 0.0, 0.0, 1.0, + 1.0, 1.0, 1.0, 1.0, + ]; + + let pixels_ptr = pixels.as_ptr() as *const c_void; + + gl::TexImage2D( + gl::TEXTURE_2D, + 0, // image level (mipmap); 0 is base + gl::RGBA as i32, // internal format + 2, + 2, + 0, // border; "must be 0" + gl::RGBA, // format of pixels + gl::FLOAT, // pixel data type + pixels_ptr, + ); + */ + + gl::GenerateMipmap(gl::TEXTURE_2D); + + println!("texture uniform: {}", texture_uniform.id()); + + // TODO: This is in my OpenGL program, but is broken here. + //gl::Uniform1i(texture_uniform.id(), 0); + + gl::ActiveTexture(gl::TEXTURE0); + } + + Ok(()) + } + + pub fn draw(&self, vao_already_bound: bool) { + if !vao_already_bound { + self.vao.bind(); + } + unsafe { + gl::BindTexture(gl::TEXTURE_2D, self.texture.id()); + + gl::DrawArrays(gl::TRIANGLES, 0, self.num_vertices as i32); + } + } +} + diff --git a/src/azure_kinect/capture/device_capturer.rs b/src/azure_kinect/capture/device_capturer.rs new file mode 100644 index 000000000..cf48fd2f3 --- /dev/null +++ b/src/azure_kinect/capture/device_capturer.rs @@ -0,0 +1,17 @@ +use crate::point_cloud::debug::capture_proxy::CaptureProxy; +use kinect::Calibration; + +pub trait CaptureProvider { + + /** Return the number of cameras. */ + fn get_num_cameras(&self) -> usize; + + /** + * Take the latest captures, if available, through interior mutability. + * This leaves the mutex holding an empty vec. + */ + fn get_captures(&self) -> Option>; + + /** Get device calibrations for all cameras. */ + fn get_calibrations(&self) -> &Vec; +} diff --git a/src/azure_kinect/capture/fake_device_capturer.rs b/src/azure_kinect/capture/fake_device_capturer.rs new file mode 100644 index 000000000..b8a9b65cd --- /dev/null +++ b/src/azure_kinect/capture/fake_device_capturer.rs @@ -0,0 +1,183 @@ +use anyhow::Result as AnyhowResult; +use crate::azure_kinect::capture::device_capturer::CaptureProvider; +use crate::point_cloud::debug::capture_proxy::CaptureProxy; +use crate::point_cloud::debug::image_proxy::ImageProxy; +use k4a_sys_temp as k4a_sys; +use kinect::{ImageFormat, Calibration}; + +pub struct FakeDeviceCaptureProvider { + num_cameras: usize, + captures: Vec, + all_device_calibrations: Vec, +} + +impl FakeDeviceCaptureProvider { + pub fn new() -> AnyhowResult { + // TODO: Color dimension stride is wrong (and maybe others) + let color = ImageProxy::from_file("output/color_frame_0", 3840, 2160, 0, ImageFormat::ColorBgra32)?; + let depth = ImageProxy::from_file("output/depth_frame_0", 640, 576, 1280, ImageFormat::Depth16)?; + let frame_1 = CaptureProxy::from_image_proxy_pair(color, depth); + + // TODO: Color dimension stride is wrong (and maybe others) + let color = ImageProxy::from_file("output/color_frame_1", 3840, 2160, 0, ImageFormat::ColorBgra32)?; + let depth = ImageProxy::from_file("output/depth_frame_1", 640, 576, 1280, ImageFormat::Depth16)?; + let frame_2 = CaptureProxy::from_image_proxy_pair(color, depth); + + let mut captures = Vec::new(); + captures.push(frame_1); + captures.push(frame_2); + + let mut calibration = Calibration::default(); + + // NB: Experimentally determined "constants" from real devices + // These are necessary to compute the depth camera transform into color camera space. + calibration.0.color_resolution = 5; + calibration.0.depth_mode = 2; + calibration.0.depth_camera_calibration.resolution_width = 640; + calibration.0.depth_camera_calibration.resolution_height = 576; + calibration.0.depth_camera_calibration.metric_radius = 1.7399998; + calibration.0.depth_camera_calibration.extrinsics.rotation = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + calibration.0.depth_camera_calibration.extrinsics.translation = [0.0, 0.0, 0.0]; + calibration.0.depth_camera_calibration.intrinsics.type_ = 4; + calibration.0.depth_camera_calibration.intrinsics.parameter_count = 14; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.cx = 325.4652; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.cy = 339.75952; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.fx = 503.72073; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.fy = 503.82776; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k1 = 5.7966313; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k2 = 4.1091304; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k3 = 0.22066948; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k4 = 6.1226273; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k5 = 6.053665; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.k6 = 1.1403816; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.codx = 0.0; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.cody = 0.0; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.p2 = -0.00003910405; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.p1 = -0.00009726079; + calibration.0.depth_camera_calibration.intrinsics.parameters.param.metric_radius = 0.0; + + calibration.0.color_camera_calibration.resolution_width = 3840; + calibration.0.color_camera_calibration.resolution_height = 2160; + calibration.0.color_camera_calibration.metric_radius = 1.7; + calibration.0.color_camera_calibration.extrinsics.rotation = [0.99998903, -0.0045318827, 0.001187322, 0.0044161947, 0.9964592, 0.08396203, -0.001563624, -0.08395587, 0.99646825]; + calibration.0.color_camera_calibration.extrinsics.translation = [-32.02683, -2.3786376, 3.9904723]; + calibration.0.color_camera_calibration.intrinsics.type_ = 4; + calibration.0.color_camera_calibration.intrinsics.parameter_count = 14; + calibration.0.color_camera_calibration.intrinsics.parameters.param.cx = 1919.9363; + calibration.0.color_camera_calibration.intrinsics.parameters.param.cy = 1105.1877; + calibration.0.color_camera_calibration.intrinsics.parameters.param.fx = 1832.5886; + calibration.0.color_camera_calibration.intrinsics.parameters.param.fy = 1832.6873; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k1 = 0.4812507; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k2 = -2.5299993; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k3 = 1.4068391; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k4 = 0.36012942; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k5 = -2.355542; + calibration.0.color_camera_calibration.intrinsics.parameters.param.k6 = 1.3372931; + calibration.0.color_camera_calibration.intrinsics.parameters.param.codx = 0.0; + calibration.0.color_camera_calibration.intrinsics.parameters.param.cody = 0.0; + calibration.0.color_camera_calibration.intrinsics.parameters.param.p2 = -0.00037526494; + calibration.0.color_camera_calibration.intrinsics.parameters.param.p1 = 0.00054938474; + calibration.0.color_camera_calibration.intrinsics.parameters.param.metric_radius = 0.0; + calibration.0.extrinsics = [ + [ + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0], + translation: [0.0, 0.0, 0.0] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99998903, -0.0045318827, 0.001187322, 0.0044161947, 0.9964592, 0.08396203, -0.001563624, -0.08395587, 0.99646825], + translation: [-32.02683, -2.3786376, 3.9904723] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.00908543, 0.1157082, -0.99324167, -0.9999575, -0.0005106917, -0.009206355, -0.001572491, 0.9932831, 0.11569864], + translation: [0.0, 0.0, 0.0] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.004534903, 0.112131886, -0.993683, -0.9999893, -0.00040909834, -0.004609848, -0.00092342496, 0.99369323, 0.11212883], + translation: [-51.077, 3.2191863, 1.8031791] + } + ], + [ + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99998903, 0.0044161947, -0.001563624, -0.0045318827, 0.9964592, -0.08395587, 0.001187322, 0.08396203, 0.99646825], + translation: [32.043224, 2.560097, -3.7386374] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [1.0, 0.00000000026193447, 0.0, 0.00000000026193447, 1.0, -0.000000007450581, 0.0, -0.000000007450581, 1.0], + translation: [0.0000038146973, 0.0, 0.0] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.007381657, 0.031944036, -0.99946237, -0.9999551, -0.005697875, -0.007567407, -0.005936545, 0.9994734, 0.03190054], + translation: [4.300721, -32.00875, 2.0599582] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.0028468627, 0.028323233, -0.99959475, -0.99998194, -0.005210849, -0.0029956135, -0.005293583, 0.9995853, 0.028307885], + translation: [-46.929596, -28.807508, 3.8983316] + } + ], + [ + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.00908543, -0.9999575, -0.001572491, 0.1157082, -0.0005106917, 0.9932831, -0.99324167, -0.009206355, 0.11569864], + translation: [0.0, 0.0, 0.0] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.007381657, -0.9999551, -0.005936545, 0.031944036, -0.005697875, 0.9994734, -0.99946237, -0.007567407, 0.03190054], + translation: [-32.02683, -2.3786376, 3.9904723] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99999994, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.99999994], + translation: [0.0, 0.0, 0.0] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99998313, 0.0045562233, -0.003596194, -0.0045539755, 0.99998945, 0.00063277053, 0.0035990402, -0.0006163829, 0.99999326], + translation: [-51.077, 3.2191863, 1.8031791] + } + ], + [ + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.004534903, -0.9999893, -0.00092342496, 0.112131886, -0.00040909834, 0.99369323, -0.993683, -0.004609848, 0.11212883], + translation: [3.452446, 3.9368703, -50.941692] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.0028468627, -0.99998194, -0.005293583, 0.028323233, -0.005210849, 0.9995853, -0.99959475, -0.0029956135, 0.028307885], + translation: [-28.652746, -2.717628, -47.10723] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99998313, -0.0045539755, 0.0035990402, 0.0045562233, 0.99998945, -0.0006163829, -0.003596194, 0.00063277053, 0.99999326], + translation: [51.08431, -2.9853227, -1.9888868] + }, + k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.99999994, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.99999994], + translation: [-0.0000038146973, 0.00000023841858, 0.00000011920929] } + ] + ]; + + let mut all_device_calibrations = Vec::new(); + for _ in 0 .. 2 { + all_device_calibrations.push(calibration.clone()); + } + + Ok(Self { + num_cameras: 2, + captures, + all_device_calibrations, + }) + } +} + +impl CaptureProvider for FakeDeviceCaptureProvider { + fn get_num_cameras(&self) -> usize { + self.num_cameras + } + + fn get_captures(&self) -> Option> { + let capture_clones = self.captures.iter() + .map(|capture| capture.clone()) + .collect(); + Some(capture_clones) + } + + fn get_calibrations(&self) -> &Vec { + &self.all_device_calibrations + } +} diff --git a/src/azure_kinect/capture/mod.rs b/src/azure_kinect/capture/mod.rs new file mode 100644 index 000000000..9c8a1fc4c --- /dev/null +++ b/src/azure_kinect/capture/mod.rs @@ -0,0 +1,3 @@ +pub mod device_capturer; +pub mod fake_device_capturer; +pub mod multi_device_capturer; diff --git a/src/azure_kinect/capture/multi_device_capturer.rs b/src/azure_kinect/capture/multi_device_capturer.rs new file mode 100644 index 000000000..286c8a4be --- /dev/null +++ b/src/azure_kinect/capture/multi_device_capturer.rs @@ -0,0 +1,263 @@ +use crate::azure_kinect::capture::device_capturer::CaptureProvider; +use crate::point_cloud::debug::capture_proxy::CaptureProxy; +use k4a_sys_temp as k4a_sys; +use kinect::{Device, KinectError, Capture, CaptureError, GetCaptureError, Calibration}; +use std::sync::Arc; +use std::sync::Mutex; + +// Allowing at least 160 microseconds between depth cameras should ensure they do not interfere with one another. +const MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC : libc::int32_t = 160; + +pub struct MultiDeviceCapturer { + num_cameras: usize, + pub primary_device: Device, // TODO: Temporary public viz + secondary_devices: Vec, + /// Calibrations for primary + all secondaries in order. + all_device_calibrations: Vec, + capture_provider: Arc, +} + +pub struct MultiDeviceCaptureProvider { + num_cameras: usize, + /// Calibrations for primary + all secondaries in order. + all_device_calibrations: Vec, + captures: Arc>>, // TODO: Shouldn't need to wrap in mutex if we wrap struct instead. +} + +impl MultiDeviceCapturer { + + // TODO: Don't panic everywere. + /// Open the primary and secondary devices. + pub fn new(num_devices: u32, limit_secondary_devices: Option) -> Result { + let mut primary_device = None; + let mut secondary_devices = Vec::new(); + let mut calibrations = Vec::new(); + + let depth_mode : k4a_sys::k4a_depth_mode_t = 2; //k4a_sys::K4A_DEPTH_MODE_NFOV_UNBINNED; + let color_format: k4a_sys::k4a_color_resolution_t = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_2160P; + + for i in 0..num_devices { + let mut device = Device::open(i)?; + let jack_status = device.get_synchronization_jack_status()?; + + if jack_status.sync_out_jack_connected && !jack_status.sync_in_jack_connected { + if primary_device.is_some() { + // NB: This assumes a daisy chain topology. + panic!("We already one primary device. We can't have two."); + } + + let calibration = device.get_calibration(depth_mode, color_format).unwrap(); + + calibrations.push(calibration); + primary_device = Some(device); + + continue; + } + + if !jack_status.sync_in_jack_connected { + panic!("Secondary device does not have 'in' jack connected!"); + } + + let calibration = device.get_calibration(depth_mode, color_format).unwrap(); + + secondary_devices.push(device); + calibrations.push(calibration); + } + + if primary_device.is_none() { + panic!("Could not find primary device!"); + } + + if let Some(limit) = limit_secondary_devices { + secondary_devices.truncate(limit); + } + + let num_cameras = 1 + secondary_devices.len(); + + let primary = primary_device.expect("There must be a primary device"); + + let calibrations2 = calibrations.clone(); + + Ok(Self { + num_cameras, + primary_device: primary, + secondary_devices, + all_device_calibrations: calibrations, + capture_provider: Arc::new(MultiDeviceCaptureProvider::new(num_cameras, calibrations2)), + }) + } + + /// Return the number of cameras + pub fn get_num_cameras(&self) -> usize { + self.num_cameras + } + + // TODO: + // Per Microsoft's "green screen" example, it's possible to differentially configure things such + // as exposure, but we won't consider that at the moment. When we do, there is a lot of good + // documentation in the 'green_screen' source code that speaks to important timing delay + // considerations. + /// Start cameras all with the same configuration + pub fn start_cameras(&self) -> Result<(), KinectError> { + let secondary_config = get_secondary_device_config(); + + // NB: Secondary devices *MUST* be started before the master! + for device in self.secondary_devices.iter() { + device.start_cameras(secondary_config)?; + } + + // Once the secondaries start, we can start the primary. + let primary_config = get_primary_device_config(); + self.primary_device.start_cameras(primary_config)?; + + Ok(()) + } + + pub fn get_synchronized_captures(&self) -> Result, GetCaptureError> { + let mut captures = Vec::with_capacity(1 + self.secondary_devices.len()); + + // TODO: -1 is K4A_WAIT_INDEFINITE + let capture = self.primary_device.get_capture(-1)?; + captures.push(capture); + + for device in self.secondary_devices.iter() { + let capture = device.get_capture(-1)?; + captures.push(capture); + } + + // TODO TODO TODO - Lots of sync logic goes here + // TODO TODO TODO - Lots of sync logic goes here + // TODO TODO TODO - Lots of sync logic goes here + // TODO TODO TODO - Lots of sync logic goes here + + Ok(captures) + } + + /// Get a capture provider for another thread + pub fn get_sync_capture_provider(&self) -> Arc { + self.capture_provider.clone() + } +} + +impl MultiDeviceCaptureProvider { + pub fn new(num_cameras: usize, all_device_calibrations: Vec) -> Self { + Self { + num_cameras, + all_device_calibrations, + captures: Arc::new(Mutex::new(Vec::new())), + } + } + + /** + * Consume captures and replace whatever we currently hold. + */ + pub fn set_captures(&self, captures: Vec) { + match self.captures.lock() { + Ok(mut lock) => { + *lock = captures; + }, + Err(_) => {}, + } + } +} + +impl CaptureProvider for MultiDeviceCaptureProvider { + + /** Return the number of cameras. */ + fn get_num_cameras(&self) -> usize { + self.num_cameras + } + + /** + * Take the latest captures, if available, through interior mutability. + * This leaves the mutex holding an empty vec. + */ + fn get_captures(&self) -> Option> { + self.captures.lock() + .ok() + .map(|mut v| (*v).split_off(0)) + .map(|mut captures| captures.into_iter() + .map(|capture| CaptureProxy::consume_k4a_capture(capture)) + .collect()) + } + + fn get_calibrations(&self) -> &Vec { + &self.all_device_calibrations + } +} + +pub fn start_capture_thread(capturer: MultiDeviceCapturer) { + let mut capture_provider = capturer.get_sync_capture_provider(); + + loop { + let captures = capturer.get_synchronized_captures() + .expect("Should get captures"); + + capture_provider.set_captures(captures); + } +} + +fn get_primary_device_config() -> k4a_sys::k4a_device_configuration_t { + let mut config = get_default_device_config(); + config.wired_sync_mode = k4a_sys::k4a_wired_sync_mode_t_K4A_WIRED_SYNC_MODE_MASTER; + config.synchronized_images_only = true; + + // Two depth images should be seperated by MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC to ensure the depth imaging + // sensor doesn't interfere with the other. To accomplish this the master depth image captures + // (MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2) before the color image, and the subordinate camera captures its + // depth image (MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2) after the color image. This gives us two depth + // images centered around the color image as closely as possible. + config.depth_delay_off_color_usec = -(MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2); + + // TODO: These are the old values I used that are NOT in 'green_screen'. + // They may need to be adjusted + config.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_2160P; + config.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_UNBINNED; + //config.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_30; + + config +} + +fn get_secondary_device_config() -> k4a_sys::k4a_device_configuration_t { + let mut config = get_default_device_config(); + config.wired_sync_mode = k4a_sys::k4a_wired_sync_mode_t_K4A_WIRED_SYNC_MODE_SUBORDINATE; + + // Two depth images should be seperated by MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC to ensure the depth imaging + // sensor doesn't interfere with the other. To accomplish this the master depth image captures + // (MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2) before the color image, and the subordinate camera captures its + // depth image (MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2) after the color image. This gives us two depth + // images centered around the color image as closely as possible. + config.depth_delay_off_color_usec = MIN_TIME_BETWEEN_DEPTH_CAMERA_PICTURES_USEC / 2; + + // TODO: These are the old values I used that are NOT in 'green_screen'. + // They may need to be adjusted + config.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_2160P; + config.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_UNBINNED; + //config.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_30; + + config +} + + +/// Sensible shared defaults +fn get_default_device_config() -> k4a_sys::k4a_device_configuration_t { + k4a_sys::k4a_device_configuration_t { + color_format : k4a_sys::k4a_image_format_t_K4A_IMAGE_FORMAT_COLOR_BGRA32, + color_resolution : k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_720P, + // No need for depth during calibration + // TODO: We need it. + depth_mode : k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_UNBINNED, + // Don't use all USB bandwidth + // TODO: This will be a problem. + camera_fps : k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_15, + // Must be zero for master + subordinate_delay_off_master_usec : 0, + synchronized_images_only : true, + + // These fields were not set and come from `K4A_DEVICE_CONFIG_INIT_DISABLE_ALL`: + depth_delay_off_color_usec: 0, + disable_streaming_indicator: false, + wired_sync_mode: k4a_sys::k4a_wired_sync_mode_t_K4A_WIRED_SYNC_MODE_STANDALONE, + } +} + diff --git a/src/azure_kinect/k4a_sys_wrapper.rs b/src/azure_kinect/k4a_sys_wrapper.rs new file mode 100644 index 000000000..f382d16b8 --- /dev/null +++ b/src/azure_kinect/k4a_sys_wrapper.rs @@ -0,0 +1,763 @@ +//! Wrappers for the k4a-sys crate to make it easier to use. +//! I tried using the open source 'k4a-rs' crate from the same author, +//! but it's in a bad state and won't compile against any version of 'k4a-sys'. + +use std::mem::MaybeUninit; +use std::{ptr, fmt}; +use std::ptr::null_mut; + +use k4a_sys; +use std::error::Error; +use std::fmt::Formatter; + +/*pub fn device_open() { + + k4a_sys::k4a_device_open(k4a_sys::K4A_DEVICE_DEFAULT) + if k4a_device_open(device_idx, &mut device) != k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + println!("Failed to open device #{}", device_idx); + continue; + } + } +}*/ + +// TODO: this is kind of lame. +/// A library error +#[derive(Debug)] +pub enum KinectError { + UnableToOpen { error_code: u32 }, + UnableToGetSerialNumber, + UnableToStartCameras { error_code: u32 }, + UnableToCreateImage { error_code: u32 }, + UnableToGetSyncJackStatus { error_code: u32 }, +} + +/// A Kinect Device Handle +#[derive(Debug)] +pub struct Device { + device_pointer: k4a_sys::k4a_device_t, +} + +impl Device { + /// Get the number of installed devices + pub fn get_installed_count() -> u32 { + unsafe { + k4a_sys::k4a_device_get_installed_count() + } + } + + /// Open a device with the given index + pub fn open(device_index: u32) -> Result { + let mut device_pointer: k4a_sys::k4a_device_t = ptr::null_mut(); + unsafe { + let result = k4a_sys::k4a_device_open(device_index, &mut device_pointer); + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(KinectError::UnableToOpen { error_code: result }) + } + } + Ok(Device { + device_pointer, + }) + } + + /// Fetch the device serial number. + pub fn get_serial_number(&self) -> Result { + // First we interrogate the serial number size. + let mut serial_number_length: usize = 0; + + let result = unsafe { + k4a_sys::k4a_device_get_serialnum(self.device_pointer, ptr::null_mut(), &mut serial_number_length) + }; + + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_TOO_SMALL { + return Err(KinectError::UnableToGetSerialNumber); + } + + // Now we request to fill a serial number buffer. + let mut serial_number = vec![0i8; serial_number_length]; + let serial_number_ptr = (&mut serial_number).as_mut_ptr(); + + let result = unsafe { + k4a_sys::k4a_device_get_serialnum(self.device_pointer, serial_number_ptr, &mut serial_number_length) + }; + + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(KinectError::UnableToGetSerialNumber); + } + + // NB: Library shouldn't be returning i8's + let serial_number = serial_number.iter().map(|v| *v as u8).collect(); + + String::from_utf8(serial_number) + .map(|s| s.trim_matches(char::from(0)).into()) // Remove trailing null byte + .map_err(|_| KinectError::UnableToGetSerialNumber) + } + + /// Get the device synchronization jack statuses. + /// Each device has an 'in' jack and an 'out' jack. + pub fn get_synchronization_jack_status(&self) -> Result { + let mut sync_in_jack_connected = false; + let mut sync_out_jack_connected = false; + + let result = unsafe { + k4a_sys::k4a_device_get_sync_jack(self.device_pointer, + &mut sync_in_jack_connected, &mut sync_out_jack_connected) + }; + + if result != k4a_sys::k4a_result_t_K4A_RESULT_SUCCEEDED { + return Err(KinectError::UnableToGetSyncJackStatus { error_code: result }); + } + + Ok(SynchronizationJackStatus { + sync_in_jack_connected, + sync_out_jack_connected, + }) + } + + // TODO: having a 'DeviceConfigurations' struct that goes unused is kind of gross. + /// Start the cameras + pub fn start_cameras(&self, + device_config: k4a_sys::k4a_device_configuration_t) + -> Result<(), KinectError> + { + let result = unsafe { + k4a_sys::k4a_device_start_cameras(self.device_pointer, &device_config) + }; + + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(KinectError::UnableToStartCameras { error_code: result }); + } + + return Ok(()) + } + + // TODO: More sensible defaults, or get rid of this entirely. + /// Start the cameras. + pub fn start_cameras_default_config(&self) -> Result<(), KinectError> { + let mut device_config = DeviceConfiguration::new(); + // NB: Although the Kinect docs say this format isn't natively supported by the color camera + // and that extra CPU is required, this is the only color mode supported by 'k4aviewer' 3D view. + device_config.0.color_format = k4a_sys::k4a_image_format_t_K4A_IMAGE_FORMAT_COLOR_BGRA32; + device_config.0.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_2160P; + device_config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_UNBINNED; + device_config.0.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_30; + + self.start_cameras(device_config.0) + } + + /// Stops the color and depth camera capture. + /// + /// The streaming of individual sensors stops as a result of this call. Once called, + /// k4a_device_start_cameras() may be called again to resume sensor streaming. + /// This function may be called while another thread is blocking in k4a_device_get_capture(). + /// Calling this function while another thread is in that function will result in that function + /// returning a failure. + pub fn stop_cameras(&self) { + unsafe { + k4a_sys::k4a_device_stop_cameras(self.device_pointer) + } + } + + /// Get capture and return a new buffer. + pub fn get_capture(&self, timeout_ms: i32) -> Result { + let mut capture_buffer: k4a_sys::k4a_capture_t = ptr::null_mut(); + self.get_capture_buffered(&mut capture_buffer, timeout_ms) + .map(|_| Capture(capture_buffer)) // TODO: Can capture be null? + } + + /// Get capture and reuse an existing buffer. + pub fn get_capture_buffered(&self, capture_buffer: &mut k4a_sys::k4a_capture_t, timeout_ms: i32) + -> Result<(), GetCaptureError> + { + let result = unsafe { + k4a_sys::k4a_device_get_capture(self.device_pointer, capture_buffer, timeout_ms) + }; + + match result { + k4a_sys::k4a_wait_result_t_K4A_WAIT_RESULT_SUCCEEDED => { /* ok, continue */ }, + k4a_sys::k4a_wait_result_t_K4A_WAIT_RESULT_TIMEOUT => { + return Err(GetCaptureError::TimeoutError); + }, + k4a_sys::k4a_wait_result_t_K4A_WAIT_RESULT_FAILED => { + return Err(GetCaptureError::TimeoutError); + } + _ => { + return Err(GetCaptureError::UnknownError(result)); + } + } + + Ok(()) + } + + /// Get the camera calibration for the entire Azure Kinect device. + /// + /// The calibration represents the data needed to transform between the camera views and may be + /// different for each operating depth_mode and color_resolution the device is configured to + /// operate in. + /// The calibration output is used as input to all calibration and transformation functions. + pub fn get_calibration(&self, + depth_mode: k4a_sys::k4a_depth_mode_t, + color_resolution: k4a_sys::k4a_color_resolution_t) + -> Result + { + // TODO: Why isn't the way I've been using to init structures before still working? + //let mut calibration_buffer: k4a_sys::k4a_calibration_t = ptr::null_mut(); + /*let mut calibration_buffer: k4a_sys::k4a_calibration_t = k4a_sys::k4a_calibration_t { + color_camera_calibration: 0, + color_resolution: 0, + depth_camera_calibration: 0, + depth_mode: 0, + extrinsics: [0,0,0,0], + };*/ + + unsafe { + let mut calibration_buffer: MaybeUninit = MaybeUninit::uninit(); + + let result = k4a_sys::k4a_device_get_calibration( + self.device_pointer, + depth_mode, + color_resolution, + calibration_buffer.as_mut_ptr(), + ); + + match result { + k4a_sys::k4a_result_t_K4A_RESULT_SUCCEEDED => { /* ok, continue */ }, + k4a_sys::k4a_result_t_K4A_RESULT_FAILED => { + return Err(GetCalibrationError::FailedError); + }, + _ => { + return Err(GetCalibrationError::UnknownError(result)); + }, + } + + let handle = calibration_buffer.assume_init(); + let calibration = Calibration(handle); + + Ok(calibration) + } + } +} + +/// Synchronization jack status. +#[derive(Debug,Copy,Clone)] +pub struct SynchronizationJackStatus { + pub sync_in_jack_connected: bool, + pub sync_out_jack_connected: bool, +} + +/// Errors for GetCapture +#[derive(Debug)] +pub enum GetCaptureError { + TimeoutError, + FailedError, + UnknownError(u32), +} + +#[derive(Debug)] +pub enum CaptureError { + NullCapture, +} + +impl fmt::Display for CaptureError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CaptureError") + } +} + +impl Error for CaptureError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +/// Errors for GetCalibration +#[derive(Debug)] +pub enum GetCalibrationError { + FailedError, + UnknownError(u32), +} + +#[derive(Clone)] +pub struct Calibration(pub k4a_sys::k4a_calibration_t); + +impl Calibration { + pub fn default() -> Self { + let extrinsics = k4a_sys::_k4a_calibration_extrinsics_t { + rotation: [0.0f32, 0.0f32, 0.0f32, 0.0f32, 0.0f32, 0.0f32, 0.0f32, 0.0f32, 0.0f32], + translation: [0.0f32, 0.0f32, 0.0f32], + }; + + let extrinsics_4 = [ + extrinsics.clone(), + extrinsics.clone(), + extrinsics.clone(), + extrinsics.clone(), + ]; + + let extrinsics_4_4 = [ + extrinsics_4.clone(), + extrinsics_4.clone(), + extrinsics_4.clone(), + extrinsics_4.clone(), + ]; + + let camera_calibration = k4a_sys::_k4a_calibration_camera_t { + resolution_width: 0, + resolution_height: 0, + metric_radius: 0.0f32, + extrinsics: extrinsics.clone(), + intrinsics: k4a_sys::_k4a_calibration_intrinsics_t { + type_: 0, + parameter_count: 0, + parameters: k4a_sys::k4a_calibration_intrinsic_parameters_t { + param: k4a_sys::k4a_calibration_intrinsic_parameters_t__param { + cx: 0.0f32, + cy: 0.0f32, + fx: 0.0f32, + fy: 0.0f32, + k1: 0.0f32, + k2: 0.0f32, + k3: 0.0f32, + k4: 0.0f32, + k5: 0.0f32, + k6: 0.0f32, + codx: 0.0f32, + cody: 0.0f32, + p2: 0.0f32, + p1: 0.0f32, + metric_radius: 0.0f32, + }, + }, + }, + }; + + let mut calibration = k4a_sys::k4a_calibration_t { + color_camera_calibration: camera_calibration.clone(), + depth_camera_calibration: camera_calibration.clone(), + color_resolution: 0, + depth_mode: 0, + extrinsics: extrinsics_4_4, + }; + + Self(calibration) + } + + // TODO: Make this the `Debug` trait output instead. + pub fn debug_print(&self) { + println!("===== CALIBRATION ====="); + println!("\t Color resolution: {}", self.0.color_resolution); + println!("\t Depth mode: {}", self.0.depth_mode); + println!("\t Extrinsics: {:?}", self.0.extrinsics); + + println!("\t depth.resolution_width: {}", self.0.depth_camera_calibration.resolution_width); + println!("\t depth.resolution_height: {}", self.0.depth_camera_calibration.resolution_height); + println!("\t depth.metric_radius: {}", self.0.depth_camera_calibration.metric_radius); + println!("\t depth.extrinsics: {:?}", self.0.depth_camera_calibration.extrinsics); + println!("\t depth.intrinsics.type: {}", self.0.depth_camera_calibration.intrinsics.type_); + println!("\t depth.intrinsics.parameter_count: {}", self.0.depth_camera_calibration.intrinsics.parameter_count); + unsafe { + // NB: This is a union field, so we have to use unsafe access + println!("\t depth.intrinsics.parameters.param.cx: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.cx); + println!("\t depth.intrinsics.parameters.param.cy: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.cy); + println!("\t depth.intrinsics.parameters.param.fx: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.fx); + println!("\t depth.intrinsics.parameters.param.fy: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.fy); + println!("\t depth.intrinsics.parameters.param.k1: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k1); + println!("\t depth.intrinsics.parameters.param.k2: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k2); + println!("\t depth.intrinsics.parameters.param.k3: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k3); + println!("\t depth.intrinsics.parameters.param.k4: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k4); + println!("\t depth.intrinsics.parameters.param.k5: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k5); + println!("\t depth.intrinsics.parameters.param.k6: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.k6); + println!("\t depth.intrinsics.parameters.param.codx: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.codx); + println!("\t depth.intrinsics.parameters.param.cody: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.cody); + println!("\t depth.intrinsics.parameters.param.p2: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.p2); + println!("\t depth.intrinsics.parameters.param.p1: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.p1); + println!("\t depth.intrinsics.parameters.param.metric_radius: {}", self.0.depth_camera_calibration.intrinsics.parameters.param.metric_radius); + } + println!("\t color.resolution_width: {}", self.0.color_camera_calibration.resolution_width); + println!("\t color.resolution_height: {}", self.0.color_camera_calibration.resolution_height); + println!("\t color.metric_radius: {}", self.0.color_camera_calibration.metric_radius); + println!("\t color.extrinsics: {:?}", self.0.color_camera_calibration.extrinsics); + println!("\t color.intrinsics.type: {}", self.0.color_camera_calibration.intrinsics.type_); + println!("\t color.intrinsics.parameter_count: {}", self.0.color_camera_calibration.intrinsics.parameter_count); + unsafe { + // NB: This is a union field, so we have to use unsafe access + println!("\t color.intrinsics.parameters.param.cx: {}", self.0.color_camera_calibration.intrinsics.parameters.param.cx); + println!("\t color.intrinsics.parameters.param.cy: {}", self.0.color_camera_calibration.intrinsics.parameters.param.cy); + println!("\t color.intrinsics.parameters.param.fx: {}", self.0.color_camera_calibration.intrinsics.parameters.param.fx); + println!("\t color.intrinsics.parameters.param.fy: {}", self.0.color_camera_calibration.intrinsics.parameters.param.fy); + println!("\t color.intrinsics.parameters.param.k1: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k1); + println!("\t color.intrinsics.parameters.param.k2: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k2); + println!("\t color.intrinsics.parameters.param.k3: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k3); + println!("\t color.intrinsics.parameters.param.k4: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k4); + println!("\t color.intrinsics.parameters.param.k5: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k5); + println!("\t color.intrinsics.parameters.param.k6: {}", self.0.color_camera_calibration.intrinsics.parameters.param.k6); + println!("\t color.intrinsics.parameters.param.codx: {}", self.0.color_camera_calibration.intrinsics.parameters.param.codx); + println!("\t color.intrinsics.parameters.param.cody: {}", self.0.color_camera_calibration.intrinsics.parameters.param.cody); + println!("\t color.intrinsics.parameters.param.p2: {}", self.0.color_camera_calibration.intrinsics.parameters.param.p2); + println!("\t color.intrinsics.parameters.param.p1: {}", self.0.color_camera_calibration.intrinsics.parameters.param.p1); + println!("\t color.intrinsics.parameters.param.metric_radius: {}", self.0.color_camera_calibration.intrinsics.parameters.param.metric_radius); + } + println!("=========="); + } +} + +/// Adapted from k4a-sys. Represents a capture. +#[derive(Debug)] +pub struct Capture(pub k4a_sys::k4a_capture_t); + +impl Capture { + pub fn get_depth_image(&self) -> Result { + let image = unsafe { + k4a_sys::k4a_capture_get_depth_image(self.0) + }; + if image.is_null() { + return Err(CaptureError::NullCapture); + } + Ok(Image(image)) + } + + pub fn get_color_image(&self) -> Result { + let image = unsafe { + k4a_sys::k4a_capture_get_color_image(self.0) + }; + if image.is_null() { + return Err(CaptureError::NullCapture); + } + Ok(Image(image)) + } + + pub fn get_ir_image(&self) -> Result { + let image = unsafe { + k4a_sys::k4a_capture_get_ir_image(self.0) + }; + if image.is_null() { + return Err(CaptureError::NullCapture); + } + Ok(Image(image)) + } + + /// Returns the underlying opaque handle *without* an additional refcount. + /// Do not deallocate it. + pub fn get_handle(&self) -> k4a_sys::k4a_capture_t { + self.0 + } +} + +/// Remove a libk4a capture refcount on every drop. +/// When the refcount drops to zero, the capture goes away. +impl Drop for Capture { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_capture_release(self.0); + self.0 = null_mut(); + } + } +} + +/// Handles are refcounted by libk4a. The final reference is destroyed +impl Clone for Capture { + fn clone(&self) -> Self { + // We must increment the refcount. + let handle = self.get_handle(); + unsafe { + k4a_sys::k4a_capture_reference(handle); + } + Self { + 0: handle, + } + } + + fn clone_from(&mut self, source: &Self) { + let handle = source.get_handle(); + unsafe { + k4a_sys::k4a_capture_reference(handle); + } + unsafe { + // drop refcount, potentially releasing if reached zero + k4a_sys::k4a_capture_release(self.0); + } + self.0 = handle; + } +} + +/// FIXME FIXME FIXME: WHY IS THIS NOT SEND WITH ARC>!? +/// We are making k4a_sys::k4a_capture_t Send only when wrapped with Arc>, +/// but the compiler can't figure that out. Freaky gross. +/// +/// "Rust automatically determines whether a type is Send and/or Sync. Anything that has a +/// raw ptr inside, which is what you have here, is considered !Send and !Sync. This isn’t because +/// it automatically means the type is unsafe in Send/Sync terms, but rather more like a lint for +/// the code author: they need to determine the safety themselves, and then if they’re safe, +/// manually impl Send and/or Sync, as appropriate, for the type." +/// +/// Actually Rust is doing a good job here. +/// +/// https://users.rust-lang.org/t/solved-how-to-move-non-send-between-threads-or-an-alternative/19928/11 +unsafe impl Send for Capture{} + +/// TODO: We also need to send device +unsafe impl Send for Device{} + +/// Adapted from k4a-sys. Represents an image within a capture. +#[derive(Debug)] +pub struct Image(pub k4a_sys::k4a_image_t); + +impl Image { + + /// Create a blank image. + /// + /// This function is used to create images of formats that have consistent stride. The function + /// is not suitable for compressed formats that may not be represented by the same number of bytes + /// per line. + /// + /// For most image formats, the function will allocate an image buffer of size + /// height_pixels * stride_bytes. Buffers K4A_IMAGE_FORMAT_COLOR_NV12 format will allocate an + /// additional height_pixels / 2 set of lines (each of stride_bytes). This function cannot be used + /// to allocate K4A_IMAGE_FORMAT_COLOR_MJPG buffers. + /// + /// To create an image object without the API allocating memory, or to represent an image that has + /// a non-deterministic stride, use k4a_image_create_from_buffer(). + /// + /// The k4a_image_t is created with a reference count of 1. + /// + /// When finished using the created image, release it with k4a_image_release. + /// + /// stride_bytes - The number of bytes per horizontal line of the image. If set to 0, the stride + /// will be set to the minimum size given the format and width_pixels. + /// + pub fn create(format: ImageFormat, + width: u32, + height: u32, + stride_bytes: u32) + -> Result + { + let mut handle = null_mut(); + + let result = unsafe { + k4a_sys::k4a_image_create( + format as k4a_sys::k4a_image_format_t, + width as i32, + height as i32, + stride_bytes as i32, + &mut handle + ) + }; + + if result != k4a_sys::k4a_result_t_K4A_RESULT_SUCCEEDED { + return Err(KinectError::UnableToCreateImage { error_code: result }); + } + + Ok(Image(handle)) + } + + pub fn get_height_pixels(&self) -> usize { + unsafe { + k4a_sys::k4a_image_get_height_pixels(self.0) as usize + } + } + + pub fn get_width_pixels(&self) -> usize { + unsafe { + k4a_sys::k4a_image_get_width_pixels(self.0) as usize + } + } + + pub fn get_stride_bytes(&self) -> usize { + unsafe { + k4a_sys::k4a_image_get_stride_bytes(self.0) as usize + } + } + + pub fn get_size(&self) -> usize { + unsafe { + k4a_sys::k4a_image_get_size(self.0) // returns size_t + } + } + + pub fn get_buffer(&self) -> *mut u8 { + unsafe { + k4a_sys::k4a_image_get_buffer(self.0) + } + } + + /// Use this function to determine the format of the image buffer. + /// This function is not expected to fail, all k4a_image_t's are created with a + /// known format. If the image_handle is invalid, the function will return + /// K4A_IMAGE_FORMAT_CUSTOM. + pub fn get_format(&self) -> ImageFormat { + let format = unsafe { + k4a_sys::k4a_image_get_format(self.0) + }; + format.into() + } + + /// Returns the underlying opaque handle *without* an additional refcount. + /// Do not deallocate it. + pub fn get_handle(&self) -> k4a_sys::k4a_image_t { + self.0 + } +} + +/// Remove a libk4a image refcount on every drop. +/// When the refcount drops to zero, the image goes away. +impl Drop for Image { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_image_release(self.0); + } + } +} + +/// Handles are refcounted by libk4a. The final reference is destroyed +impl Clone for Image { + fn clone(&self) -> Self { + // We must increment the refcount. + let handle = self.get_handle(); + unsafe { + k4a_sys::k4a_image_reference(handle); + } + Self { + 0: handle, + } + } + + fn clone_from(&mut self, source: &Self) { + let handle = source.get_handle(); + unsafe { + k4a_sys::k4a_image_reference(handle); + } + unsafe { + // drop refcount, potentially releasing if reached zero + k4a_sys::k4a_image_release(self.0); + } + self.0 = handle; + } +} + +#[derive(Debug,Clone,Copy)] +pub enum ImageFormat { + ColorMjpg, + ColorNv12, + ColorYuy2, + ColorBgra32, + /// Depth image type DEPTH16. + /// Each pixel of DEPTH16 data is two bytes of little endian unsigned depth data. + /// The unit of the data is in millimeters from the origin of the camera. + /// Stride indicates the length of each line in bytes and should be used to determine + /// the start location of each line of the image in memory. + Depth16, + Ir16, + Custom8, + Custom16, + Custom, + UnknownFormatError, // FIXME: Just return Result? +} + +impl ImageFormat { + pub fn to_k4a(&self) -> k4a_sys::k4a_image_format_t { + match self { + ImageFormat::ColorMjpg => 0, + ImageFormat::ColorNv12 => 1, + ImageFormat::ColorYuy2 => 2, + ImageFormat::ColorBgra32 => 3, + ImageFormat::Depth16 => 4, + ImageFormat::Ir16 => 5, + ImageFormat::Custom8 => 6, + ImageFormat::Custom16 => 7, + ImageFormat::Custom => 8, + ImageFormat::UnknownFormatError => 255, // TODO: Better handling? + } + } +} + +impl From for ImageFormat { + fn from(format: k4a_sys::k4a_image_format_t) -> Self { + match format { + 0 => ImageFormat::ColorMjpg, + 1 => ImageFormat::ColorNv12, + 2 => ImageFormat::ColorYuy2, + 3 => ImageFormat::ColorBgra32, + 4 => ImageFormat::Depth16, + 5 => ImageFormat::Ir16, + 6 => ImageFormat::Custom8, + 7 => ImageFormat::Custom16, + 8 => ImageFormat::Custom, + _ => ImageFormat::UnknownFormatError, + } + } +} + +/// Deallocate open device handles +impl Drop for Device { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_device_close(self.device_pointer); + } + } +} + +/// Copied from k4a-sys +pub struct DeviceConfiguration (k4a_sys::k4a_device_configuration_t); + +/// Copied from k4a-sys +impl DeviceConfiguration { + pub fn new() -> Self { + Self (k4a_sys::k4a_device_configuration_t { + color_format: k4a_sys::k4a_image_format_t_K4A_IMAGE_FORMAT_COLOR_MJPG, + color_resolution: k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_720P, + depth_mode: k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_2X2BINNED, + camera_fps: 0, + synchronized_images_only: false, + depth_delay_off_color_usec: 0, + wired_sync_mode: 0, + subordinate_delay_off_master_usec: 0, + disable_streaming_indicator: false, + }) + } +} + +#[derive(Clone,Debug)] +pub struct Resolution { + pub width: i32, + pub height: i32, +} + +#[derive(Clone,Debug)] +pub struct Transformation { + transformation: k4a_sys::k4a_transformation_t, + pub color_resolution: Resolution, + pub depth_resolution: Resolution, +} + +impl Transformation { + /// Creates a transformation associated with a calibration + pub fn from_calibration(calibration: &Calibration) -> Self { + let transformation = unsafe { + k4a_sys::k4a_transformation_create(&calibration.0) + }; + Self { + transformation, + color_resolution: Resolution { + width: calibration.0.color_camera_calibration.resolution_width, + height: calibration.0.color_camera_calibration.resolution_height, + }, + depth_resolution: Resolution { + width: calibration.0.depth_camera_calibration.resolution_width, + height: calibration.0.depth_camera_calibration.resolution_height, + }, + } + } + + /// Returns the underlying opaque handle *without* an additional refcount. + /// Do not deallocate it. + pub fn get_handle(&self) -> k4a_sys::k4a_transformation_t { + self.transformation + } +} + +impl Drop for Transformation { + fn drop(&mut self) { + unsafe { + k4a_sys::k4a_transformation_destroy(self.transformation); + } + } +} diff --git a/src/azure_kinect/mod.rs b/src/azure_kinect/mod.rs new file mode 100644 index 000000000..132ce37ab --- /dev/null +++ b/src/azure_kinect/mod.rs @@ -0,0 +1,2 @@ +pub mod capture; +pub mod sensor_control; \ No newline at end of file diff --git a/src/azure_kinect/sensor_control.rs b/src/azure_kinect/sensor_control.rs new file mode 100644 index 000000000..1d41ecf08 --- /dev/null +++ b/src/azure_kinect/sensor_control.rs @@ -0,0 +1,71 @@ +use crate::image_debug::depth_to_image; +use image::DynamicImage; +use kinect::{Capture, Device}; +use std::sync::Arc; +use std::sync::Mutex; + +/** Self-locking holder of Capture objects. */ +pub struct CaptureProvider { + capture: Arc>>, +} + +impl CaptureProvider { + pub fn new() -> Self { + CaptureProvider { + capture: Arc::new(Mutex::new(None)), + } + } + + /** + * Take the latest capture, if available, through interior mutability. + * This leaves the mutex holding None. + */ + pub fn get_capture(&self) -> Option { + self.capture.lock() + .ok() + .and_then(|mut lock| lock.take()) + } + + /** + * Consume capture and replace whatever we currently hold. + */ + pub fn set_capture(&self, capture: Capture) { + match self.capture.lock() { + Ok(mut lock) => { + *lock = Some(capture) + }, + Err(_) => {}, + } + } +} + +pub fn capture_thread(capture_provider: Arc, device: Option, start_camera: bool) { + let start_camera = device.is_none() || start_camera; + + let device = match device { + Some(device) => device, + None => { + let installed_devices = Device::get_installed_count(); + println!("Installed devices: {}", installed_devices); + + let device = Device::open(0).unwrap(); + println!("Device: {:?}", device); + device + }, + }; + + let serial_number = device.get_serial_number().unwrap(); + println!("Device: {:?}", serial_number); + + if start_camera { + println!("Starting cameras..."); + device.start_cameras_default_config().unwrap(); + } + + loop { + let capture = device.get_capture(1000) + .expect("Should be able to get frame capture."); + + capture_provider.set_capture(capture); + } +} diff --git a/src/core_types.rs b/src/core_types.rs new file mode 100644 index 000000000..e3ad1a556 --- /dev/null +++ b/src/core_types.rs @@ -0,0 +1,62 @@ + +/// A core color type +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RgbaF32 { + pub red: f32, + pub green: f32, + pub blue: f32, + pub alpha: f32, +} + +impl RgbaF32 { + /// Black (alpha 1.0) + pub fn black() -> Self { + Self { + red: 0.0, + green: 0.0, + blue: 0.0, + alpha: 1.0, + } + } + + pub fn rgb(r: f32, g: f32, b: f32) -> Self { + Self { + red: r, + green: g, + blue: b, + alpha: 1.0, + } + } + + pub fn gray(gray: f32) -> Self { + Self { + red: gray, + green: gray, + blue: gray, + alpha: 1.0, + } + } +} + +#[cfg(test)] +mod test { + use crate::core_types::RgbaF32; + + #[test] + pub fn test_rgbaf32() { + let black = RgbaF32::black(); + assert_eq!(black.red, 0.0f32); + assert_eq!(black.green, 0.0f32); + assert_eq!(black.blue, 0.0f32); + assert_eq!(black.alpha, 1.0f32); + } + + #[test] + pub fn test_gray() { + let gray = RgbaF32::gray(0.5); + assert_eq!(gray.red, 0.5f32); + assert_eq!(gray.green, 0.5f32); + assert_eq!(gray.blue, 0.5f32); + assert_eq!(gray.alpha, 1.0f32); + } +} diff --git a/src/files/mod.rs b/src/files/mod.rs new file mode 100644 index 000000000..1fd10f23e --- /dev/null +++ b/src/files/mod.rs @@ -0,0 +1,2 @@ +pub mod read_file_string_contents; +pub mod write_to_file_from_byte_ptr; diff --git a/src/files/read_file_string_contents.rs b/src/files/read_file_string_contents.rs new file mode 100644 index 000000000..a70c3e1b6 --- /dev/null +++ b/src/files/read_file_string_contents.rs @@ -0,0 +1,10 @@ +use anyhow::Result as AnyhowResult; +use std::fs::File; +use std::io::Read; + +pub fn read_file_string_contents(filename: &str) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + Ok(contents) +} diff --git a/src/files/write_to_file_from_byte_ptr.rs b/src/files/write_to_file_from_byte_ptr.rs new file mode 100644 index 000000000..eaa99f523 --- /dev/null +++ b/src/files/write_to_file_from_byte_ptr.rs @@ -0,0 +1,23 @@ +use anyhow::Result as AnyhowResult; +use std::fs::OpenOptions; +use std::io::Write; +use std::ptr; + +/// Raw memory copy into a file +pub fn write_to_file_from_byte_ptr(filename: &str, byte_src: *const u8, size_bytes: usize) -> AnyhowResult<()> { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open(filename)?; + + let mut vec = vec![0u8; size_bytes]; + + unsafe { + let vec_ptr = vec.as_mut_ptr(); + //ptr::write_bytes(vec_ptr, color_src, size_bytes); + ptr::copy_nonoverlapping(byte_src, vec_ptr, size_bytes); + } + + file.write(&vec)?; + Ok(()) +} diff --git a/src/graphics_imgui.rs b/src/graphics_imgui.rs new file mode 100644 index 000000000..82595b835 --- /dev/null +++ b/src/graphics_imgui.rs @@ -0,0 +1,276 @@ +use anyhow::Result as AnyhowResult; +use kinect::Calibration; +use crate::ProgramArgs; +use crate::core_types::RgbaF32; +use crate::gui::enhanced_window::EnhancedWindow; +use crate::gui::mouse_camera_arcball::MouseCameraArcball; +use crate::azure_kinect::capture::device_capturer::CaptureProvider; +use crate::point_cloud::pixel_structs::BgraPixel; +use crate::point_cloud::point_cloud_visualiser::{ColorizationStrategy, PointCloudVisualizer, PointCloudVisualizerError}; +use crate::point_cloud::viewer_image::ViewerImage; +use crate::webcam::WebcamWriter; +use gl; +use imgui::*; +use imgui::Image; +use std::path::Path; +use std::sync::{Arc, Mutex, PoisonError, MutexGuard}; +use std::time::{Instant, Duration}; + +pub fn run(capture_provider: Arc, program_args: ProgramArgs) -> AnyhowResult<()> { + let mut webcam_writer = None; + + if program_args.enable_webcam_output_writing { + webcam_writer = Some(WebcamWriter::open_file("/dev/video0", 1280, 720, 3) + .expect("should be able to create webcamwriter")); + } + + let sdl_arcball = Arc::new(Mutex::new(MouseCameraArcball::new(1280, 720))); + + let sdl_context = sdl2::init().unwrap(); + let video = sdl_context.video().unwrap(); + + { + let gl_attr = video.gl_attr(); + gl_attr.set_context_profile(sdl2::video::GLProfile::Core); + gl_attr.set_context_version(3, 0); + } + + let window = video.window("rust-imgui-sdl2 demo", 3000, 2000) + .position_centered() + .resizable() + .opengl() + .allow_highdpi() + .build() + .unwrap(); + + let _gl_context = window.gl_create_context().expect("Couldn't create GL context"); + gl::load_with(|s| video.gl_get_proc_address(s) as _); + + //enable_opengl_debugging(); + + let mut imgui = imgui::Context::create(); + imgui.set_ini_filename(None); + imgui.io_mut().config_windows_move_from_title_bar_only = true; + + /*let gl_texture_n64 = load_texture("n64logo.png"); + let imgui_texture_n64 = TextureId::from(gl_texture_n64 as usize); + let mut gl_texture_snes = load_texture("sneslogo.png"); + let mut imgui_texture_snes = TextureId::from(gl_texture_snes as usize);*/ + + let colorization_strategy = ColorizationStrategy::Color; + + let clear_color = RgbaF32::gray(0.5); + + let mut visualizer = PointCloudVisualizer::new( + capture_provider.get_num_cameras(), + capture_provider.get_calibrations(), + true, + colorization_strategy, + clear_color, + sdl_arcball.clone(), + ); + + //rebinder.restore(); + + let texture = ViewerImage::create( + // TODO: The following commented-out values work and look great, + // but now we're doing suboptimal webcam stuff: + //1280, + //1152, + 1280, + 720, + ).expect("ViewerImage texture creation should work"); + + let mut imgui_visualizer_xyz_texture_0 : Option = None; + let mut imgui_visualizer_xyz_texture_1 : Option = None; + + let imgui_point_cloud_convert_xy_table= TextureId::from(visualizer.point_cloud_converters.get(0).unwrap().xy_table_texture.id() as usize); + + let imgui_kinect_final_output = TextureId::from(texture.texture_id() as usize); + + let mut imgui_sdl2 = imgui_sdl2::ImguiSdl2::new(&mut imgui, &window); + + let renderer = imgui_opengl_renderer::Renderer::new(&mut imgui, |s| video.gl_get_proc_address(s) as _); + + let mut event_pump = sdl_context.event_pump().unwrap(); + + let mut last_frame = Instant::now(); + let mut last_change = Instant::now(); + + let mut camera_index = 0; + + visualizer.setup_rendering()?; + + 'running: loop { + use sdl2::event::Event; + use sdl2::keyboard::Keycode; + + for event in event_pump.poll_iter() { + imgui_sdl2.handle_event(&mut imgui, &event); + + // TODO: This is how Imgui appears to tell SDL to ignore certain events. + if imgui_sdl2.ignore_event(&event) { continue; } + + match event { + Event::Quit {..} | Event::KeyDown { keycode: Some(Keycode::Escape), .. } => { + break 'running + }, + _ => {} + } + + /*// Also send to the arcball to update the view matrix. + match sdl_arcball.lock() { + Ok(mut arcball) => arcball.process_event(&event), + Err(_) => {}, + }*/ + } + + imgui_sdl2.prepare_frame(imgui.io_mut(), &window, &event_pump.mouse_state()); + + let now = Instant::now(); + let delta = now - last_frame; + let delta_s = delta.as_secs() as f32 + delta.subsec_nanos() as f32 / 1_000_000_000.0; + last_frame = now; + imgui.io_mut().delta_time = delta_s; + + let ui = imgui.frame(); + + //ui.show_demo_window(&mut true); + + const window_width : f32 = 1290.0; + const window_height : f32 = 760.0; + + if imgui_visualizer_xyz_texture_0.is_none() { + // TODO: TEMPORARY MESS TO SUPPORT MULTI-CAMERA + if visualizer.xyz_textures.get(0).unwrap().id() != 0 { + imgui_visualizer_xyz_texture_0 = Some(TextureId::from(visualizer.xyz_textures.get(0).unwrap().id() as usize)); + } + } + + if imgui_visualizer_xyz_texture_1.is_none() { + // TODO: TEMPORARY MESS TO SUPPORT MULTI-CAMERA + if visualizer.xyz_textures.get(1).unwrap().id() != 0 { + imgui_visualizer_xyz_texture_1 = Some(TextureId::from(visualizer.xyz_textures.get(1).unwrap().id() as usize)); + } + } + + Window::new(im_str!("Visualizer XYZ Texture (0)")) + .scrollable(true) + .size([window_width, window_height], Condition::FirstUseEver) + .position([0.0, 0.0], Condition::FirstUseEver) + .build(&ui, || { + match imgui_visualizer_xyz_texture_0.as_ref() { + None => {}, + Some(xyz_texture) => { + Image::new(xyz_texture.clone(), [1280.0, 720.0]).build(&ui); + }, + } + }); + + Window::new(im_str!("Visualizer XYZ Texture (1)")) + .size([window_width, window_height], Condition::FirstUseEver) + .position([window_width + 50.0, 0.0], Condition::FirstUseEver) + .movable(false) + .build(&ui, || { + match imgui_visualizer_xyz_texture_1.as_ref() { + None => {}, + Some(xyz_texture) => { + Image::new(xyz_texture.clone(), [1280.0, 720.0]).build(&ui); + }, + } + }); + + Window::new(im_str!("Point Cloud Converter XY Table")) + .size([window_width, window_height], Condition::FirstUseEver) + .position([0.0, window_height + 50.0], Condition::FirstUseEver) + .build(&ui, || { + Image::new(imgui_point_cloud_convert_xy_table, [1280.0, 720.0]).build(&ui); + }); + + // NB: This is larger. The texture is 1280x1152. + EnhancedWindow::new(&im_str!("Final Output: {:?}", colorization_strategy)) + .size([window_width, window_height + 435.0], Condition::FirstUseEver) + .position([window_width + 50.0, window_height + 50.0], Condition::FirstUseEver) + .mouse_inputs(true) + .build(&ui, |window| { + + if let Some(mouse_state) = window.get_window_bounded_mouse_state() { + match sdl_arcball.lock() { + Ok(mut arcball) => arcball.process_mouse_state(mouse_state), + Err(_) => {}, + } + } + + Image::new(imgui_kinect_final_output, [1280.0, 720.0]).build(&ui); + }); + + unsafe { + gl::ClearColor(0.2, 0.2, 0.2, 1.0); + gl::Clear(gl::COLOR_BUFFER_BIT); + } + + imgui_sdl2.prepare_render(&ui, &window); + renderer.render(ui); + + window.gl_swap_window(); + + if let Some(mut captures) = capture_provider.get_captures() { + if program_args.save_captures { + // Only save the last frame (for now) + for (i, capture) in captures.iter().enumerate() { + let suffix = format!("{}", i); + capture.debug_save_files(Path::new("output/"), &suffix).unwrap(); + } + } + + visualizer.update_texture_id(texture.texture_id(), captures) + .map(|_| {}) + .map_err(|err| { + match err { + PointCloudVisualizerError::MissingDepthImage => { println!("Missing depth image"); }, + PointCloudVisualizerError::MissingColorImage => { println!("Missing color image"); }, + _ => { + unreachable!("Error: {:?}", err); + } + } + }); + } + + match sdl_arcball.lock() { + Ok(mut arcball) => { + arcball.animate(); + }, + Err(_) => {}, + } + + if let Some(mut webcam) = webcam_writer.as_mut() { + webcam.write_current_frame_to_file(texture.texture_id()) + .expect("should write"); + } + + let change_delta = last_frame - last_change; + if change_delta > Duration::from_millis(5_000) { + camera_index = (camera_index + 1) % capture_provider.get_num_cameras(); + last_change = Instant::now(); + } + + /*if change_delta > Duration::from_millis(5000) { + colorization_strategy = match colorization_strategy { + ColorizationStrategy::Simple => ColorizationStrategy::Shaded, // NB: We won't do simple. + ColorizationStrategy::Shaded => ColorizationStrategy::Color, + ColorizationStrategy::Color => ColorizationStrategy::Shaded, + }; + + println!("Changing colorization strategy: {:?}", colorization_strategy); + + visualizer.set_colorization_strategy(colorization_strategy); + last_change = Instant::now(); + }*/ + + //rebinder.restore(); + + std::thread::sleep(::std::time::Duration::new(0, 1_000_000_000u32 / 60)); + } + + Ok(()) +} diff --git a/src/gui/enhanced_window.rs b/src/gui/enhanced_window.rs new file mode 100644 index 000000000..a3127ffbb --- /dev/null +++ b/src/gui/enhanced_window.rs @@ -0,0 +1,168 @@ +/// A wrapper around `Imgui::Window` that adds window-scoped devices. +pub struct EnhancedWindow <'a> { + pub window: imgui::Window<'a>, +} + +impl <'a> EnhancedWindow <'a> { + pub fn new(name: &'a imgui::ImStr) -> Self { + let mut window = imgui::Window::new(name) + .scrollable(true); + + Self { + window, + } + } + + /// Enables/disables catching mouse input. + /// + /// Enabled by default. + /// Note: Hovering test will pass through when disabled + #[inline] + pub fn mouse_inputs(mut self, value: bool) -> Self { + // NB: Builder API consumes window, so we have to move it. + self.window = self.window.mouse_inputs(value); + self + } + + /// Sets the window position, which is applied based on the given condition value + #[inline] + pub fn position(mut self, position: [f32; 2], condition: imgui::Condition) -> Self { + // NB: Builder API consumes window, so we have to move it. + self.window = self.window.position(position, condition); + self + } + + /// Sets the window size, which is applied based on the given condition value + #[inline] + pub fn size(mut self, size: [f32; 2], condition: imgui::Condition) -> Self { + // NB: Builder API consumes window, so we have to move it. + self.window = self.window.size(size, condition); + self + } + + /// Creates a window and runs a closure to construct the contents. + /// + /// Note: the closure is not called if no window content is visible (e.g. window is collapsed + /// or fully clipped). + pub fn build(self, ui: &imgui::Ui, f: F) + where F: FnOnce(WindowScopedUi) + { + if let Some(window) = self.begin(ui) { + let win_helper = WindowScopedUi::new(&ui); + f(win_helper); + window.end(ui); + } + } + + /// Creates a window and starts appending to it. + /// + /// Returns `Some(WindowToken)` if the window is visible. After content has been + /// rendered, the token must be ended by calling `.end()`. + /// + /// Returns `None` if the window is not visible and no content should be rendered. + #[must_use] + pub fn begin(self, ui: &imgui::Ui) -> Option { + self.window.begin(ui) + } +} + +/// State for a mouse mapped to the dimensions of an `Imgui::Window`. +#[derive(Debug, Clone)] +pub struct CompleteMouseState { + // Cursor position + pub cursor_x: f32, + pub cursor_y: f32, + // Scroll wheel velocity (typically just '0', '1', or '-1'). + pub scroll_wheel: f32, + pub scroll_wheel_h: f32, + // Buttons + pub left_button_down: bool, + pub right_button_down: bool, + pub middle_button_down: bool, + pub extra_button_1_down: bool, + pub extra_button_2_down: bool, +} + +/// Exposes some reusable calculations on top of `imgui::Ui` in the context of window usage. +pub struct WindowScopedUi<'a> { + ui: &'a imgui::Ui<'a>, +} + +impl <'a> WindowScopedUi<'a> { + pub fn new(ui: &'a imgui::Ui) -> Self { + Self { + ui + } + } + + /// Returns true if the current window is hovered + #[inline] + pub fn is_window_hovered(&self) -> bool { + self.ui.is_window_hovered() + } + + /// Return the cursor position within the window. + /// None is returned if the window isn't hovered, or the mouse is logically not within the drawn + /// texture area. + pub fn get_cursor(&self) -> Option<[f32; 2]> { + if !self.is_window_hovered() { + return None; + } + + // Global mouse coordinate + let mouse_pos = self.ui.io().mouse_pos; + // Anchor point of the window's top left corner + let window_pos = self.ui.window_pos(); + // Start of textured data (padding offset created by title and window padding). + let cursor_pos = self.ui.cursor_pos(); + + let x = mouse_pos[0] - window_pos[0] - cursor_pos[0]; + let y = mouse_pos[1] - window_pos[1] - cursor_pos[1]; + + if x.is_sign_negative() || y.is_sign_negative() { + return None; // Outside of bounds (eg. cursor within title area) + } + + Some([x, y]) + } + + /// Return the mouse scroll state. If the mouse is not hovering our window, None is returned. + pub fn get_mouse_wheel(&self) -> Option { + if self.get_cursor().is_none() { + return None; + } + Some(self.ui.io().mouse_wheel) + } + + /// Mouse buttons: 0=left, 1=right, 2=middle + extras + pub fn get_mouse_down(&self) -> Option<[bool; 5]> { + if self.get_cursor().is_none() { + return None; + } + + Some(self.ui.io().mouse_down) + } + + /// Get the current mouse state + /// If the cursor isn't within the window, None is returned. + pub fn get_window_bounded_mouse_state(&self) -> Option { + self.get_cursor() + .map(|cursor| { + let io = self.ui.io(); + let mouse = io.mouse_down; + + CompleteMouseState { + cursor_x: cursor[0], + cursor_y: cursor[1], + scroll_wheel: io.mouse_wheel, + scroll_wheel_h: io.mouse_wheel_h, + left_button_down: mouse[0], + right_button_down: mouse[1], + middle_button_down: mouse[2], + extra_button_1_down: mouse[3], + extra_button_2_down: mouse[4], + } + }) + } +} + diff --git a/src/gui/mod.rs b/src/gui/mod.rs new file mode 100644 index 000000000..4efbe5020 --- /dev/null +++ b/src/gui/mod.rs @@ -0,0 +1,3 @@ +//! This module will contain all the Imgui, SDL, mouse controls, etc. code +pub mod enhanced_window; +pub mod mouse_camera_arcball; \ No newline at end of file diff --git a/src/gui/mouse_camera_arcball.rs b/src/gui/mouse_camera_arcball.rs new file mode 100644 index 000000000..866c9dc78 --- /dev/null +++ b/src/gui/mouse_camera_arcball.rs @@ -0,0 +1,214 @@ +use arcball::ArcballCamera; +use cgmath::Vector3; +use cgmath::{Matrix4, Vector2}; +use crate::gui::enhanced_window::CompleteMouseState; +use sdl2::event::Event; +use sdl2::mouse::MouseButton; + +const MIN_ZOOM : f32 = 1.0; +const MAX_ZOOM : f32 = 120.0; +const DEFAULT_ZOOM: f32 = 65.0; + +// TODO: Enable camera resizing (or change of texture to differently sized texture) +/// Wrap a camera arcball for use with SDL +pub struct MouseCameraArcball { + // Arcball camera state + arcball: ArcballCamera, + + // Dimensions of the window + window_width: u32, + window_height: u32, + + // Last mouse state + left_mouse_active: bool, + right_mouse_active: bool, + last_x: f32, + last_y: f32, + last_zoom: f32, + + animation_counter: f32, +} + +impl MouseCameraArcball { + /// CTOR + pub fn new(window_width: u32, window_height: u32) -> Self { + let center = Vector3::new(0.0, 0.0, 0.0); + let zoom_speed = 1.0; + let screen = [window_width as f32, window_height as f32]; + let mut arcball = ArcballCamera::new(center, zoom_speed, screen); + + // TODO: This should be cleaned up to use pure math rather than stupid experimentally determined fixes + arcball.zoom(-8.0, 0.16); + + let mut prev = Vector2::new(window_width as f32, (window_height/2) as f32); + + for i in 0 .. window_width / 2 { + let pos = Vector2::new(prev.x - 1.0, (window_height/2) as f32); + arcball.rotate(prev, pos); + prev = pos.clone(); + } + + for i in 0..600 { + let delta = Vector2::new(0.0, 1.0); + arcball.pan(delta, 0.16); + } + + Self { + arcball, + window_width, + window_height, + left_mouse_active: false, + right_mouse_active: false, + last_x: 0.0, + last_y: 0.0, + last_zoom: DEFAULT_ZOOM, + + animation_counter: 0.0, + } + } + + /// Process an SDL event. + /// This is a bad idea in a multi-window Imgui setup since the SDL coordinates and events take up + /// the entire process window and may be sequestered on a per-window level. That said, this is + /// reasonable for single-window apps. + pub fn process_event(&mut self, event: &Event) { + match event { + Event::MouseWheel { timestamp: _, window_id: _, which: _, x: _, y, direction: _ } => { + //println!("Mouse wheel event."); + let y = *y as f32; + self.mouse_wheel(y); + }, + Event::MouseButtonDown { timestamp: _, window_id: _, which: _, mouse_btn, clicks: _, x: _, y: _ } => { + //println!("Mouse button down event."); + self.mouse_button_press(mouse_btn, true); + }, + Event::MouseButtonUp { timestamp: _, window_id: _, which: _, mouse_btn, clicks: _, x: _, y: _ } => { + //println!("Mouse button up event."); + self.mouse_button_press(mouse_btn, false); + }, + Event::MouseMotion { timestamp: _, window_id: _, which: _, mousestate: _, x, y, xrel: _, yrel: _ } => { + let x = *x as f32; + let y = *y as f32; + self.mouse_motion(x, y); + self.last_x = x; + self.last_y = y; + }, + _ => {}, + } + } + + /// Process Imgui window-scoped mouse state. + /// Since we don't process individual events, we set all of the internal state at once. + pub fn process_mouse_state(&mut self, mouse_state: CompleteMouseState) { + // TODO: Cleanup + self.right_mouse_active = mouse_state.right_button_down; + self.left_mouse_active = mouse_state.left_button_down; + self.mouse_motion(mouse_state.cursor_x, mouse_state.cursor_y); + self.mouse_wheel(mouse_state.scroll_wheel); + self.last_x = mouse_state.cursor_x; + self.last_y = mouse_state.cursor_y; + } + + /// Get the view matrix + pub fn get_view_matrix(&self) -> Matrix4 { + self.arcball.get_mat4() + } + + /// Get the perspective matrix + pub fn get_perspective_matrix(&self) -> Matrix4 { + let zoom = self.last_zoom; + let y_fov = zoom.to_radians(); + let aspect = self.window_width as f32 / self.window_height as f32; + let n = 0.1f32; + let f = 1000.0f32; + + // From linmath.h included in Microsoft's k4a open source code + let a = 1.0f32 / (y_fov / 2.0).tan(); + + // NB: This is column-major, which is dumb. + Matrix4::new( + // Column 0 + a / aspect, + 0.0, + 0.0, + 0.0, + + // Column 1 + 0.0, + a, + 0.0, + 0.0, + + // Column 2 + 0.0, + 0.0, + -((f + n) / (f - n)), + -((2.0 * f * n) / (f - n)), + + // Column 3 + 0.0, + 0.0, + -1.0, // NB: This is not zero + 0.0, + ) + } + + pub fn animate(&mut self) { + let x = self.animation_counter.sin(); + let y = self.animation_counter.cos(); + self.animation_counter.sin(); + let mouse_delta = Vector2::new(x * 4.0, y * 4.0); + self.arcball.pan(mouse_delta, 0.16); + + + let x = self.animation_counter.sin(); + + if x > 0.0 { + let y = 1.0; + let prev = Vector2::new(0.0, 0.0); + let cur = Vector2::new(y, y); + self.arcball.rotate(prev, cur); + } else { + let y = 1.0; + let prev = Vector2::new(y, y); + let cur = Vector2::new(0.0, 0.0); + self.arcball.rotate(prev, cur); + }; + + + self.animation_counter += 0.05; + } + + fn mouse_wheel(&mut self, y: f32) { + self.arcball.zoom(y, 0.16); + + // NB: 'y' is a scroll amount, typically 1 or -1. + let zoom = (self.last_zoom + y) + .max(MIN_ZOOM) + .min(MAX_ZOOM); + + self.last_zoom = zoom; + } + + fn mouse_button_press(&mut self, mouse_button: &MouseButton, down: bool) { + match mouse_button { + MouseButton::Left => self.left_mouse_active = down, + MouseButton::Right => self.right_mouse_active = down, + _ => {}, + } + } + + fn mouse_motion(&mut self, x: f32, y: f32) { + if self.left_mouse_active { + let previous = Vector2::new(self.last_x, self.last_y); + let current = Vector2::new(x, y); + self.arcball.rotate(previous, current); + } else if self.right_mouse_active { + let mouse_delta = Vector2::new( + x - self.last_x, + self.last_y - y, // Invert scrolling + ); + self.arcball.pan(mouse_delta, 0.16); + } + } +} diff --git a/src/image_debug.rs b/src/image_debug.rs new file mode 100644 index 000000000..cdde3b5e0 --- /dev/null +++ b/src/image_debug.rs @@ -0,0 +1,159 @@ +//! Format conversion +//! k4a frames, image-rs, opencv, OpenGL textures, etc. + +use crate::point_cloud::pixel_structs::{BgraPixel, DepthPixel}; +use glium::texture::RawImage2d; +use image::DynamicImage; +use image::GenericImage; +use image::ImageBuffer; +use image::ImageError; +use image::Rgba; +use image::RgbaImage; +use image::error::{ImageFormatHint, UnsupportedError}; +use image::flat::{FlatSamples, SampleLayout}; +use k4a_sys_temp as k4a_sys; +use kinect::Image; +use kinect::ImageFormat; +use std::mem::size_of; +use std::slice; + +// https://docs.rs/image/0.23.2/image/flat/index.html +pub fn depth_to_image(image: &Image) -> Result { + let len = image.get_size(); + let samples = unsafe { slice::from_raw_parts(image.get_buffer(), len) }; + + let layout = SampleLayout::row_major_packed( + 4, + image.get_width_pixels() as u32, + image.get_height_pixels() as u32); + + let buffer = FlatSamples { + samples, + layout, + color_hint: None, + }; + + let view = buffer.as_view::>() + .expect("view should work"); + + let img: RgbaImage = ImageBuffer::new( + image.get_width_pixels() as u32, + image.get_height_pixels() as u32); + let mut img = DynamicImage::ImageRgba8(img); + + img.copy_from(&view, 0, 0).expect("Should be able to copy"); + + Ok(img) +} + +// Not efficient, but good for debugging +pub fn k4a_image_to_rust_image_for_debug(image: &Image) -> Result { + let width = image.get_width_pixels() as u32; + let height= image.get_height_pixels() as u32; + let image_buffer = image.get_buffer(); + let image_format = image.get_format(); + + println!("k4a_image_to_rust: {}x{} (format={:?}", width, height, image_format); + + let output_image = DynamicImage::new_rgba16(width, height); + let mut rgba_image = output_image.to_rgba(); + let mut offset = 0; + + match image_format { + ImageFormat::Depth16 => { + let typed_buffer = image_buffer as *const DepthPixel; + for y in 0 .. height { + for x in 0 .. width { + let pixel = unsafe { *typed_buffer.offset(offset) }; + let scaled_pixel = pixel as f32 / std::u16::MAX as f32; + let scaled_pixel = (scaled_pixel * std::u8::MAX as f32) as u8; + rgba_image.put_pixel(x, y, Rgba([scaled_pixel, scaled_pixel, scaled_pixel, 255])); + offset += 1; + } + } + }, + ImageFormat::ColorBgra32 => { + let typed_buffer = image_buffer as *const BgraPixel; + for y in 0 .. height { + for x in 0 .. width { + let pixel = unsafe { &*typed_buffer.offset(offset) }; + // BGRA32: The fourth byte is the alpha channel and is unused in the Azure Kinect APIs. + rgba_image.put_pixel(x, y, Rgba([pixel.red, pixel.green, pixel.blue, 255])); + offset += 1; + } + } + }, + ImageFormat::Custom => { + // Stride is image_width * bytes_per_pixel, so we solve for bytes_per_pixel + let stride = image.get_stride_bytes(); + let image_width = image.get_width_pixels(); + let bytes_per_pixel = stride / image_width; + + // The only format we know how to represent is k4a_float2_t. + let known_size = size_of::(); + + if bytes_per_pixel != known_size { + return Err(ImageError::Unsupported(UnsupportedError::from(ImageFormatHint::Unknown))) + } + + let typed_buffer = image_buffer as *const k4a_sys::k4a_float2_t; + + let mut max_x = -10000.0; + let mut max_y = -10000.0; + let mut min_x= 10000.0; + let mut min_y = 10000.0; + + offset = 0; + for _ in 0 .. height { + for _ in 0 .. width { + unsafe { + let pixel = &*typed_buffer.offset(offset); + let xp = pixel.xy.x; + let yp = pixel.xy.y; + if xp < min_x { + min_x = xp; + } + if xp > max_x { + max_x = xp; + } + if yp < min_y { + min_y = yp; + } + if yp > max_y { + max_y = yp; + } + } + offset += 1; + } + } + + //println!("Max x: {}", max_x); + //println!("Max y: {}", max_y); + //println!("Min x: {}", min_x); + //println!("Min y: {}", min_y); + + offset = 0; + for y in 0 .. height { + for x in 0 .. width { + unsafe { + let pixel = &*typed_buffer.offset(offset); + let xp = pixel.xy.x; + let yp = pixel.xy.y; + + let scaled_x = (xp - min_x) / (max_x - min_x); + let scaled_x = (255.0 * scaled_x) as u8; + + let scaled_y = (yp - min_y) / (max_y - min_y); + let scaled_y = (255.0 * scaled_y) as u8; + + rgba_image.put_pixel(x, y, Rgba([scaled_x, 0, scaled_y, 255])); + } + offset += 1; + } + } + }, + _ => unimplemented!("conversion not implemented for: {:?}", image_format), + }; + + return Ok(rgba_image) +} diff --git a/src/imgui_support/clip.rs b/src/imgui_support/clip.rs new file mode 100644 index 000000000..5f1cc3430 --- /dev/null +++ b/src/imgui_support/clip.rs @@ -0,0 +1,19 @@ +use clipboard::{ClipboardContext, ClipboardProvider}; +use imgui::{ClipboardBackend, ImStr, ImString}; + +pub struct ClipboardSupport(ClipboardContext); + +pub fn init() -> Option { + ClipboardContext::new() + .ok() + .map(|ctx| ClipboardSupport(ctx)) +} + +impl ClipboardBackend for ClipboardSupport { + fn get(&mut self) -> Option { + self.0.get_contents().ok().map(|text| text.into()) + } + fn set(&mut self, text: &ImStr) { + let _ = self.0.set_contents(text.to_str().to_owned()); + } +} diff --git a/src/imgui_support/mod.rs b/src/imgui_support/mod.rs new file mode 100644 index 000000000..d6079de66 --- /dev/null +++ b/src/imgui_support/mod.rs @@ -0,0 +1,139 @@ +use std::time::Instant; + +use glium::{Display, Surface}; +use glium::glutin; +use glium::glutin::event::{Event, WindowEvent}; +use glium::glutin::event_loop::{ControlFlow, EventLoop}; +use glium::glutin::window::WindowBuilder; +use imgui::{Context, FontConfig, FontSource, Ui}; +use imgui_glium_renderer::Renderer; +use imgui_winit_support::{HiDpiMode, WinitPlatform}; + +mod clip; + +pub struct System { + pub event_loop: EventLoop<()>, + pub display: glium::Display, + pub imgui: Context, + pub platform: WinitPlatform, + pub renderer: Renderer, + pub font_size: f32, +} + +pub fn init(title: &str) -> System { + let title = match title.rfind('/') { + Some(idx) => title.split_at(idx + 1).1, + None => title, + }; + let event_loop = EventLoop::new(); + let context = glutin::ContextBuilder::new().with_vsync(true); + let builder = WindowBuilder::new() + .with_title(title.to_owned()) + .with_inner_size(glutin::dpi::LogicalSize::new(1024f64, 768f64)); + let display = + Display::new(builder, context, &event_loop).expect("Failed to initialize display"); + + let mut imgui = Context::create(); + imgui.set_ini_filename(None); + + if let Some(backend) = clip::init() { + imgui.set_clipboard_backend(Box::new(backend)); + } else { + eprintln!("Failed to initialize clipboard"); + } + + let mut platform = WinitPlatform::init(&mut imgui); + { + let gl_window = display.gl_window(); + let window = gl_window.window(); + platform.attach_window(imgui.io_mut(), &window, HiDpiMode::Rounded); + } + + // Load the OpenGL function pointers + gl::load_with(|symbol| display.gl_window().get_proc_address(symbol)); + + let hidpi_factor = platform.hidpi_factor(); + let font_size = (13.0 * hidpi_factor) as f32; + imgui.fonts().add_font(&[ + FontSource::DefaultFontData { + config: Some(FontConfig { + size_pixels: font_size, + ..FontConfig::default() + }), + }, + /*FontSource::TtfData { + data: include_bytes!("../../../resources/mplus-1p-regular.ttf"), + size_pixels: font_size, + config: Some(FontConfig { + rasterizer_multiply: 1.75, + glyph_ranges: FontGlyphRanges::japanese(), + ..FontConfig::default() + }), + },*/ + ]); + + imgui.io_mut().font_global_scale = (1.0 / hidpi_factor) as f32; + + let renderer = Renderer::init(&mut imgui, &display).expect("Failed to initialize renderer"); + + System { + event_loop, + display, + imgui, + platform, + renderer, + font_size, + } +} + +impl System { + pub fn main_loop(self, mut run_ui: F) { + let System { + event_loop, + display, + mut imgui, + mut platform, + mut renderer, + .. + } = self; + let mut last_frame = Instant::now(); + + event_loop.run(move |event, _, control_flow| match event { + Event::NewEvents(_) => last_frame = imgui.io_mut().update_delta_time(last_frame), + Event::MainEventsCleared => { + let gl_window = display.gl_window(); + platform + .prepare_frame(imgui.io_mut(), &gl_window.window()) + .expect("Failed to prepare frame"); + gl_window.window().request_redraw(); + } + Event::RedrawRequested(_) => { + let mut ui = imgui.frame(); + + let mut run = true; + run_ui(&mut run, &mut ui); + if !run { + *control_flow = ControlFlow::Exit; + } + + let gl_window = display.gl_window(); + let mut target = display.draw(); + target.clear_color_srgb(1.0, 1.0, 1.0, 1.0); + platform.prepare_render(&ui, gl_window.window()); + let draw_data = ui.render(); + renderer + .render(&mut target, draw_data) + .expect("Rendering failed"); + target.finish().expect("Failed to swap buffers"); + } + Event::WindowEvent { + event: WindowEvent::CloseRequested, + .. + } => *control_flow = ControlFlow::Exit, + event => { + let gl_window = display.gl_window(); + platform.handle_event(imgui.io_mut(), gl_window.window(), &event); + } + }) + } +} diff --git a/src/main_opencv.rs b/src/main_opencv.rs new file mode 100644 index 000000000..877a3e6a5 --- /dev/null +++ b/src/main_opencv.rs @@ -0,0 +1,132 @@ +#![allow(warnings)] + +#[macro_use] extern crate clap; +#[macro_use] extern crate glium; +#[macro_use] extern crate imgui; +#[macro_use] extern crate log; + +extern crate anyhow; +extern crate arcball; +extern crate cgmath; +extern crate clipboard; +extern crate gl; +extern crate glfw; +extern crate glfw_sys; +extern crate image; +extern crate imgui_glium_renderer; +extern crate imgui_opengl_renderer; +extern crate imgui_sdl2; +extern crate imgui_winit_support; +extern crate libc; +extern crate rand; +extern crate sdl2; +extern crate tobj; +extern crate winit; +extern crate gltf; + +use std::sync::Arc; +use std::{thread, env}; + +use anyhow::Result as AnyhowResult; +use anyhow::bail; +use kinect::{Device, Calibration}; +use azure_kinect::sensor_control::capture_thread; +use azure_kinect::capture::multi_device_capturer::{MultiDeviceCapturer, start_capture_thread}; +use azure_kinect::capture::fake_device_capturer::FakeDeviceCaptureProvider; +use azure_kinect::capture::device_capturer::CaptureProvider; +use clap::{Clap}; + +pub mod assets; +pub mod azure_kinect; +pub mod core_types; +pub mod files; +pub mod graphics_imgui; +pub mod gui; +pub mod image_debug; +pub mod imgui_support; +pub mod opengl; +pub mod point_cloud; +pub mod webcam; + +const ENV_RUST_LOG : &'static str = "RUST_LOG"; +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // NB: We don't use actix, but this is preserved for syntax ref + +/// This is the serial number for the camera I have mounted to the utility wall. +const PRIMARY_DEVICE_SERIAL : &'static str = "000513594512"; + +/// This is a second device +const SECONDARY_DEVICE_SERIAL : &'static str = "000886694512"; + +#[derive(Clap, Debug)] +struct Opts { + #[clap(long, parse(try_from_str = true_or_false), default_value = "true")] + pub enable_cameras: bool, + #[clap(long, parse(try_from_str = true_or_false), default_value = "false")] + pub enable_webcam: bool, + #[clap(long, parse(try_from_str = true_or_false), default_value = "false")] + pub save_captures: bool, +} + +fn true_or_false(s: &str) -> Result { + match s { + "true" => Ok(true), + "false" => Ok(false), + _ => Err("expected `true` or `false`"), + } +} + +/// Arguments to pass through the program +pub struct ProgramArgs { + pub save_captures: bool, + pub enable_webcam_output_writing: bool, +} + +pub fn main() -> AnyhowResult<()> { + if env::var(ENV_RUST_LOG) + .as_ref() + .ok() + .is_none() + { + println!("Setting default logging level to \"{}\", override with env var {}.", + DEFAULT_RUST_LOG, ENV_RUST_LOG); + env::set_var(ENV_RUST_LOG, DEFAULT_RUST_LOG); + } + + env_logger::init(); + + let opts = Opts::parse(); + + println!("opts: {:?}", opts); + + // Validate args. + if opts.save_captures && !opts.enable_cameras { + bail!("Can't save captures without cameras enabled."); + } + + let program_args = ProgramArgs { + enable_webcam_output_writing: opts.enable_webcam, + save_captures: opts.save_captures, + }; + + let capture_provider: Arc = if opts.enable_cameras { + let multi_device = MultiDeviceCapturer::new(2, Some(1)) + .expect("multi-device create"); + + multi_device.start_cameras().expect("start cameras"); + + let capture_provider = multi_device.get_sync_capture_provider(); + + thread::spawn(move || start_capture_thread(multi_device)); + + capture_provider + } else { + Arc::new(FakeDeviceCaptureProvider::new().unwrap()) + }; + + //let calibration = capture_provider.get_calibration().clone(); + //calibration.debug_print(); + + graphics_imgui::run(capture_provider, program_args)?; + + Ok(()) +} diff --git a/src/main_points.rs b/src/main_points.rs new file mode 100644 index 000000000..64dfd74c7 --- /dev/null +++ b/src/main_points.rs @@ -0,0 +1,39 @@ + +pub mod zeromq; +pub use anyhow::Result as AnyhowResult; +use anyhow::anyhow; +use crate::zeromq::xy_table::create_xy_table; +use k4a_sys_temp as k4a_sys; +use kinect::{Device, DeviceConfiguration}; +use std::thread; +use std::time::Duration; +use crate::zeromq::cpu_calculate_point_cloud::calculate_point_cloud; + +fn main() -> AnyhowResult<()> { + let device = Device::open(0)?; + + let mut config = DeviceConfiguration::init_disable_all(); + config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_2X2BINNED; + config.0.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_30; + + let calibration = device.get_calibration(config.0.depth_mode, config.0.color_resolution)?; + + let xy_table = create_xy_table(&calibration)?; + + device.start_cameras(&config)?; + + loop { + println!("Get capture"); + + let capture = device.get_capture(500)?; + + let depth_image = capture.get_depth_image() + .ok_or(anyhow!("capture not present"))?; + + let point_cloud = calculate_point_cloud(&depth_image, &xy_table)?; + + println!("Number points: {}", point_cloud.point_count); + } + + Ok(()) +} diff --git a/src/main_zeromq.rs b/src/main_zeromq.rs new file mode 100644 index 000000000..898fcd0e7 --- /dev/null +++ b/src/main_zeromq.rs @@ -0,0 +1,283 @@ +//! Generate random points to send over ZeroMQ to a consumer + +#[allow(unused_imports)] + +mod zeromq; + +use anyhow::Result as AnyhowResult; +use anyhow::anyhow; +use byteorder::{WriteBytesExt, LittleEndian, BigEndian}; +use k4a_sys_temp as k4a_sys; +use kinect::{Device, DeviceConfiguration, Image}; +use std::io::Write; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use zeromq::cpu_calculate_point_cloud::PointCloudResult; +use zeromq::cpu_calculate_point_cloud::calculate_point_cloud2; +use zeromq::cpu_calculate_point_cloud::calculate_point_cloud3; +use zeromq::color::Color; +use zeromq::cpu_transformation::DepthTransformer; +use zeromq::point::Point; +use zeromq::xy_table::create_xy_table; +use zeromq::xy_table::create_xy_table_from_depth_calibration; +use zeromq::xy_table::create_xy_table_from_color_calibration; +use zmq::{Error, Socket, Context, DONTWAIT}; + +const SOCKET_ADDRESS : &'static str = "tcp://127.0.0.1:8888"; + +const DATA_LENGTH_COMMAND : u32 = 1; // Denotes the command that sends the data length +const POINT_DATA_BEGIN_PAYLOAD_COMMAND : u32 = 2; // Denotes the command that sends the variable-length data +const POINT_DATA_CONTINUE_PAYLOAD_COMMAND : u32 = 3; // Denotes the command that sends the variable-length data + +/// The maximum number of points to send per ZeroMQ "packet". +/// This is also defined in C++, so it needs to be adjusted in multiple places. +const MAX_SEND_POINTS_PER_PACKET : usize = 3000; + +enum MessagingState { + Sending_DataLength, + //Receiving_DataLengthAck, + Sending_PointDataBegin, + //Receiving_PointDataAck, + Sending_PointDataContinue, + GrabPointCloud, +} + +fn main() -> AnyhowResult<()> { + let device = Device::open(0)?; + + let mut config = DeviceConfiguration::init_disable_all(); + config.0.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_15; + //config.0.camera_fps = k4a_sys::k4a_fps_t_K4A_FRAMES_PER_SECOND_30; + + // TODO: Pick correct binning + FOV. + + // NB: Similar to NFOV_UNBINNED, but less dense (faster). + // Not used by any program I'm aware of. + //config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_2X2BINNED; // 320x228 + + // NB: NFOV_UNBINNED was used by original Rust experiment. + // This appears to be much denser, and is a tighter angle. Slow on CPU. + //config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_UNBINNED; // looks good w/ 720P + + // NB: WFOV_2X2BINNED was used by the original 'cloudcam_zeromq'. + // It's less dense, and wider. Much more performant. Fast on CPU. + //config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_2X2BINNED; // DISTORTED w/ 720P + + // Not used by anything, AFAICT. + // I got this working in Rust + Color Camera + config.0.depth_mode = k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_UNBINNED; // 1024x1024 (looks good w/ 720P) + + + config.0.color_format = k4a_sys::k4a_image_format_t_K4A_IMAGE_FORMAT_COLOR_BGRA32; + config.0.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_720P; // 1280x721 + //config.0.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_1080P; // 1920x1080 (this gets truncated.) + //config.0.color_resolution = k4a_sys::k4a_color_resolution_t_K4A_COLOR_RESOLUTION_2160P; // 4K, what the original program did + + let calibration = device.get_calibration(config.0.depth_mode, config.0.color_resolution)?; + + //let xy_table = create_xy_table_from_depth_calibration(&calibration)?; + let xy_table = create_xy_table_from_color_calibration(&calibration)?; + + let transformer = DepthTransformer::new(&calibration); + + device.start_cameras(&config)?; + + // =============================================================== + + let context = zmq::Context::new(); + + let mut socket = context.socket(zmq::PUSH).unwrap(); + //let mut socket = context.socket(zmq::REQ).unwrap(); + + //socket.bind(SOCKET_ADDRESS).unwrap(); + socket.connect(SOCKET_ADDRESS).unwrap(); + + let mut messaging_state = MessagingState::Sending_DataLength; + let mut color = Color::get_time_based_color(); + + let mut points : Vec = Vec::new(); + + loop { + let maybe_points = get_point_cloud(&device, &xy_table, color, &transformer); + points = match maybe_points { + Err(e) => { + println!("Error: {:?}", e); + thread::sleep(Duration::from_millis(5000)); + continue; + }, + Ok(points) => points, + }; + break; + }; + + if !points.is_empty() { + //print_pointcloud_maxima(&points); + } else { + println!("> No points?"); + } + + let mut packet_number = 0; + + loop { + match messaging_state { + MessagingState::Sending_DataLength => { + //println!("Sending DATA LENGTH: {}", points.len()); + //let message = encode_data_length(&points); + //send_message(&socket, &message)?; + messaging_state = MessagingState::Sending_PointDataBegin; + }, + MessagingState::Sending_PointDataBegin => { + //println!("Begin Sending PCD (points: {})", points.len()); + let message = encode_point_data(&mut points, true); + send_message(&socket, &message)?; + + if points.is_empty() { + messaging_state = MessagingState::GrabPointCloud; + } else { + messaging_state = MessagingState::Sending_PointDataContinue; + } + + }, + MessagingState::Sending_PointDataContinue => { + // NB: Unfortunately the C++ program has difficulty with all points. + if packet_number > 2000000 { + println!("Packet elapsed"); + packet_number = 0; + messaging_state = MessagingState::GrabPointCloud; + continue; + } + + //println!("Continue Sending PCD (points: {})", points.len()); + let message = encode_point_data(&mut points, false); + send_message(&socket, &message)?; + + if points.is_empty() { + messaging_state = MessagingState::GrabPointCloud; + } else { + messaging_state = MessagingState::Sending_PointDataContinue; + } + + packet_number += 1; + }, + MessagingState::GrabPointCloud => { + //println!("Grabbing another frame..."); + color = Color::get_time_based_color(); + points = get_point_cloud(&device, &xy_table, color, &transformer)?; + + if !points.is_empty() { + //print_pointcloud_maxima(&points); + } else { + println!("No points?"); + } + messaging_state = MessagingState::Sending_DataLength; + continue; + }, + } + } + + Ok(()) +} + +fn get_point_cloud(device: &Device, xy_table: &Image, color: Color, transformer: &DepthTransformer) + -> AnyhowResult> +{ + let capture = device.get_capture(5000)?; + + let depth_image = capture.get_depth_image() + .ok_or(anyhow!("depth image not present"))?; + + let color_image = capture.get_color_image() + .ok_or(anyhow!("color image not present"))?; + + + let depth_image2 = transformer.transform(&depth_image)?; + + let mut points = + calculate_point_cloud3( + &depth_image2, + &xy_table, + //color, + &color_image + )?; + + //println!("Points: {}", points.len()); + + Ok(points) +} + +/// Returns the fixed size data length command. +fn encode_data_length(points: &Vec) -> Vec { + let mut buf = Vec::with_capacity(8); + buf.write_u32::(DATA_LENGTH_COMMAND); + buf.write_u32::(points.len() as u32); + buf +} + +/// Returns a variable length point data payload. +fn encode_point_data(points: &mut Vec, is_beginning: bool) -> Vec { + let point_bytes = Point::size_bytes() * MAX_SEND_POINTS_PER_PACKET; + + let mut buf = Vec::with_capacity(4 + point_bytes); + + if is_beginning { + //println!("begin"); + buf.write_u32::(POINT_DATA_BEGIN_PAYLOAD_COMMAND); // COMMAND # + } else { + //println!("continue"); + buf.write_u32::(POINT_DATA_CONTINUE_PAYLOAD_COMMAND); // COMMAND # + } + + let subset = if points.len() > MAX_SEND_POINTS_PER_PACKET { + points.drain(0..MAX_SEND_POINTS_PER_PACKET).collect::>() + } else { + points.drain(0..points.len()).collect::>() + }; + + buf.write_u32::(subset.len() as u32); // LENGTH + + for point in subset { + let bytes = point.to_bytes(); + buf.write_all(&bytes); + } + + buf +} + +/// Send data over the socket +fn send_message(socket: &Socket, data_bytes: &Vec) -> AnyhowResult<()> { + socket.send(&data_bytes, 0)?; + Ok(()) +} + +/// Receive ack from server +fn receive_ack(socket: &Socket) -> AnyhowResult<()> { + let _result = socket.recv_bytes(DONTWAIT)?; + Ok(()) +} + +fn reconnect_socket(context: &Context, socket: Socket, address: &str) -> Socket { + //println!("[reconnect] Creating new socket..."); + + let mut socket = match context.socket(zmq::REQ) { + Ok(s) => { + //println!("New socket created."); + s + }, + Err(e) => { + //println!("Error creating new socket: {:?}", e); + return socket; + }, + }; + + //println!("Connecting new socket..."); + match socket.connect(address) { + Ok(_) => { + //println!("New socket connected."); + }, + Err(err) => { + //println!("Error connecting new socket: {:?}", err); + }, + } + + return socket; +} diff --git a/src/opengl/compile_shader.rs b/src/opengl/compile_shader.rs new file mode 100644 index 000000000..be98c79ef --- /dev/null +++ b/src/opengl/compile_shader.rs @@ -0,0 +1,43 @@ +use std::ffi::CString; +use std::ptr; +use std::str; + +use gl; +use gl::types::*; + +pub fn compile_shader(src: &str, shader_type: GLenum) -> GLuint { + let shader; + unsafe { + shader = gl::CreateShader(shader_type); + // Attempt to compile the shader + let c_str = CString::new(src.as_bytes()).unwrap(); + gl::ShaderSource(shader, 1, &c_str.as_ptr(), ptr::null()); + gl::CompileShader(shader); + + // Get the compile status + let mut status = gl::FALSE as GLint; + gl::GetShaderiv(shader, gl::COMPILE_STATUS, &mut status); + + // Fail on error + if status != (gl::TRUE as GLint) { + let mut len = 0; + gl::GetShaderiv(shader, gl::INFO_LOG_LENGTH, &mut len); + let mut buf = Vec::with_capacity(len as usize); + buf.set_len((len as usize) - 1); // subtract 1 to skip the trailing null character + gl::GetShaderInfoLog( + shader, + len, + ptr::null_mut(), + buf.as_mut_ptr() as *mut GLchar, + ); + panic!( + "{}", + str::from_utf8(&buf) + .ok() + .expect("ShaderInfoLog not valid utf8") + ); + } + } + shader +} + diff --git a/src/opengl/debug.rs b/src/opengl/debug.rs new file mode 100644 index 000000000..a254b23db --- /dev/null +++ b/src/opengl/debug.rs @@ -0,0 +1,34 @@ +//! Add an OpenGL debugging callback. + +use std::ffi::CStr; +use std::os::raw::c_void; +use std::ptr::null; + +use gl; +use gl::types::*; + +extern "system" fn glDebugOutput( + _source: GLenum, + _gltype: GLenum, + id: GLuint, + _severity: GLenum, + _length: GLsizei, + message: *const GLchar, + _userParam: *mut c_void) +{ + let message = unsafe { CStr::from_ptr(message) }; + //println!(">>> [GL DEBUG] id: {:?} message: {:?} source: {:?} type: {:?} severity: {:?}", + // id, message, source, gltype, severity); + println!(">>> [GL DEBUG] id: {:?} message: {:?}", id, message); +} + +pub fn enable_opengl_debugging() { + unsafe { + // Set debugging. This can be expensive, so we should be able to trigger it with a flag. + gl::Enable(gl::DEBUG_OUTPUT); + gl::Enable(gl::DEBUG_OUTPUT_SYNCHRONOUS); + gl::DebugMessageCallback(Some(glDebugOutput), null()); + // Opt into everything. + gl::DebugMessageControl(gl::DONT_CARE, gl::DONT_CARE, gl::DONT_CARE, 0, null(), gl::TRUE); + } +} diff --git a/src/opengl/link_program.rs b/src/opengl/link_program.rs new file mode 100644 index 000000000..4b01e194e --- /dev/null +++ b/src/opengl/link_program.rs @@ -0,0 +1,35 @@ +use gl::types::*; +use std::ptr; +use std::str; + +pub fn link_shader_program(program: GLuint, vertex_shader: GLuint, fragment_shader: GLuint) -> GLuint { + unsafe { + gl::AttachShader(program, vertex_shader); + gl::AttachShader(program, fragment_shader); + gl::LinkProgram(program); + // Get the link status + let mut status = gl::FALSE as GLint; + gl::GetProgramiv(program, gl::LINK_STATUS, &mut status); + + // Fail on error + if status != (gl::TRUE as GLint) { + let mut len: GLint = 0; + gl::GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len); + let mut buf = Vec::with_capacity(len as usize); + buf.set_len((len as usize) - 1); // subtract 1 to skip the trailing null character + gl::GetProgramInfoLog( + program, + len, + ptr::null_mut(), + buf.as_mut_ptr() as *mut GLchar, + ); + panic!( + "{}", + str::from_utf8(&buf) + .ok() + .expect("ProgramInfoLog not valid utf8") + ); + } + program + } +} diff --git a/src/opengl/load_texture_from_file.rs b/src/opengl/load_texture_from_file.rs new file mode 100644 index 000000000..4555285e5 --- /dev/null +++ b/src/opengl/load_texture_from_file.rs @@ -0,0 +1,53 @@ +use gl::types::*; +use gl; +use image; +use std::ffi::c_void; +use std::path::Path; + +/// Load a texture from a file. +pub fn load_texture_from_file(filename: &str) -> GLuint { + let mut texture_id = 0; + + unsafe { + gl::GenTextures(1, &mut texture_id); + gl::BindTexture(gl::TEXTURE_2D, texture_id); + + // Set up filtering params and other texture attributes + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::LINEAR as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::LINEAR as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_S, gl::REPEAT as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_T, gl::REPEAT as i32); + } + + let img = image::open(&Path::new(filename)) + .expect("failed to load") + .to_rgba(); + + let width = img.dimensions().0 as i32; + let height = img.dimensions().1 as i32; + let data = img.into_raw(); + + println!("Loaded image: {}x{}", width, height); + + unsafe { + gl::PixelStorei(gl::UNPACK_ROW_LENGTH, 0); + gl::TexImage2D( + gl::TEXTURE_2D, + 0, + gl::RGBA as i32, + //format as i32, + width, + height, + 0, + gl::RGBA, + //format, + gl::UNSIGNED_BYTE, + &data[0] as *const u8 as *const c_void, + ); + + //gl::BindTexture(gl::TEXTURE_2D, 0); + } + + println!("Returning texture id: {}", texture_id); + return texture_id; +} diff --git a/src/opengl/matrices.rs b/src/opengl/matrices.rs new file mode 100644 index 000000000..15ffe47e2 --- /dev/null +++ b/src/opengl/matrices.rs @@ -0,0 +1,77 @@ +//! Useful matrix functions + +pub const fn translation_matrix_4x4_flat(x: f32, y: f32, z: f32) -> [f32; 16] { + [ + 1.0, 0.0, 0.0, x, + 0.0, 1.0, 0.0, y, + 0.0, 0.0, 1.0, z, + 0.0, 0.0, 0.0, 1.0, + ] +} + +pub const fn identity_matrix_4x4() -> [[f32; 4]; 4] { + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] +} + +pub const fn identity_matrix_4x4_flat() -> [f32; 16] { + [ + 1.0, 0.0, 0.0, 0.0, + 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0, + ] +} + +pub const fn zero_matrix_4x4_flat() -> [f32; 16] { + return [ + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, + ]; +} + +/// This is the view matrix k4aviewer starts with +pub const fn initial_view_matrix_4x4() -> [[f32; 4]; 4] { + [ + [-1.0, 0.0, 8.74228e-08, 0.0], + [0.0, 1.0, 0.0, 0.0], + [-8.74228e-08, 0.0, -1.0, 0.0], + [2.62268e-07, 1.0, -5.0, 1.0], + ] +} + +/// This is the view matrix k4aviewer starts with +pub const fn initial_view_matrix_4x4_flat() -> [f32; 16] { + [ + -1.0, 0.0, 8.74228e-08, 0.0, + 0.0, 1.0, 0.0, 0.0, + -8.74228e-08, 0.0, -1.0, 0.0, + 2.62268e-07, 1.0, -5.0, 1.0, + ] +} + +/// This is the projection matrix k4aviewer starts with +pub const fn initial_projection_matrix_4x4() -> [[f32; 4]; 4] { + [ + [1.41272, 0.0, 0.0, 0.0], + [0.0, 1.56969, 0.0, 0.0], + [0.0, 0.0, -1.002, -1.0], + [0.0, 0.0, -0.2002, 0.0], + ] +} + +/// This is the projection matrix k4aviewer starts with +pub const fn initial_projection_matrix_4x4_flat() -> [f32; 16] { + [ + 1.41272, 0.0, 0.0, 0.0, + 0.0, 1.56969, 0.0, 0.0, + 0.0, 0.0, -1.002, -1.0, + 0.0, 0.0, -0.2002, 0.0, + ] +} diff --git a/src/opengl/mod.rs b/src/opengl/mod.rs new file mode 100644 index 000000000..5fc670c31 --- /dev/null +++ b/src/opengl/mod.rs @@ -0,0 +1,7 @@ +pub mod compile_shader; +pub mod debug; +pub mod link_program; +pub mod load_texture_from_file; +pub mod matrices; +pub mod rebinder; +pub mod wrapper; \ No newline at end of file diff --git a/src/opengl/rebinder.rs b/src/opengl/rebinder.rs new file mode 100644 index 000000000..ad68bcb41 --- /dev/null +++ b/src/opengl/rebinder.rs @@ -0,0 +1,75 @@ +use gl; +use gl::types::*; + +/// Rebinder records the current state of OpenGL bindings so that it can restore them later. +/// I'm writing this because the code I wrote/adapted is unfortunately a mess. +pub struct Rebinder { + texture_id: GLint, + array_buffer_id: GLint, + element_array_buffer_id: GLint, + pixel_unpack_buffer_id: GLint, + renderbuffer_binding: GLint, + vertex_array_binding: GLint, + draw_framebuffer_binding: GLint, + read_framebuffer_binding: GLint, + current_program: GLint, + active_texture: GLint, + is_depth_test: u8, +} + +impl Rebinder { + /// Capture a current snapshot of the bindings. + pub fn snapshot() -> Self { + let mut bindings = Self { + texture_id: 0, + array_buffer_id: 0, + element_array_buffer_id: 0, + pixel_unpack_buffer_id: 0, + renderbuffer_binding: 0, + vertex_array_binding: 0, + draw_framebuffer_binding: 0, + read_framebuffer_binding: 0, + current_program: 0, + active_texture: 0, + is_depth_test: 0, + }; + + unsafe { + gl::GetIntegerv(gl::TEXTURE_BINDING_2D, &mut bindings.texture_id); + gl::GetIntegerv(gl::ARRAY_BUFFER_BINDING, &mut bindings.array_buffer_id); + gl::GetIntegerv(gl::ELEMENT_ARRAY_BUFFER_BINDING, &mut bindings.element_array_buffer_id); + gl::GetIntegerv(gl::PIXEL_UNPACK_BUFFER_BINDING, &mut bindings.pixel_unpack_buffer_id); + gl::GetIntegerv(gl::RENDERBUFFER_BINDING, &mut bindings.renderbuffer_binding); + gl::GetIntegerv(gl::VERTEX_ARRAY_BINDING, &mut bindings.vertex_array_binding); + gl::GetIntegerv(gl::DRAW_FRAMEBUFFER_BINDING, &mut bindings.draw_framebuffer_binding); + gl::GetIntegerv(gl::READ_FRAMEBUFFER_BINDING, &mut bindings.read_framebuffer_binding); + gl::GetIntegerv(gl::CURRENT_PROGRAM, &mut bindings.current_program); + gl::GetIntegerv(gl::ACTIVE_TEXTURE, &mut bindings.active_texture); + gl::GetBooleanv(gl::DEPTH_TEST, &mut bindings.is_depth_test); + } + + bindings + } + + /// Restore the snapshot of the bindings. + /// This consumes the rebinder. + pub fn restore(self) { + unsafe { + gl::BindTexture(gl::TEXTURE_2D, self.texture_id as u32); + gl::BindBuffer(gl::ARRAY_BUFFER, self.array_buffer_id as u32); + gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.element_array_buffer_id as u32); + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, self.pixel_unpack_buffer_id as u32); + gl::BindRenderbuffer(gl::RENDERBUFFER, self.renderbuffer_binding as u32); + gl::BindVertexArray(self.vertex_array_binding as u32); + gl::BindFramebuffer(gl::DRAW_FRAMEBUFFER, self.draw_framebuffer_binding as u32); + gl::BindFramebuffer(gl::READ_FRAMEBUFFER, self.read_framebuffer_binding as u32); + gl::UseProgram(self.current_program as u32); + gl::ActiveTexture(self.active_texture as u32); + if self.is_depth_test == 0 { + gl::Disable(gl::DEPTH_TEST); + } else { + gl::Enable(gl::DEPTH_TEST); + } + } + } +} \ No newline at end of file diff --git a/src/opengl/wrapper/attribute.rs b/src/opengl/wrapper/attribute.rs new file mode 100644 index 000000000..bfe917723 --- /dev/null +++ b/src/opengl/wrapper/attribute.rs @@ -0,0 +1,49 @@ +use anyhow::Result as AnyhowResult; +use anyhow::bail; +use gl::types::*; +use std::os::raw::c_char; +use std::ffi::CString; + +/// A typesafe handle on attribute IDs with convenience methods. +/// +/// Unlike "Uniforms", which are global in GLSL, "Attributes" are applied on a per-vertex basis. +/// They can be used to convey typical information (eg. normals) or completely custom information +/// (eg. tangents or whatever). +/// +/// https://www.opengl.org/sdk/docs/tutorials/ClockworkCoders/attributes.php +/// +#[derive(Clone, Debug, Copy)] +pub struct Attribute { + id: GLint, +} + +impl Attribute { + pub fn lookup(identifier: &str, opengl_program_id: GLuint) -> AnyhowResult { + let id_cstr : CString = CString::new(identifier)?; + let id_cstr_ptr : *const c_char = id_cstr.as_ptr() as *const c_char; + + let mut loc = -1; + unsafe { + loc = gl::GetAttribLocation(opengl_program_id, id_cstr_ptr); + } + + if loc < 0 { + bail!("Couldn't find attribute named '{}' in OpenGL program {}.", identifier, opengl_program_id); + } + + Ok(Self { + id: loc, + }) + } + + pub fn id(&self) -> GLint { + self.id + } + + /*pub fn set_uint(&self, value: GLuint) { + let loc = self.id() as GLuint; + unsafe { + gl::VertexAttribI1ui(loc, value); + } + }*/ +} diff --git a/src/opengl/wrapper/buffer.rs b/src/opengl/wrapper/buffer.rs new file mode 100644 index 000000000..bb43cc280 --- /dev/null +++ b/src/opengl/wrapper/buffer.rs @@ -0,0 +1,64 @@ +//! These wrappers make it easier to use certain OpenGL types. +//! This draws inspiration from some of the code in Microsoft's MIT-licensed libk4a, +//! particularly the code in `openglhelpers.h`. + +use gl::types::*; + +/// A slightly more convenient OpenGL Buffer type +#[derive(Clone, Debug)] +pub struct Buffer { + id: GLuint, +} + +impl Buffer { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + pub fn bind_as_array_buffer(&self) { + unsafe { + gl::BindBuffer(gl::ARRAY_BUFFER, self.id); + } + } + + pub fn gen(&mut self) { + unsafe { + gl::GenBuffers(1, &mut self.id); + } + } + + pub fn delete(&mut self) { + unsafe { + gl::DeleteBuffers(1, &mut self.id); + } + } +} + diff --git a/src/opengl/wrapper/mod.rs b/src/opengl/wrapper/mod.rs new file mode 100644 index 000000000..02e1feaed --- /dev/null +++ b/src/opengl/wrapper/mod.rs @@ -0,0 +1,6 @@ +pub mod attribute; +pub mod buffer; +pub mod other_misc_wrapper; +pub mod texture; +pub mod uniform; +pub mod vertex_array; diff --git a/src/opengl/wrapper/other_misc_wrapper.rs b/src/opengl/wrapper/other_misc_wrapper.rs new file mode 100644 index 000000000..02d3594c6 --- /dev/null +++ b/src/opengl/wrapper/other_misc_wrapper.rs @@ -0,0 +1,206 @@ +//! These wrappers make it easier to use certain OpenGL types. +//! This draws inspiration from some of the code in Microsoft's MIT-licensed libk4a, +//! particularly the code in `openglhelpers.h`. + +use std::fmt::Formatter; +use std::ptr::null; + +use gl; +use gl::types::*; + +/// A slightly more convenient OpenGL Framebuffer type +#[derive(Clone, Debug)] +pub struct Framebuffer { + id: GLuint, +} + +impl Framebuffer { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + pub fn gen(&mut self) { + unsafe { + gl::GenFramebuffers(1, &mut self.id); + } + } + + pub fn delete(&mut self) { + unsafe { + gl::DeleteFramebuffers(1, &mut self.id); + } + } +} + +/// A slightly more convenient OpenGL Renderbuffer type +#[derive(Clone, Debug)] +pub struct Renderbuffer { + id: GLuint, +} + +impl Renderbuffer { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + fn gen(&mut self) { + unsafe { + gl::GenRenderbuffers(1, &mut self.id); + } + } + + fn delete(&mut self) { + unsafe { + gl::DeleteRenderbuffers(1, &mut self.id); + } + } +} + +/// Represents the OpenGL errors returned by `glGetError()`. +/// https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/glGetError.xhtml +/// When an error flag is set, results of a GL operation are undefined only if GL_OUT_OF_MEMORY +/// has occurred. In all other cases, the command generating the error is ignored and has no effect +/// on the GL state or frame buffer contents. If the generating command returns a value, it +/// returns 0. If glGetError itself generates an error, it returns 0. +#[derive(Copy,Clone,Debug)] +pub enum OpenGlError { + /// No error has been recorded. The value of this symbolic constant is guaranteed to be 0. + /// NB: While GL_NO_ERROR is represented here, it won't be returned by the convenience function. + NoError, + /// An unacceptable value is specified for an enumerated argument. The offending command is + /// ignored and has no other side effect than to set the error flag. + InvalidEnum, + /// A numeric argument is out of range. The offending command is ignored and has no other side + /// effect than to set the error flag. + InvalidValue, + /// The specified operation is not allowed in the current state. The offending command is + /// ignored and has no other side effect than to set the error flag. + InvalidOperation, + /// The framebuffer object is not complete. The offending command is ignored and has no other + /// side effect than to set the error flag. + InvalidFramebufferOperation, + /// There is not enough memory left to execute the command. The state of the GL is undefined, + /// except for the state of the error flags, after this error is recorded. + OutOfMemory, + /// An attempt has been made to perform an operation that would cause an internal stack to + /// underflow. + StackUnderflow, + /// An attempt has been made to perform an operation that would cause an internal stack to + /// overflow. + StackOverflow, + /// Invented error if we can't determine which enum was returned. + UnknownError(u32), +} + +impl std::fmt::Display for OpenGlError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let explain = match self { + OpenGlError::NoError => "OpenGL error: No Error (this shouldn't ever be set)".into(), + OpenGlError::InvalidEnum => "OpenGL error: Invalid Enum".into(), + OpenGlError::InvalidValue => "OpenGL error: Invalid Value".into(), + OpenGlError::InvalidOperation => "OpenGL error: Invalid Operation".into(), + OpenGlError::InvalidFramebufferOperation => "OpenGL error: Invalid Framebuffer Operation".into(), + OpenGlError::OutOfMemory => "OpenGL error: Out of Memory".into(), + OpenGlError::StackUnderflow => "OpenGL error: Stack Underflow".into(), + OpenGlError::StackOverflow => "OpenGL error: Stack Overflow".into(), + OpenGlError::UnknownError(result) => format!("OpenGL error: Unknown Error {}", result), + }; + write!(f, "{}", explain) + } +} + +impl std::error::Error for OpenGlError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + +/// Query OpenGL for the current error. +pub fn gl_get_error() -> Result<(), OpenGlError> { + let result = unsafe { + gl::GetError() + }; + match result { + gl::NO_ERROR => Ok(()), + gl::INVALID_ENUM => Err(OpenGlError::InvalidEnum), + gl::INVALID_VALUE => Err(OpenGlError::InvalidValue), + gl::INVALID_OPERATION => Err(OpenGlError::InvalidOperation), + gl::INVALID_FRAMEBUFFER_OPERATION => Err(OpenGlError::InvalidFramebufferOperation), + gl::OUT_OF_MEMORY => Err(OpenGlError::OutOfMemory), + gl::STACK_UNDERFLOW => Err(OpenGlError::StackUnderflow), + gl::STACK_OVERFLOW => Err(OpenGlError::StackOverflow), + _ => Err(OpenGlError::UnknownError(result)), + } +} + +/// Calculate the stride width for OpenGL +/// Useful for `gl::VertexAttribPointer`. +pub fn get_stride(size: usize) -> gl::types::GLint { + (size * std::mem::size_of::()) as gl::types::GLint +} + +/// Calculate the offset for OpenGL +/// Useful for `gl::VertexAttribPointer`. +pub fn get_pointer_offset(offset: usize) -> *const gl::types::GLvoid { + match offset { + 0 => null(), + _ => (offset * std::mem::size_of::()) as *const gl::types::GLvoid, + } +} diff --git a/src/opengl/wrapper/texture.rs b/src/opengl/wrapper/texture.rs new file mode 100644 index 000000000..f58748a94 --- /dev/null +++ b/src/opengl/wrapper/texture.rs @@ -0,0 +1,63 @@ +//! These wrappers make it easier to use certain OpenGL types. +//! This draws inspiration from some of the code in Microsoft's MIT-licensed libk4a, +//! particularly the code in `openglhelpers.h`. + +use gl::types::*; + +/// A slightly more convenient OpenGL Texture type +#[derive(Clone, Debug)] +pub struct Texture { + id: GLuint, +} + +impl Texture { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + pub fn bind_as_texture_2d(&self) { + unsafe { + gl::BindTexture(gl::TEXTURE_2D, self.id); + } + } + + fn gen(&mut self) { + unsafe { + gl::GenTextures(1, &mut self.id); + } + } + + fn delete(&mut self) { + unsafe { + gl::DeleteTextures(1, &mut self.id); + } + } +} diff --git a/src/opengl/wrapper/uniform.rs b/src/opengl/wrapper/uniform.rs new file mode 100644 index 000000000..996e767a9 --- /dev/null +++ b/src/opengl/wrapper/uniform.rs @@ -0,0 +1,35 @@ +use anyhow::Result as AnyhowResult; +use anyhow::bail; +use gl::types::*; +use std::os::raw::c_char; +use std::ffi::CString; + +/// A typesafe handle on uniform IDs with convenience methods. +#[derive(Clone, Debug, Copy)] +pub struct Uniform { + id: GLint, +} + +impl Uniform { + pub fn lookup(identifier: &str, opengl_program_id: GLuint) -> AnyhowResult { + let id_cstr : CString = CString::new(identifier)?; + let id_cstr_ptr : *const c_char = id_cstr.as_ptr() as *const c_char; + + let mut loc = -1; + unsafe { + loc = gl::GetUniformLocation(opengl_program_id, id_cstr_ptr); + } + + if loc < 0 { + bail!("Couldn't find uniform named '{}' in OpenGL program {}.", identifier, opengl_program_id); + } + + Ok(Self { + id: loc, + }) + } + + pub fn id(&self) -> GLint { + self.id + } +} diff --git a/src/opengl/wrapper/vertex_array.rs b/src/opengl/wrapper/vertex_array.rs new file mode 100644 index 000000000..9df292edd --- /dev/null +++ b/src/opengl/wrapper/vertex_array.rs @@ -0,0 +1,64 @@ +//! These wrappers make it easier to use certain OpenGL types. +//! This draws inspiration from some of the code in Microsoft's MIT-licensed libk4a, +//! particularly the code in `openglhelpers.h`. + +use gl::types::*; + +/// A slightly more convenient OpenGL VertexArray type +#[derive(Clone, Debug)] +pub struct VertexArray { + id: GLuint, +} + +impl VertexArray { + pub fn new() -> Self { + return Self { + id: 0, + } + } + + pub fn new_initialized() -> Self { + let mut new_object= Self::new(); + new_object.init(); + new_object + } + + pub fn is_initialized(&self) -> bool { + self.id != 0 + } + + pub fn id(&self) -> GLuint { + self.id + } + + pub fn init(&mut self) { + self.reset(); + self.gen(); + } + + pub fn reset(&mut self) { + if self.is_initialized() { + self.delete(); + self.id = 0; + } + } + + pub fn bind(&self) { + unsafe { + gl::BindVertexArray(self.id); + } + } + + fn gen(&mut self) { + unsafe { + gl::GenVertexArrays(1, &mut self.id); + } + } + + fn delete(&mut self) { + unsafe { + gl::DeleteVertexArrays(1, &mut self.id); + } + } +} + diff --git a/src/point_cloud/debug/capture_proxy.rs b/src/point_cloud/debug/capture_proxy.rs new file mode 100644 index 000000000..49ee8fa19 --- /dev/null +++ b/src/point_cloud/debug/capture_proxy.rs @@ -0,0 +1,123 @@ +use anyhow::Result as AnyhowResult; +use crate::point_cloud::debug::image_proxy::ImageProxy; +use k4a_sys_temp as k4a_sys; +use kinect::CaptureError; +use kinect; +use std::path::Path; +use std::sync::{RwLock, Mutex, PoisonError, MutexGuard, Arc}; +use winit::event::VirtualKeyCode::Mute; + +/// Store either a wrapped capture, or a k4a Capture. +/// Useful for debugging. +#[derive(Clone)] +pub struct CaptureProxy { + storage: UnderlyingStorage, +} + +#[derive(Clone)] +enum UnderlyingStorage { + CameraImageBytes { + color_image: ImageProxy, + depth_image: ImageProxy, + }, + K4aCapture { + capture: kinect::Capture, + maybe_color_image: Option, + maybe_depth_image: Option, + } +} + +impl CaptureProxy { + + pub fn from_image_proxy_pair(color_image: ImageProxy, depth_image: ImageProxy) -> Self { + Self { + storage: UnderlyingStorage::CameraImageBytes { + color_image, + depth_image, + } + } + } + + pub fn from_k4a_capture(capture: &kinect::Capture) -> Self { + // NB: We need to increase the refcount. + // K4a manages the memory under the hood. + let capture = capture.clone(); + Self::consume_k4a_capture(capture) + } + + pub fn consume_k4a_capture(capture: kinect::Capture) -> Self { + // NB: I tried to lazily unpack these, but interior mutability Sync/Send was a nightmare. + // The poor ergonomics were not worth it. + let color_image = capture.get_color_image() + .map(|image| ImageProxy::consume_k4a_image(image)) + .ok(); + + let depth_image = capture.get_depth_image() + .map(|image| ImageProxy::consume_k4a_image(image)) + .ok(); + + Self { + storage: UnderlyingStorage::K4aCapture { + capture, + maybe_color_image: color_image, + maybe_depth_image: depth_image, + } + } + } + + pub fn is_k4a(&self) -> bool { + match &self.storage { + UnderlyingStorage::CameraImageBytes{ .. } => false, + UnderlyingStorage::K4aCapture { .. } => true, + } + } + + pub fn get_color_image(&self) -> AnyhowResult<&ImageProxy> { + match &self.storage { + UnderlyingStorage::CameraImageBytes { color_image, .. } => Ok(color_image), + UnderlyingStorage::K4aCapture{ maybe_color_image, .. } => { + let inner = maybe_color_image.as_ref() + .ok_or(CaptureError::NullCapture)?; + Ok(inner) + } + } + } + + pub fn get_depth_image(&self) -> AnyhowResult<&ImageProxy> { + match &self.storage { + UnderlyingStorage::CameraImageBytes { depth_image, .. } => Ok(depth_image), + UnderlyingStorage::K4aCapture{ maybe_depth_image, .. } => { + let inner = maybe_depth_image.as_ref() + .ok_or(CaptureError::NullCapture)?; + Ok(inner) + } + } + } + + pub fn debug_get_capture(&self) -> kinect::Capture { + match &self.storage { + UnderlyingStorage::CameraImageBytes{ .. } => unimplemented!("Cannot get capture for non-k4a"), + UnderlyingStorage::K4aCapture { capture, .. } => capture.clone(), + } + } + + pub fn debug_save_files(&self, path_prefix: &Path, suffix: &str) -> AnyhowResult<()> { + match &self.storage { + UnderlyingStorage::CameraImageBytes{ .. } => unimplemented!("No need to debug save for raw bytes"), + UnderlyingStorage::K4aCapture { maybe_color_image, maybe_depth_image, .. } => { + + if let Some(color_image) = maybe_color_image { + let filename = path_prefix.join(format!("./color_frame_{}", suffix)); + color_image.debug_save_file(filename.to_str().unwrap())?; + } + + if let Some(depth_image) = maybe_depth_image { + let filename = path_prefix.join(format!("./depth_frame_{}", suffix)); + depth_image.debug_save_file(filename.to_str().unwrap())?; + } + + Ok(()) + }, + } + } +} diff --git a/src/point_cloud/debug/image_proxy.rs b/src/point_cloud/debug/image_proxy.rs new file mode 100644 index 000000000..384f89de4 --- /dev/null +++ b/src/point_cloud/debug/image_proxy.rs @@ -0,0 +1,177 @@ +use anyhow::Result as AnyhowResult; +use crate::files::write_to_file_from_byte_ptr::write_to_file_from_byte_ptr; +use k4a_sys_temp as k4a_sys; +use kinect::ImageFormat; +use kinect; +use std::fs::File; +use std::fs; +use std::io::Read; +use std::ptr::null_mut; + +/// Store either raw bytes, or wrap a k4a::Image +/// This is meant to be plumbed through the system instead of a k4a::Image (depth or color image) +#[derive(Clone)] +pub struct ImageProxy { + storage: UnderlyingStorage, +} + +/// The underlying data store +#[derive(Clone)] +enum UnderlyingStorage { + /// Our own serialization + Bytes { + bytes: Vec, + width: usize, + height: usize, + stride_bytes: usize, + format: ImageFormat, + }, + /// From the Kinect camera + K4aImage(kinect::Image), +} + +impl ImageProxy { + + pub fn from_file( + filename: &str, + width: usize, + height: usize, + stride_bytes: usize, + format: ImageFormat) -> AnyhowResult + { + let mut file = File::open(filename)?; + let metadata = fs::metadata(&filename)?; + + let mut buffer = vec![0u8; metadata.len() as usize]; + file.read(&mut buffer)?; + + let storage = UnderlyingStorage::Bytes { + bytes: buffer, + width, + height, + stride_bytes, + format, + }; + + Ok(Self { + storage, + }) + } + + pub fn from_k4a_image(image: &kinect::Image) -> Self { + // NB: We need to increase the refcount. + // K4a manages the memory under the hood. + let image = image.clone(); + let storage = UnderlyingStorage::K4aImage(image); + Self { + storage, + } + } + + pub fn consume_k4a_image(image: kinect::Image) -> Self { + let storage = UnderlyingStorage::K4aImage(image); + Self { + storage, + } + } + + pub fn is_k4a(&self) -> bool { + match &self.storage { + UnderlyingStorage::Bytes { .. } => false, + UnderlyingStorage::K4aImage(_) => true, + } + } + + pub fn len(&self) -> usize { + match &self.storage { + UnderlyingStorage::Bytes { bytes, .. } => bytes.len(), + UnderlyingStorage::K4aImage(image) => image.get_size(), + } + } + + pub fn as_ptr(&self) -> *const u8 { + match &self.storage { + UnderlyingStorage::Bytes{ ref bytes, .. } => { + bytes.as_ptr() + }, + UnderlyingStorage::K4aImage(ref image) => { + image.get_buffer() as *const u8 + }, + } + } + + pub fn get_buffer(&self) -> *const u8 { + self.as_ptr() + } + + pub fn get_width_pixels(&self) -> usize { + match &self.storage { + UnderlyingStorage::Bytes { width, .. } => *width, + UnderlyingStorage::K4aImage(image) => image.get_width_pixels(), + } + } + + pub fn get_height_pixels(&self) -> usize { + match &self.storage { + UnderlyingStorage::Bytes { height, .. } => *height, + UnderlyingStorage::K4aImage(image) => image.get_height_pixels(), + } + } + + pub fn get_stride_bytes(&self) -> usize { + match &self.storage { + UnderlyingStorage::Bytes { stride_bytes, .. } => *stride_bytes, + UnderlyingStorage::K4aImage(image) => image.get_stride_bytes(), + } + } + + pub fn get_format(&self) -> ImageFormat { + match &self.storage { + UnderlyingStorage::Bytes { format, .. } => *format, + UnderlyingStorage::K4aImage(image) => image.get_format(), + } + } + + pub fn get_handle(&self) -> k4a_sys::k4a_image_t { + match &self.storage { + UnderlyingStorage::Bytes { bytes, width, height, stride_bytes, format } => { + // TODO: We're not handing over ownership of the buffer! + // The k4a_image cannot be used after this ImageProxy dies! + let mut handle = null_mut(); + + //println!("fake.get_handle(): {:?}, {}, {}x{}", format, stride_bytes, width, height); + + unsafe { + k4a_sys::k4a_image_create_from_buffer( + format.to_k4a(), + *width as i32, + *height as i32, + *stride_bytes as i32, + bytes.as_ptr() as *mut u8, // TODO: Technically we won't write. + bytes.len(), + None, + null_mut(), + &mut handle, + ); + } + + handle + }, + UnderlyingStorage::K4aImage(image) => { + //println!("real.get_handle(): {:?}, {}, {}x{}", image.get_format(), image.get_stride_bytes(), image.get_width_pixels(), image.get_height_pixels()); + image.get_handle() + }, + } + } + + pub fn debug_save_file(&self, filename: &str) -> AnyhowResult<()> { + match &self.storage { + UnderlyingStorage::Bytes { .. } => unimplemented!("No need to debug save raw bytes"), + UnderlyingStorage::K4aImage(image) => { + let byte_src = image.get_buffer(); + let size_bytes = image.get_size(); + write_to_file_from_byte_ptr(filename, byte_src, size_bytes) + } + } + } +} diff --git a/src/point_cloud/debug/mod.rs b/src/point_cloud/debug/mod.rs new file mode 100644 index 000000000..e4df8b837 --- /dev/null +++ b/src/point_cloud/debug/mod.rs @@ -0,0 +1,2 @@ +pub mod image_proxy; +pub mod capture_proxy; diff --git a/src/point_cloud/gpu_point_cloud_converter.rs b/src/point_cloud/gpu_point_cloud_converter.rs new file mode 100644 index 000000000..4468847ab --- /dev/null +++ b/src/point_cloud/gpu_point_cloud_converter.rs @@ -0,0 +1,564 @@ +//! This is a port of Microsoft's libk4a `gpudepthtopointcloudconverter.h`. +//! This code turns depth images into point clouds. + +use std::ffi::CString; +use std::fmt::Formatter; +use std::mem::size_of; +use std::os::raw::{c_char, c_int, c_void}; +use std::ptr; +use std::ptr::null; +use std::str; + +use gl; +use gl::types::*; + +use crate::files::read_file_string_contents::read_file_string_contents; +use crate::files::write_to_file_from_byte_ptr::write_to_file_from_byte_ptr; +use crate::opengl::compile_shader::compile_shader; +use crate::opengl::wrapper::buffer::Buffer; +use crate::opengl::wrapper::other_misc_wrapper::OpenGlError; +use crate::opengl::wrapper::other_misc_wrapper::gl_get_error; +use crate::opengl::wrapper::texture::Texture; +use crate::point_cloud::debug::image_proxy::ImageProxy; +use crate::point_cloud::pixel_structs::DepthPixel; +use k4a_sys_temp as k4a_sys; +use kinect::{Image, Calibration, ImageFormat}; +use kinect; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug)] +pub enum PointCloudComputeError { + OpenGlError(OpenGlError), + UnknownError, +} + +impl From for PointCloudComputeError { + fn from(error: OpenGlError) -> Self { + PointCloudComputeError::OpenGlError(error) + } +} + +impl std::fmt::Display for PointCloudComputeError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let description = match self { + PointCloudComputeError::OpenGlError(inner) => { + format!("Compute OpenGL error: {}", inner) + }, + PointCloudComputeError::UnknownError => "Unknown Compute Error".into(), + }; + + write!(f, "{}", description) + } +} + +impl std::error::Error for PointCloudComputeError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + +/// The format that the point cloud texture uses internally to store points. +/// If you want to use the texture that this outputs from your shader, you +/// need to pass this as the format argument to glBindImageTexture(). +pub const POINT_CLOUD_TEXTURE_FORMAT : GLuint = gl::RGBA32F; + +pub struct GpuPointCloudConverter { + /// The OpenGL program + shader_program_id: GLuint, + + /// The x,y table as a texture + pub xy_table_texture: Texture, + + /// Preallocated texture for depth image so we don't have to reallocate every frame. + pub depth_image_texture: Texture, + + depth_image_pixel_buffer: Buffer, + + /// Uniform location in the shader program + // TODO: Turns out this does nothing + //dest_tex_id: GLint, + + /// Uniform location in the shader program + xy_table_id: GLint, + + /// Uniform location in the shader program + depth_image_id: GLint, +} + +impl GpuPointCloudConverter { + + pub fn new() -> Self { + let program_id = unsafe { gl::CreateProgram() }; + + let point_cloud_compute_shader = read_file_string_contents("src/point_cloud/shaders/point_cloud_compute_shader.glsl").unwrap(); + + let shader_id = compile_shader(&point_cloud_compute_shader, gl::COMPUTE_SHADER); + + link_program(program_id, shader_id); + + let mut dest_tex_id = 0; + let mut xy_table_id = 0; + let mut depth_image_id = 0; + + /// Uniform variable name in OpenGL shader program + // TODO: Turns out this does nothing. + let DEST_TEX : CString = CString::new("destTex").expect("string is correct"); + let DEST_TEX_PTR : *const c_char = DEST_TEX.as_ptr() as *const c_char; + + /// Uniform variable name in OpenGL shader program + let XY_TABLE : CString = CString::new("xyTable").expect("string is correct"); + let XY_TABLE_PTR: *const c_char = XY_TABLE.as_ptr() as *const c_char; + + /// Uniform variable name in OpenGL shader program + let DEPTH_IMAGE : CString = CString::new("depthImage").expect("string is correct"); + let DEPTH_IMAGE_PTR : *const c_char = DEPTH_IMAGE.as_ptr() as *const c_char; + + unsafe { + dest_tex_id = gl::GetUniformLocation(program_id, DEST_TEX_PTR); + xy_table_id = gl::GetUniformLocation(program_id, XY_TABLE_PTR); + depth_image_id = gl::GetUniformLocation(program_id, DEPTH_IMAGE_PTR); + } + + GpuPointCloudConverter { + shader_program_id: program_id, + //shader_id, + //dest_tex_id, + xy_table_id, + depth_image_id, + depth_image_texture: Texture::new(), + xy_table_texture: Texture::new(), + depth_image_pixel_buffer: Buffer::new(), + } + } + + /// Takes depth data and turns it into a texture containing the XYZ coordinates of the depth map + /// using the most recently set-to-active XY table. The input depth image and output texture + /// (if already set) must be of the same resolution that was used to generate that XY table, or + /// else behavior is undefined. + /// + /// Essentially a reimplementation of k4a::transform::depth_image_to_point_cloud on the GPU. + /// This is much more performant than k4a::transform::depth_image_to_point_cloud, but is a bit + /// more unwieldly to use since you have to use its output in shaders. + /// + /// The output texture has an internal format of GL_RGBA32F and is intended to be used directly + /// by other OpenGL shaders as an image2d uniform. + /// + /// To avoid excess image allocations, you can reuse a texture that was previously output + /// by this function, provided the depth image and XY table previously used was for the same + /// sized texture. + /// + pub fn convert(&self, + //depth_image: &kinect::Image, + depth_image: &ImageProxy, + output_texture: &mut Texture, + camera_index: usize + ) -> Result<()> { + if !self.xy_table_texture.is_initialized() { + return Err(PointCloudComputeError::UnknownError); + } + + // Create output texture if it doesn't already exist + // + // We don't use the alpha channel, but it turns out OpenGL doesn't + // actually have a 3-component (i.e. RGB32F) format - you get + // 1, 2, or 4 components. + // + let width = depth_image.get_width_pixels() as i32; + let height = depth_image.get_height_pixels() as i32; + + if !output_texture.is_initialized() { + output_texture.init(); + + unsafe { + gl::ActiveTexture(gl::TEXTURE0); + gl::BindTexture(gl::TEXTURE_2D, output_texture.id()); + + // The format that the point cloud texture uses internally to store points. + // If you want to use the texture that this outputs from your shader, you + // need to pass this as the format argument to glBindImageTexture(). + // static constexpr GLenum PointCloudTextureFormat = GL_RGBA32F; + // gl::TexStorage2D(gl::TEXTURE_2D, 1, POINT_CLOUD_TEXTURE_FORMAT, width, height); + gl::TexStorage2D(gl::TEXTURE_2D, 1, gl::RGBA32F, width, height); + + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::NEAREST as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::NEAREST as i32); + } + } + + unsafe { + // Upload data to uniform texture + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, self.depth_image_pixel_buffer.id()); + gl::BindTexture(gl::TEXTURE_2D, self.depth_image_texture.id()); + + let num_bytes: GLuint = (width * height * size_of::() as i32) as GLuint; // libc::uint16_t = u16 + let length = (width * height) as usize; + //let num_bytes: GLuint = (width * height * size_of::() as i32) as GLuint; // libc::uint16_t = u16 + + // GLubyte *textureMappedBuffer = reinterpret_cast(...) + let texture_mapped_buffer = gl::MapBufferRange( + gl::PIXEL_UNPACK_BUFFER, + 0, + num_bytes as isize, + //(std::mem::size_of::() * length) as GLsizeiptr, + gl::MAP_WRITE_BIT | gl::MAP_INVALIDATE_BUFFER_BIT + //); + ) as *mut DepthPixel; + + if texture_mapped_buffer as usize == 0 { + return Err(PointCloudComputeError::UnknownError); + } + + let depth_image_buffer = depth_image.get_buffer(); + + //let filename = format!("output/depth_src_{}", camera_index); + //let size = (width * height) as usize * size_of::(); + //write_to_file_from_byte_ptr(&filename, depth_image_buffer, size).unwrap(); + + let typed_depth_image_buffer = depth_image_buffer as *const DepthPixel; + + std::ptr::copy::(typed_depth_image_buffer, texture_mapped_buffer, length as usize); + + let result = gl::UnmapBuffer(gl::PIXEL_UNPACK_BUFFER); + if result == gl::FALSE { + return Err(PointCloudComputeError::UnknownError); + } + + gl::TexSubImage2D( + gl::TEXTURE_2D, // target + 0, // level + 0, // x offset + 0, // y offset + width, + height, + gl::RED_INTEGER, //constexpr GLenum depthImageDataFormat = GL_RED_INTEGER; + gl::UNSIGNED_SHORT, //constexpr GLenum depthImageDataType = GL_UNSIGNED_SHORT; + null(), // data + ); + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, 0); + + gl::UseProgram(self.shader_program_id); + + // Bind textures that we're going to pass to the texture + gl::ActiveTexture(gl::TEXTURE0); + gl::BindTexture(gl::TEXTURE_2D, output_texture.id()); + gl::BindImageTexture( + 0, + output_texture.id(), + 0, + gl::FALSE, + 0, + gl::WRITE_ONLY, + //POINT_CLOUD_TEXTURE_FORMAT + //static constexpr GLenum PointCloudTextureFormat = GL_RGBA32F; + gl::RGBA32F, + ); + + gl::ActiveTexture(gl::TEXTURE1); + gl::BindTexture(gl::TEXTURE_2D, self.depth_image_texture.id()); + gl::BindImageTexture( + 1, + self.depth_image_texture.id(), + 0, + gl::FALSE, + 0, + gl::READ_ONLY, + gl::R16UI, //constexpr GLenum depthImageInternalFormat = GL_R16UI; + ); + gl::Uniform1i(self.depth_image_id, 1); + + gl::ActiveTexture(gl::TEXTURE2); + gl::BindTexture(gl::TEXTURE_2D, self.xy_table_texture.id()); + gl::BindImageTexture( + 2, + self.xy_table_texture.id(), + 0, + gl::FALSE, + 0, + gl::READ_ONLY, + gl::RG32F, //constexpr GLenum xyTableInternalFormat = GL_RG32F; + ); + gl::Uniform1i(self.xy_table_id, 2); + + gl::DispatchCompute( + depth_image.get_width_pixels() as u32, + depth_image.get_height_pixels() as u32, + 1, + ); + + // Wait for the rendering to finish before allowing reads to the texture we just wrote + gl::MemoryBarrier(gl::TEXTURE_FETCH_BARRIER_BIT); + + gl_get_error()?; + + // NB(bt): Restore to default active texture? Looks like this is required for 'imgui' + // This is needed despite using glGetInteger(GL_ACTIVE_TEXTURE) and resetting that value. + //gl::ActiveTexture(gl::TEXTURE0); + + /* + // Try to save the texture buffer to file... (not working yet) + let size_bytes = (width * height * 4); + let mut vec = vec![0u8; size_bytes as usize]; + let mut pointer = vec.as_mut_ptr() as *mut c_void; + + // Pull the texture out as bytes. + //gl::ActiveTexture(gl::TEXTURE1); + gl::BindTexture(gl::TEXTURE_2D, output_texture.id()); + + gl_get_error().unwrap(); + + gl::GetTextureImage( + output_texture.id(), + 0, // level + gl::RGBA32F, // format + gl::UNSIGNED_BYTE, // type + size_bytes, // buffer size + pointer, + ); + + gl_get_error().unwrap(); + */ + + Ok(()) + } + } + + /// Set the XY table that will be used by future calls to Convert(). Get an XY table by calling + /// GenerateXyTable(). + pub fn set_active_xy_table(&mut self, xy_table: &ImageProxy) -> Result<()> { + let width = xy_table.get_width_pixels() as i32; + let height = xy_table.get_height_pixels() as i32; + + // Upload the XY table as a texture so we can use it as a uniform + self.xy_table_texture.init(); + + unsafe { + gl::BindTexture(gl::TEXTURE_2D, self.xy_table_texture.id()); + gl::TexStorage2D( + gl::TEXTURE_2D, + 1, + gl::RG32F, // constexpr GLenum xyTableInternalFormat = GL_RG32F; + width, + height, + ); + + let xy_table_buffer = xy_table.get_buffer(); + + gl::TexSubImage2D( + gl::TEXTURE_2D, + 0, // level + 0, // xoffset + 0, // yoffset + width, + height, + gl::RG, //constexpr GLenum xyTableDataFormat = GL_RG; + gl::FLOAT, //constexpr GLenum xyTableDataType = GL_FLOAT; + xy_table_buffer as *const c_void, + ); + + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::NEAREST as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::NEAREST as i32); + } + + // Pre-allocate a texture for the depth images so we don't have to + // reallocate on every frame + self.depth_image_texture.init(); + self.depth_image_pixel_buffer.init(); + + let depth_image_size_bytes = (width * height * size_of::() as i32) as isize; // libc::uint16_t = u16 + + unsafe { + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, self.depth_image_pixel_buffer.id()); + gl::BufferData( + gl::PIXEL_UNPACK_BUFFER, + depth_image_size_bytes, + ptr::null_mut(), + gl::STREAM_DRAW, + ); + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, 0); + + gl::BindTexture(gl::TEXTURE_2D, self.depth_image_texture.id()); + + gl::TexStorage2D( + gl::TEXTURE_2D, + 1, + gl::R16UI, // constexpr GLenum depthImageInternalFormat = GL_R16UI; + width, + height, + ); + + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::NEAREST as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::NEAREST as i32); + } + + gl_get_error() + .map_err(|err| PointCloudComputeError::OpenGlError(err)) + } + + /// Creates a k4a::image containing the XY tables from calibration based on calibrationType. + /// The table is a 2D array of k4a_float2_t's with the same resolution as the camera of calibrationType + /// specified in calibration. + /// + /// You can use this table to convert a depth image into a point cloud, e.g. by using the Convert method. + /// Conversion is done by multiplying the depth pixel value by the XY table values - i.e. the result + /// pixel will be (xyTable[p].x * depthImage[p], xyTable[p].y * depthImage[p], depthImage[p]), where + /// p is the index of a given pixel. + pub fn generate_xy_table(calibration: &Calibration, + calibration_type: k4a_sys::k4a_calibration_type_t) + -> Result + { + let camera_calibration : k4a_sys::k4a_calibration_camera_t = match calibration_type { + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_COLOR => calibration.0.color_camera_calibration, + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_DEPTH => calibration.0.depth_camera_calibration, + _ => return Err(PointCloudComputeError::UnknownError), + }; + + let width = camera_calibration.resolution_width as u32; + let height = camera_calibration.resolution_height as u32; + let stride_bytes = width * size_of::() as u32; + + let xy_table = Image::create( + ImageFormat::Custom, + width, + height, + stride_bytes, + ).map_err(|_| PointCloudComputeError::UnknownError)?; + + // typedef union + // { + // // XY or array representation of vector + // struct _xy + // { + // float x; // < X component of a vector + // float y; // < Y component of a vector + // } xy; // < X, Y representation of a vector + // float v[2]; // < Array representation of a vector + // } k4a_float2_t; + let mut p = k4a_sys::k4a_float2_t { + xy: k4a_sys::k4a_float2_t__xy { + x: 0.0, + y: 0.0, + } + }; + + // typedef union + // { + // // XYZ or array representation of vector. + // struct _xyz + // { + // float x; // < X component of a vector. + // float y; // < Y component of a vector. + // float z; // < Z component of a vector. + // } xyz; // < X, Y, Z representation of a vector. + // float v[3]; // < Array representation of a vector. + // } k4a_float3_t; + let mut ray = k4a_sys::k4a_float3_t { + xyz: k4a_sys::k4a_float3_t__xyz { + x: 0.0, + y: 0.0, + z: 0.0, + } + }; + + let mut idx = 0; + unsafe { + let table_data = xy_table.get_buffer(); + let typed_buffer = table_data as *mut k4a_sys::k4a_float2_t; + + for y in 0..height { + p.xy.y = y as f32; + + for x in 0..width { + p.xy.x = x as f32; + + let mut valid: c_int = -1; + let result = unsafe { + // https://docs.rs/k4a-sys/0.2.0/k4a_sys/fn.k4a_calibration_2d_to_3d.html + k4a_sys::k4a_calibration_2d_to_3d( + &calibration.0, + &p, // source point 2d + 1.0, // source depth mm + calibration_type, // source camera + calibration_type, // target camera + &mut ray, // target point3d mm + &mut valid // set to 1 when valid result, 0 when coordinate is not valid + ) + }; + + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(PointCloudComputeError::UnknownError); + } + + if valid == 1 { + unsafe { + /* + println!("[size: {}x{}] 2D coord {}x{} --> 3D cord {}x{}x{}", + width, height, p.xy.x, p.xy.y, ray.xyz.x, ray.xyz.y, ray.xyz.z); + [size: 1280x720] 2D coord 0x0 --> 3D cord -1.0120127x-0.5833072x1 + [size: 1280x720] 2D coord 1x0 --> 3D cord -1.010335x-0.5832499x1 + [size: 1280x720] 2D coord 2x0 --> 3D cord -1.0086561x-0.58319193x1 + ... + [size: 1280x720] 2D coord 1277x719 --> 3D cord 1.0083642x0.55422443x1 + [size: 1280x720] 2D coord 1278x719 --> 3D cord 1.0100702x0.55429035x1 + [size: 1280x720] 2D coord 1279x719 --> 3D cord 1.0117699x0.55435246x1 + ... + [size: 640x576] 2D coord 0x0 --> 3D cord -0.97488284x-1.0173141x1 + [size: 640x576] 2D coord 1x0 --> 3D cord -0.97017723x-1.0155252x1 + [size: 640x576] 2D coord 2x0 --> 3D cord -0.9654954x-1.0137504x1 + ... + [size: 640x576] 2D coord 637x575 --> 3D cord 0.7947599x0.60008353x1 + [size: 640x576] 2D coord 638x575 --> 3D cord 0.79833484x0.60085475x1 + [size: 640x576] 2D coord 639x575 --> 3D cord 0.8019226x0.60163087x1 + */ + (*typed_buffer.offset(idx)).xy.x = ray.xyz.x; + (*typed_buffer.offset(idx)).xy.y = ray.xyz.y; + } + } else { + unsafe { + // This pixel is invalid + (*typed_buffer.offset(idx)).xy.x = 0.0; + (*typed_buffer.offset(idx)).xy.y = 0.0; + } + } + + idx += 1; + } + } + } + + Ok(xy_table) + } +} + +fn link_program(program: GLuint, shader: GLuint) -> GLuint { + unsafe { + gl::AttachShader(program, shader); + gl::LinkProgram(program); + + // Get the link status + let mut status = gl::FALSE as GLint; + gl::GetProgramiv(program, gl::LINK_STATUS, &mut status); + + // Fail on error + if status != (gl::TRUE as GLint) { + let mut len: GLint = 0; + gl::GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len); + let mut buf = Vec::with_capacity(len as usize); + buf.set_len((len as usize) - 1); // subtract 1 to skip the trailing null character + gl::GetProgramInfoLog( + program, + len, + ptr::null_mut(), + buf.as_mut_ptr() as *mut GLchar, + ); + panic!( + "{}", + str::from_utf8(&buf) + .ok() + .expect("ProgramInfoLog not valid utf8") + ); + } + program + } +} diff --git a/src/point_cloud/mod.rs b/src/point_cloud/mod.rs new file mode 100644 index 000000000..28b4389fc --- /dev/null +++ b/src/point_cloud/mod.rs @@ -0,0 +1,7 @@ +pub mod debug; +pub mod gpu_point_cloud_converter; +pub mod pixel_structs; +pub mod point_cloud_renderer; +pub mod point_cloud_visualiser; +pub mod util; +pub mod viewer_image; \ No newline at end of file diff --git a/src/point_cloud/pixel_structs.rs b/src/point_cloud/pixel_structs.rs new file mode 100644 index 000000000..4e7f64ce9 --- /dev/null +++ b/src/point_cloud/pixel_structs.rs @@ -0,0 +1,12 @@ +use libc::uint16_t; +use libc::uint8_t; + +#[repr(C)] +pub struct BgraPixel { + pub blue: uint8_t, + pub green: uint8_t, + pub red: uint8_t, + pub alpha: uint8_t, +} + +pub type DepthPixel = uint16_t; diff --git a/src/point_cloud/point_cloud_renderer.rs b/src/point_cloud/point_cloud_renderer.rs new file mode 100644 index 000000000..5e20f7e89 --- /dev/null +++ b/src/point_cloud/point_cloud_renderer.rs @@ -0,0 +1,453 @@ +//! This is a port of Microsoft's libk4a `k4apointcloudrenderer.cpp`. +//! This provides the visual output. + +use anyhow::Result as AnyhowResult; +use arcball::ArcballCamera; +use crate::assets::obj_loader::{load_wavefront, ExtractedVertex}; +use crate::assets::renderable_object::RenderableObject; +use crate::files::read_file_string_contents::read_file_string_contents; +use crate::files::write_to_file_from_byte_ptr::write_to_file_from_byte_ptr; +use crate::gui::mouse_camera_arcball::MouseCameraArcball; +use crate::opengl::compile_shader::compile_shader; +use crate::opengl::link_program::link_shader_program; +use crate::opengl::wrapper::buffer::Buffer; +use crate::opengl::wrapper::other_misc_wrapper::{gl_get_error, OpenGlError}; +use crate::opengl::wrapper::texture::Texture; +use crate::opengl::wrapper::vertex_array::VertexArray; +use crate::point_cloud::debug::image_proxy::ImageProxy; +use crate::point_cloud::pixel_structs::BgraPixel; +use gl::types::*; +use gl; +use gltf::Gltf; +use std::ffi::CString; +use std::fmt::Formatter; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write, BufReader}; +use std::mem::size_of; +use std::os::raw::{c_char, c_void}; +use std::path::Path; +use std::ptr::{null, null_mut}; +use std::ptr; +use std::str; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tobj::{load_obj, load_mtl}; +use crate::assets::positionable_object::{PositionableObject, ObjectType}; +use cgmath::{Matrix4, SquareMatrix, Matrix}; +use crate::opengl::wrapper::uniform::Uniform; +use crate::opengl::matrices::{initial_projection_matrix_4x4_flat, initial_view_matrix_4x4_flat, identity_matrix_4x4, initial_view_matrix_4x4, initial_projection_matrix_4x4}; +use image::DynamicImage; +use crate::opengl::wrapper::attribute::Attribute; +use nalgebra::RealField; +use cgmath::num_traits::ToPrimitive; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug)] +pub enum PointCloudRendererError { + OpenGlError(OpenGlError), + UnknownError, +} + +impl From for PointCloudRendererError { + fn from(error: OpenGlError) -> Self { + PointCloudRendererError::OpenGlError(error) + } +} + +impl std::fmt::Display for PointCloudRendererError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let description = match self { + PointCloudRendererError::OpenGlError(inner) => { + format!("Renderer OpenGL error: {}", inner) + }, + PointCloudRendererError::UnknownError => "Unknown Renderer Error".into(), + }; + + write!(f, "{}", description) + } +} + +impl std::error::Error for PointCloudRendererError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + +pub struct PointCloudRenderer { + num_cameras: usize, + + arcball_camera: Arc>, + + /// The OpenGL program + shader_program_id: GLuint, + /// The OpenGL vertex shader + vertex_shader_id: GLuint, + /// The OpenGL fragment shader + fragment_shader_id: GLuint, + + // TODO: better matrix types + view: [f32; 16], + projection: [f32; 16], + view_matrix: [[f32; 4]; 4], + projection_matrix: [[f32; 4]; 4], + /// Matrix applied to each model + /// If a model doesn't set its own, it defaults to identity matrix + default_model_view_matrix: [[f32; 4]; 4], + + /// Renderer setting: size of the rendered points + point_size: u8, + /// Renderer setting: shading enabled + enable_shading: bool, + /// Uniform location in the shader program. + enable_shading_index: GLint, + + /// Point array size + vertex_arrays_size_bytes: Vec, + + /// Uniform location in the shader program. + view_transform_id: Uniform, + /// Uniform location in the shader program. + projection_transform_id: Uniform, + /// Uniform location in the shader program. + model_transform_id: Uniform, + + /// An integer uniform that tells GLSL what type of data it's dealing with. + /// This should allow us to mix Point Cloud data and Geometric Objects + /// 1 = object + /// 2 = pointCloud + uniform_vertex_type: Uniform, + + /// Used to signal which camera's point cloud texture is active. + uniform_point_cloud_texture_to_use: Uniform, + + /// Uniform for object textures in the shader program. + object_texture_uniform: Uniform, + /// Location of the texture coordinate input + texture_coordinates_attribute: Attribute, + + /// Uniform location in the shader program. + point_cloud_texture_indices: Vec, + /// 'in vec4 inColor' + color_vertex_attribute_location: GLint, + + vertex_array_objects: Vec, + vertex_color_buffer_objects: Vec, + + renderable_objects: Vec +} + + +impl PointCloudRenderer { + + pub fn new(num_cameras: usize, arcball: Arc>) -> AnyhowResult { + unsafe { + gl::Enable(gl::PROGRAM_POINT_SIZE); + } + + let program_id = unsafe { gl::CreateProgram() }; + + let mut point_cloud_vertex_shader = read_file_string_contents("src/point_cloud/shaders/point_cloud_vertex_shader.glsl").unwrap(); + let mut point_cloud_fragment_shader = read_file_string_contents("src/point_cloud/shaders/point_cloud_fragment_shader.glsl").unwrap(); + + //let mut point_cloud_vertex_shader = read_file_string_contents("src/point_cloud/shaders/simple_vertex_shader.glsl").unwrap(); + + let vertex_shader_id = compile_shader(&point_cloud_vertex_shader, gl::VERTEX_SHADER); + let fragment_shader_id = compile_shader(&point_cloud_fragment_shader, gl::FRAGMENT_SHADER); + + link_shader_program(program_id, vertex_shader_id, fragment_shader_id); + + /// Uniform variable name in OpenGL shader program + let ENABLE_SHADING : CString = CString::new("enableShading").expect("string is correct"); + let ENABLE_SHADING_PTR : *const c_char = ENABLE_SHADING.as_ptr() as *const c_char; + + /// Uniform variable name in OpenGL shader program + let POINT_CLOUD_0 : CString = CString::new("pointCloudTexture0").expect("string is correct"); + let POINT_CLOUD_0_PTR : *const c_char = POINT_CLOUD_0.as_ptr() as *const c_char; + + let POINT_CLOUD_1 : CString = CString::new("pointCloudTexture1").expect("string is correct"); + let POINT_CLOUD_1_PTR : *const c_char = POINT_CLOUD_1.as_ptr() as *const c_char; + + let COLOR_LOCATION : CString = CString::new("inColor").expect("string is correct"); + let COLOR_LOCATION_PTR : *const c_char = COLOR_LOCATION.as_ptr() as *const c_char; + + let mut point_cloud_texture_indices = Vec::with_capacity(num_cameras); + + let mut view_index = 0; + let mut projection_index = 0; + let mut enable_shading_index = 0; + + unsafe { + // This function returns -1 if name does not correspond to an active uniform variable in + // program, if name starts with the reserved prefix "gl_", or if name is associated with an + // atomic counter or a named uniform block. + // FIXME: THe 'view' and 'projection' uniforms are not binding for some reason. + enable_shading_index = gl::GetUniformLocation(program_id, ENABLE_SHADING_PTR); + // TODO: This is hardcoded to 2 + point_cloud_texture_indices.push(gl::GetUniformLocation(program_id, POINT_CLOUD_0_PTR)); + point_cloud_texture_indices.push(gl::GetUniformLocation(program_id, POINT_CLOUD_1_PTR)); + } + + let model_transform = Uniform::lookup("model", program_id)?; + let view_transform = Uniform::lookup("view", program_id)?; + let projection_transform = Uniform::lookup("projection", program_id)?; + + let object_texture_uniform = Uniform::lookup("objTexture", program_id)?; + + let texture_coordinate_attribute = Attribute::lookup("vTextureCoord", program_id)?; + + let uniform_vertex_type = Uniform::lookup("vertexType", program_id)?; + + let uniform_point_cloud_texture_to_use = Uniform::lookup("pointCloudTextureToUse", program_id)?; + + let color_vertex_attribute_location = unsafe { + let location = gl::GetAttribLocation(program_id, COLOR_LOCATION_PTR); + + gl::EnableVertexAttribArray(location as u32); + + gl::VertexAttribPointer( + location as u32, + gl::BGRA as i32, + gl::UNSIGNED_BYTE, + gl::TRUE, + 0 as i32, + 0 as *const c_void, + ); + + location + }; + + Ok(Self { + num_cameras, + arcball_camera: arcball, + view: initial_view_matrix_4x4_flat(), + projection: initial_projection_matrix_4x4_flat(), + view_matrix: initial_view_matrix_4x4(), + projection_matrix: initial_projection_matrix_4x4(), + default_model_view_matrix: identity_matrix_4x4(), + shader_program_id: program_id, + vertex_shader_id, + fragment_shader_id, + object_texture_uniform, + texture_coordinates_attribute: texture_coordinate_attribute, + uniform_vertex_type, + uniform_point_cloud_texture_to_use, + point_size: 1, + enable_shading: false, + view_transform_id: view_transform, + projection_transform_id: projection_transform, + model_transform_id: model_transform, + enable_shading_index, + point_cloud_texture_indices, + vertex_array_objects: Vec::with_capacity(num_cameras), + vertex_arrays_size_bytes : Vec::with_capacity(num_cameras), + vertex_color_buffer_objects: Vec::with_capacity(num_cameras), + color_vertex_attribute_location, + renderable_objects: Vec::new(), + }) + } + + /// + /// Run once to set up initial rendering + /// + pub fn setup_rendering(&mut self) -> AnyhowResult<()> { + for _ in 0 .. self.num_cameras { + let vao = VertexArray::new_initialized(); + vao.bind(); + self.vertex_array_objects.push(vao); + + let buffer = Buffer::new_initialized(); + self.vertex_color_buffer_objects.push(buffer); + self.vertex_arrays_size_bytes.push(0); + } + + Ok(()) + } + + /// + /// Update the view matrices + /// + pub fn update_view_projection(&mut self, view: [[f32; 4]; 4], _projection: [[f32; 4]; 4]) { + self.view_matrix = view; + self.projection_matrix = initial_projection_matrix_4x4(); + } + + /// + /// + /// + pub fn update_point_clouds(&mut self, + color_images: &Vec, + point_cloud_textures: &Vec) -> Result<()> + { + for i in 0 .. self.num_cameras { + let color_image_bytes = color_images.get(i).unwrap(); + let color_image_size_bytes = color_image_bytes.len() as i32; + + let vao = self.vertex_array_objects.get(i).unwrap(); + let vertex_color_buffer = self.vertex_color_buffer_objects.get(i).unwrap(); + let mut vertex_array_size_bytes = self.vertex_arrays_size_bytes.get_mut(i).unwrap(); + + vao.bind(); + vertex_color_buffer.bind_as_array_buffer(); + + //if *vertex_array_size_bytes != color_image_size_bytes { + //println!("Establishing buffer"); + unsafe { + gl::BufferData( + gl::ARRAY_BUFFER, + color_image_size_bytes as isize, + null(), + gl::STREAM_DRAW + ); + } + *vertex_array_size_bytes = color_image_size_bytes; + //} + + let vertex_mapped_buffer = unsafe { + gl::MapBufferRange( + gl::ARRAY_BUFFER, + 0, + color_image_size_bytes as isize, + gl::MAP_WRITE_BIT | gl::MAP_INVALIDATE_BUFFER_BIT + ) as *mut u8 + }; + + // if vertex_mapped_buffer as usize == 0 { + // let error = gl_get_error().expect_err("should be map buffer range error"); + // return Err(PointCloudRendererError::OpenGlError(error)); + // } + + let result = unsafe { + // NB: This is the working texturing: + std::ptr::copy_nonoverlapping::( + color_image_bytes.as_ptr(), + vertex_mapped_buffer, + color_image_size_bytes as usize); + + gl::UnmapBuffer(gl::ARRAY_BUFFER) + }; + + // if result == gl::FALSE { + // let error = gl_get_error().expect_err("should be unmap buffer error"); + // return Err(PointCloudRendererError::OpenGlError(error)); + // } + + // THIS IS THE POINT CLOUD + let point_cloud_texture = point_cloud_textures.get(i).unwrap(); + unsafe { + gl::BindTexture(gl::TEXTURE_2D, point_cloud_texture.id()); + gl::BindImageTexture( + i as GLuint, // image unit (zero-indexed) // TODO - why does it work this way? + point_cloud_texture.id(), // texture + 0, // level + gl::FALSE, // layered + 0, // layer + gl::READ_ONLY, // access + gl::RGBA32F, //POINT_CLOUD_TEXTURE_FORMAT, + ); + + gl::ActiveTexture(gl::TEXTURE0); + + gl::EnableVertexAttribArray(self.color_vertex_attribute_location as u32); + + gl::VertexAttribPointer( + self.color_vertex_attribute_location as u32, + gl::BGRA as i32, + gl::UNSIGNED_BYTE, + gl::TRUE, + 0 as i32, + 0 as *const c_void, + ); + } + } + + unsafe { + gl::BindVertexArray(0); + } + + gl_get_error() + .map_err(|err| PointCloudRendererError::OpenGlError(err)) + } + + /// + /// + /// + pub fn render(&mut self) -> Result<()> { + // Animate the models. + for obj in self.renderable_objects.iter_mut() { + match obj.object_type { + ObjectType::Level => continue, + ObjectType::Skybox => continue, + _ => {}, + } + obj.increment_rotation_y(0.05); // TODO: Should be based on wall clock instead + } + + unsafe { + gl::UseProgram(self.shader_program_id); + + gl::Enable(gl::DEPTH_TEST); + gl::Enable(gl::BLEND); + gl::BlendFunc(gl::SRC_ALPHA, gl::ONE_MINUS_SRC_ALPHA); + gl::PointSize(self.point_size as f32); + + // Update view/projection matrices in shader + let typed_projection = self.projection_matrix.as_ptr() as *const GLfloat; + gl::UniformMatrix4fv(self.projection_transform_id.id(), 1, gl::FALSE, typed_projection); + + let typed_view = self.view_matrix.as_ptr() as *const GLfloat; + gl::UniformMatrix4fv(self.view_transform_id.id(), 1, gl::FALSE, typed_view); + + // Default model transform + let typed_model_view = self.default_model_view_matrix.as_ptr() as *const GLfloat; + gl::UniformMatrix4fv(self.model_transform_id.id(), 1, gl::FALSE, typed_model_view); + + // Update render settings in shader + let _enable_shading = if self.enable_shading { 1 } else { 0 }; + let enable_shading = 0; // TODO FIXME FIXME FIXME + + gl::Uniform1i(self.enable_shading_index, enable_shading); + + // Tell GLSL we're doing point clouds + gl::Uniform1i(self.uniform_vertex_type.id() as GLint, 2); + + for i in 0 .. self.num_cameras { + let vao = self.vertex_array_objects.get(i).unwrap(); + let vertex_array_size_bytes = self.vertex_arrays_size_bytes.get(i).unwrap(); + let size = vertex_array_size_bytes / size_of::() as i32; + + vao.bind(); + + gl::Uniform1i(self.uniform_point_cloud_texture_to_use.id() as GLint, i as GLint); + + gl::DrawArrays(gl::POINTS, 0, size); + } + + gl::BindVertexArray(0); + + // Now tell GLSL we're doing objects + gl::Uniform1i(self.uniform_vertex_type.id() as GLint, 1); + + for positionable in self.renderable_objects.iter() { + positionable.draw(self.model_transform_id); + } + + gl_get_error() + .map_err(|err| PointCloudRendererError::OpenGlError(err)) + } + } + + pub fn set_point_size(&mut self, point_size: u8) { + self.point_size = point_size; + } + + pub fn set_enable_shading(&mut self, enable_shading: bool) { + self.enable_shading = enable_shading; + } +} + +// TODO: This is useful to write solid colors instead of webcam color data +// std::ptr::copy::(color_src, vertex_mapped_buffer as *mut u8, color_image_size_bytes as usize); +// std::ptr::write_bytes(vertex_mapped_buffer, 255, color_image_size_bytes as usize); + diff --git a/src/point_cloud/point_cloud_visualiser.rs b/src/point_cloud/point_cloud_visualiser.rs new file mode 100644 index 000000000..039736faa --- /dev/null +++ b/src/point_cloud/point_cloud_visualiser.rs @@ -0,0 +1,577 @@ +//! Based off Microsoft's MIT-licensed k4a library code and examples, specifically +//! `k4apointcloudvisualizer.h` + +use std::fmt::Formatter; +use std::mem::size_of; +use std::sync::{Arc, Mutex}; + +use gl::types::*; + +use anyhow::Result as AnyhowResult; +use crate::azure_kinect::sensor_control::CaptureProvider; +use crate::core_types::RgbaF32; +use crate::gui::mouse_camera_arcball::MouseCameraArcball; +use crate::opengl::wrapper::other_misc_wrapper::OpenGlError; +use crate::opengl::wrapper::other_misc_wrapper::{Framebuffer, Renderbuffer}; +use crate::opengl::wrapper::texture::Texture; +use crate::point_cloud::debug::capture_proxy::CaptureProxy; +use crate::point_cloud::debug::image_proxy::ImageProxy; +use crate::point_cloud::gpu_point_cloud_converter::{GpuPointCloudConverter, PointCloudComputeError}; +use crate::point_cloud::pixel_structs::BgraPixel; +use crate::point_cloud::pixel_structs::DepthPixel; +use crate::point_cloud::point_cloud_renderer::{PointCloudRenderer, PointCloudRendererError}; +use crate::point_cloud::util::{colorize_depth_blue_to_red, get_depth_mode_range, ValueRange}; +use crate::point_cloud::viewer_image::ViewerImage; +use k4a_sys_temp as k4a_sys; +use kinect::{Image, Calibration, ImageFormat, Capture, Transformation}; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug)] +pub enum PointCloudVisualizerError { + OpenGlError(OpenGlError), + PointCloudRendererError(PointCloudRendererError), + PointCloudComputeError(PointCloudComputeError), + FramebufferError, + DepthToColorConversionFailed, + /// Capture is missing depth info; drop the frame + MissingDepthImage, + /// Capture is missing color info; drop the frame + MissingColorImage, + // TODO: Implement this + ColorSupportNotYetImplemented, + UnsupportedMode, + UnknownError, +} + +impl From for PointCloudVisualizerError { + fn from(error: OpenGlError) -> Self { + PointCloudVisualizerError::OpenGlError(error) + } +} + +impl From for PointCloudVisualizerError { + fn from(error: PointCloudComputeError) -> Self { + PointCloudVisualizerError::PointCloudComputeError(error) + } +} + +impl std::fmt::Display for PointCloudVisualizerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let description = match self { + PointCloudVisualizerError::OpenGlError(inner) => { + format!("Visualizer OpenGL error: {}", inner) + }, + PointCloudVisualizerError::PointCloudRendererError(inner) => { + format!("Visualizer PointCloudRenderer error: {}", inner) + }, + PointCloudVisualizerError::PointCloudComputeError(inner) => { + format!("Visualizer PointCloudCompute error: {}", inner) + }, + PointCloudVisualizerError::FramebufferError => "Visualizer Framebuffer Error".into(), + PointCloudVisualizerError::DepthToColorConversionFailed => "Visualizer Depth To Color Conversion Error".into(), + PointCloudVisualizerError::UnsupportedMode => "Visualizer Unsupported Mode Error".into(), + PointCloudVisualizerError::MissingDepthImage => "Visualizer Missing Depth Image (drop frame)".into(), + PointCloudVisualizerError::MissingColorImage => "Visualizer Missing Color Image (drop frame)".into(), + PointCloudVisualizerError::ColorSupportNotYetImplemented => "Visualizer Color Support Not Yet Implemented (TODO)".into(), + PointCloudVisualizerError::UnknownError => "Visualizer Unknown Error".into(), + }; + + write!(f, "{}", description) + } +} + +impl std::error::Error for PointCloudVisualizerError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum ColorizationStrategy { + Simple, + Shaded, + Color, +} + +pub struct PointCloudVisualizer { + arcball_camera: Arc>, + + num_cameras: usize, + + m_dimensions_width : i32, + m_dimensions_height : i32, + + enable_color_point_cloud: bool, + colorization_strategy: ColorizationStrategy, + + pub point_cloud_renderer: PointCloudRenderer, + pub point_cloud_converters: Vec, + + /// Calibration data for all cameras + all_device_calibrations: Vec, + + /// Transformations are constructed from camera calibrations + transformations: Vec, + + /// Near and far minima/maxima for the current depth sensor mode. + /// These are constructed for each camera from the camera calibrations. + expected_depth_value_ranges: Vec, + + frame_buffer: Framebuffer, + depth_buffer: Renderbuffer, + + last_captures: Vec>, + + /// Buffer that holds the depth image transformed to the color coordinate space. + /// Used in color mode only. + transformed_depth_images: Vec>, + + /// In color mode, this is just a shallow copy of the latest color image. + /// In depth mode, this is a buffer that holds the colorization of the depth image. + point_cloud_colorizations: Vec>, + + /// Holds the XYZ point cloud as a texture. + /// Format is XYZA, where A (the alpha channel) is unused. + pub xyz_textures: Vec, + + color_xy_tables: Vec, + depth_xy_tables: Vec, + + /// Background clear color for OpenGL. + clear_color: RgbaF32, + + /// For debugging + /// If present, use these instead of camera captures. + debug_static_color_frames: Vec, + debug_static_depth_frames: Vec, +} + +// TODO: Dedup +struct CleanupGuard {} + +impl Drop for CleanupGuard { + fn drop(&mut self) { + unsafe { + gl::BindFramebuffer(gl::FRAMEBUFFER, 0); + } + } +} + +impl PointCloudVisualizer { + + /// + /// CTOR + /// + pub fn new(num_cameras: usize, + all_device_calibrations: &Vec, + enable_color_point_cloud: bool, + initial_colorization_strategy: ColorizationStrategy, + clear_color: RgbaF32, + arcball_camera: Arc>) -> Self + { + // Resolution of the point cloud texture + // constexpr ImageDimensions PointCloudVisualizerTextureDimensions = { 1280, 1152 }; + let width = 1280; + let height= 1152; + + let depth_buffer = Renderbuffer::new_initialized(); + + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, depth_buffer.id()); + gl::RenderbufferStorage(gl::RENDERBUFFER, gl::DEPTH_COMPONENT, width, height); + } + + let mut expected_depth_value_ranges = Vec::new(); + let mut transformations = Vec::new(); + + for calibration in all_device_calibrations { + let value_range = get_depth_mode_range(calibration.0.depth_mode) + .expect("Should be in correct depth sensor mode."); + expected_depth_value_ranges.push(value_range); + + let transformation = Transformation::from_calibration(&calibration); + transformations.push(transformation); + } + + let mut point_cloud_converters = Vec::with_capacity(num_cameras); + let mut transformed_depth_images = Vec::with_capacity(num_cameras); + let mut point_cloud_colorizations = Vec::with_capacity(num_cameras); + let mut xyz_textures = Vec::with_capacity(num_cameras); + let mut last_captures = Vec::with_capacity(num_cameras); + let mut color_xy_tables = Vec::with_capacity(num_cameras); + let mut depth_xy_tables = Vec::with_capacity(num_cameras); + + for i in 0 .. num_cameras { + point_cloud_converters.push(GpuPointCloudConverter::new()); + transformed_depth_images.push(None); + point_cloud_colorizations.push(None); + last_captures.push(None); + xyz_textures.push(Texture::new()); + + let calibration = all_device_calibrations.get(i).unwrap(); + + // TODO: generate color xytable only on `enable_color_point_cloud`. + let color_xy_table = GpuPointCloudConverter::generate_xy_table( + &calibration, + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_COLOR, + ).unwrap(); + + let depth_xy_table = GpuPointCloudConverter::generate_xy_table( + &calibration, + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_DEPTH, + ).unwrap(); + + color_xy_tables.push(ImageProxy::consume_k4a_image(color_xy_table)); + depth_xy_tables.push(ImageProxy::consume_k4a_image(depth_xy_table)); + } + + let mut debug_static_color_frames = Vec::new(); + //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_0", 0, 0, 0, ImageFormat::ColorBgra32).unwrap()); + //debug_static_color_frames.push(ImageProxy::from_file("output/color_src_1", 0, 0, 0, ImageFormat::ColorBgra32).unwrap()); + + let mut debug_static_depth_frames = Vec::new(); + //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_0", 3840, 2160, 7680, ImageFormat::Depth16).unwrap()); + //debug_static_depth_frames.push(ImageProxy::from_file("output/depth_src_1", 3840, 2160, 7680, ImageFormat::Depth16).unwrap()); + + let mut visualizer = Self { + num_cameras, + arcball_camera: arcball_camera.clone(), + m_dimensions_width: width, + m_dimensions_height: height, + enable_color_point_cloud, + point_cloud_renderer: PointCloudRenderer::new(num_cameras, arcball_camera.clone()).expect("should build"), + point_cloud_converters, + frame_buffer: Framebuffer::new_initialized(), + depth_buffer, + color_xy_tables, + depth_xy_tables, + colorization_strategy: initial_colorization_strategy, + transformations, + last_captures, + transformed_depth_images, + point_cloud_colorizations, + xyz_textures, + expected_depth_value_ranges, + clear_color, + debug_static_color_frames, + debug_static_depth_frames, + all_device_calibrations: all_device_calibrations.clone(), + }; + + visualizer.set_colorization_strategy(initial_colorization_strategy).expect("Should work"); + + visualizer + } + + /// Setup initial rendering + /// + pub fn setup_rendering(&mut self) -> AnyhowResult<()> { + self.point_cloud_renderer.setup_rendering() + } + + /// + /// + /// + pub fn set_point_size(&mut self, point_size: u8) { + self.point_cloud_renderer.set_point_size(point_size); + } + + /// + /// + /// + pub fn update_texture(&mut self, texture: &ViewerImage, mut captures: Vec) -> Result<()> { + self.update_texture_id(texture.texture_id(), captures) + } + + /// + /// Take the latest capture, calculate updates to the xyz texture, then render the point cloud to + /// the output texture. + /// + pub fn update_texture_id(&mut self, texture_id: GLuint, mut captures: Vec) -> Result<()> { + // Update the point cloud renderer with the latest point data + self.update_point_clouds(captures)?; + + // Set up rendering to a texture + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, self.depth_buffer.id()); + gl::BindFramebuffer(gl::FRAMEBUFFER, self.frame_buffer.id()); + } + + let _cleanup_guard = CleanupGuard {}; + + unsafe { + gl::FramebufferRenderbuffer( + gl::FRAMEBUFFER, + gl::DEPTH_ATTACHMENT, + gl::RENDERBUFFER, + self.depth_buffer.id() + ); + + gl::FramebufferTexture(gl::FRAMEBUFFER, gl::COLOR_ATTACHMENT0, texture_id, 0); + + // DrawBuffers writes to a COLOR BUFFER + // COLOR_ATTACHMENT{N} - The fragment shader output value is written into the nth color + // attachment of the current framebuffer. + gl::DrawBuffers(1, &gl::COLOR_ATTACHMENT0); + + let frame_buffer_status = gl::CheckFramebufferStatus(gl::FRAMEBUFFER); + + if frame_buffer_status != gl::FRAMEBUFFER_COMPLETE { + return Err(PointCloudVisualizerError::FramebufferError); + } + + gl::Viewport(0, 0, self.m_dimensions_width, self.m_dimensions_height); + gl::Enable(gl::DEPTH_TEST); + gl::ClearColor( + self.clear_color.red, + self.clear_color.green, + self.clear_color.blue, + self.clear_color.alpha, + ); + gl::ClearDepth(1.0); + + gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT); + } + + match self.arcball_camera.lock() { + Ok(arcball) => { + let view: [[f32; 4]; 4] = (arcball.get_view_matrix()).into(); + let perspective: [[f32; 4]; 4] = (arcball.get_perspective_matrix()).into(); + self.point_cloud_renderer.update_view_projection(view, perspective); + }, + Err(_) => {}, + } + + let render_status = self.point_cloud_renderer.render(); + + unsafe { + gl::BindRenderbuffer(gl::RENDERBUFFER, 0); + } + + render_status + .map_err(|err| PointCloudVisualizerError::PointCloudRendererError(err)) + } + + /// + /// + /// + fn update_point_clouds(&mut self, mut captures: Vec) -> Result<()> { + if captures.len() != self.num_cameras { + println!("Length of captures in insufficient (unplug the cameras from USB and retry): {}", captures.len()); + return Ok(()); + } + + for (camera_index, capture) in captures.drain(0 .. self.num_cameras) + .into_iter() + .enumerate() { + self.update_point_clouds_for_camera(camera_index, capture)?; + } + + let mut color_captures = Vec::new(); + + let color_frames = if self.debug_static_color_frames.len() > 0 { + &self.debug_static_color_frames + } else { + // TODO: wrt k4a::Image.clone(), k4a should use ref counting. (I did this one other place a few commits ago. + // Hope it's not ref leaking.) + let colorizations: Vec = self.point_cloud_colorizations.iter() + .map(|img| img.as_ref().map(|img| img.clone()).unwrap()) // Potential ref leak #1 + .collect(); + + color_captures = colorizations; + &color_captures + }; + + self.point_cloud_renderer.update_point_clouds( + &color_frames, + &self.xyz_textures + ).map_err(|err| PointCloudVisualizerError::PointCloudRendererError(err)) + } + + fn update_point_clouds_for_camera(&mut self, camera_index: usize, capture: CaptureProxy) -> Result<()> { + // + // Depth Image extractor + // + + let transformation = self.transformations.get(camera_index).unwrap(); + + let mut depth_image = match capture.get_depth_image() { + Ok(img) => img, + Err(_e) => { + // Capture doesn't have depth info. Drop the capture. + return Err(PointCloudVisualizerError::MissingDepthImage); + }, + }; + + let maybe_color_image = capture.get_color_image(); + + if self.enable_color_point_cloud { + if maybe_color_image.is_err() { + return Err(PointCloudVisualizerError::MissingColorImage); + } + if self.colorization_strategy == ColorizationStrategy::Color { + // TODO: TEMP SUPPORT MULTI-CAMERA + if let Some(transformed_depth_image) = self.transformed_depth_images.get_mut(camera_index) + .unwrap() + .as_mut() { + unsafe { + + let result = k4a_sys::k4a_transformation_depth_image_to_color_camera( + transformation.get_handle(), + depth_image.get_handle(), + transformed_depth_image.get_handle(), + ); + + if result != k4a_sys::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(PointCloudVisualizerError::DepthToColorConversionFailed); + } + + depth_image = transformed_depth_image; + } + } + } + } + + // + // Convert depth image to depth texture + // + + let use_depth_image = self.debug_static_depth_frames.get(camera_index) + .unwrap_or(&depth_image); + + let result = self.point_cloud_converters.get(camera_index).unwrap().convert( + &use_depth_image, + &mut self.xyz_textures.get_mut(camera_index).unwrap(), // TODO: TEMP MULTI CAMERA SUPPORT + camera_index + ); + + if let Err(err) = result { + return Err(PointCloudVisualizerError::PointCloudComputeError(err)); + } + + // TODO: TEMP MULTI CAMERA SUPPORT + if let Some(mut inner) = self.last_captures.get_mut(camera_index) { + *inner = Some(capture.clone()); + } + + if self.colorization_strategy == ColorizationStrategy::Color { + let color_image = maybe_color_image.expect("logic above should ensure present"); + // TODO: TEMP MULTI CAMERA SUPPORT + if let Some(mut inner) = self.point_cloud_colorizations.get_mut(camera_index) { + *inner = Some(color_image.clone()); + } + + } else { + // This creates a color spectrum based on depth. + let dst_length = use_depth_image.get_width_pixels() * use_depth_image.get_height_pixels(); + + let depth_value_range = self.expected_depth_value_ranges.get(camera_index).unwrap(); + + unsafe { + // src: DepthPixel + let src_pixel_buffer = use_depth_image.get_buffer(); + let typed_src_pixel_buffer = src_pixel_buffer as *const DepthPixel; + + // TODO TEMP MULTI-CAMERA SUPPORT + // dst: BgraPixel + let dst_pixel_buffer = self.point_cloud_colorizations.get_mut(camera_index) + .unwrap() + .as_ref() + .expect("point cloud color image must be set") + .get_buffer(); + + let typed_dst_pixel_buffer = dst_pixel_buffer as *mut BgraPixel; + + for i in 0 .. dst_length as isize { + let src_pixel = *typed_src_pixel_buffer.offset(i); + let pixel = colorize_depth_blue_to_red(src_pixel, depth_value_range); + (*typed_dst_pixel_buffer.offset(i)).red = pixel.red; + (*typed_dst_pixel_buffer.offset(i)).green = pixel.green; + (*typed_dst_pixel_buffer.offset(i)).blue = pixel.blue; + (*typed_dst_pixel_buffer.offset(i)).alpha = pixel.alpha; + } + } + } + + Ok(()) + } + + /// + /// + /// + pub fn set_colorization_strategy(&mut self, strategy: ColorizationStrategy) -> Result<()> { + for i in 0 .. self.num_cameras { + self.set_colorization_strategy_for_camera(i, strategy)?; + } + Ok(()) + } + + fn set_colorization_strategy_for_camera(&mut self, camera_index: usize, strategy: ColorizationStrategy) -> Result<()> { + if strategy == ColorizationStrategy::Color && !self.enable_color_point_cloud { + return Err(PointCloudVisualizerError::UnsupportedMode); + } + + let calibration = self.all_device_calibrations.get(camera_index).unwrap(); + + self.colorization_strategy = strategy; + self.point_cloud_renderer.set_enable_shading(strategy == ColorizationStrategy::Shaded); + + if strategy == ColorizationStrategy::Color { + let width = calibration.0.color_camera_calibration.resolution_width; + let height = calibration.0.color_camera_calibration.resolution_height; + let stride = width * size_of::() as i32; + + // TODO: TEMP MULTI-CAMERA SUPPORT + if let Some(mut transformed_depth_image) = self.transformed_depth_images.get_mut(camera_index) { + *transformed_depth_image = Some(ImageProxy::consume_k4a_image(Image::create( + ImageFormat::Depth16, + width as u32, + height as u32, + stride as u32, + ).expect("Construction should work FIXME"))); + } + + for point_cloud_converter in self.point_cloud_converters.iter_mut() { + point_cloud_converter.set_active_xy_table(&self.color_xy_tables.get(camera_index).unwrap())?; // TODO: TEMP MULTI-CAMERA SUPPORT + } + + } else { + let width = calibration.0.depth_camera_calibration.resolution_width as u32; + let height = calibration.0.depth_camera_calibration.resolution_height as u32; + let stride = width as i32 * size_of::() as i32; + + // TODO: TEMP MULTI-CAMERA SUPPORT + if let Some(mut point_cloud_colorization) = self.point_cloud_colorizations.get_mut(camera_index) { + *point_cloud_colorization = Some(ImageProxy::consume_k4a_image(Image::create( + ImageFormat::ColorBgra32, + width, + height, + stride as u32, + ).expect("Construction should work FIXME"))); + } + + for point_cloud_converter in self.point_cloud_converters.iter_mut() { + let xy_table = self.depth_xy_tables.get(camera_index).unwrap(); + point_cloud_converter.set_active_xy_table(&xy_table)?; + } + } + + self.xyz_textures.get_mut(0).unwrap().reset(); // TODO: TEMP MULTI-CAMERA SUPPORT + + // TODO: TEMP MULTI-CAMERA SUPPORT + if let Some(capture) = self.last_captures.get_mut(camera_index) + .unwrap() + .as_ref() + { + let capture = (*capture).clone(); + // TODO: TEMP MULTI-CAMERA SUPPORT + let mut captures = Vec::new(); + captures.push(capture); + self.update_point_clouds(captures); + } + + Ok(()) + } +} diff --git a/src/point_cloud/shaders/point_cloud_compute_shader.glsl b/src/point_cloud/shaders/point_cloud_compute_shader.glsl new file mode 100644 index 000000000..22e9d6a38 --- /dev/null +++ b/src/point_cloud/shaders/point_cloud_compute_shader.glsl @@ -0,0 +1,53 @@ +// This is taken from Microsoft's MIT-licensed k4a libraries. +// From the file `tools/k4aviewer/graphics/shaders/gpudepthtopointcloudconverter.cpp` +#version 430 + +layout(location=0, rgba32f) writeonly uniform image2D destTex; + +layout(location=1, r16ui) readonly uniform uimage2D depthImage; +layout(location=2, rg32f) readonly uniform image2D xyTable; + +layout(local_size_x = 1, local_size_y = 1) in; + +void main() +{ + ivec2 pixel = ivec2(gl_GlobalInvocationID.xy); + + float vertexValue = float(imageLoad(depthImage, pixel)); + // TODO: The problem is that these are bound to (0.0, 0.0). It isn't working. + vec2 xyValue = imageLoad(xyTable, pixel).xy; + + float alpha = 1.0f; + // vertexPosition format: RGB colors as floats, but not range [0.0, 1.0], but rather [0.0, 255.0] + // HOWEVER, there are operations that reverse the sign of the 'x' channel ('r') and scale down by 1000 below. + vec3 vertexPosition = vec3(vertexValue * xyValue.x, vertexValue * xyValue.y, vertexValue); + + // Invalid pixels have their XY table values set to 0. + // Set the rest of their values to 0 so clients can pick them out. + // + if (xyValue.x == 0.0f && xyValue.y == 0.0f) + { + alpha = 0.0f; + vertexValue = 0.0f; + } + + // Vertex positions are in millimeters, but everything else is in meters, so we need to convert + // + vertexPosition /= 1000.0f; + + // OpenGL and K4A have different conventions on which direction is positive - + // we need to flip the X coordinate. + // + vertexPosition.x *= -1; + + // Distance Threshold + // TODO: Make configurable. + /*if (vertexPosition.z > 1.5) { + alpha = 0.0f; + vertexPosition.x = 0.0; + vertexPosition.y = 0.0; + vertexPosition.z = 0.0; + }*/ + + imageStore(destTex, pixel, vec4(vertexPosition, alpha)); +} diff --git a/src/point_cloud/shaders/point_cloud_fragment_shader.glsl b/src/point_cloud/shaders/point_cloud_fragment_shader.glsl new file mode 100644 index 000000000..01d972c84 --- /dev/null +++ b/src/point_cloud/shaders/point_cloud_fragment_shader.glsl @@ -0,0 +1,39 @@ +// This is taken from Microsoft's MIT-licensed k4a libraries. +// From the file `tools/k4aviewer/graphics/shaders/k4apointcloudshaders.h` +#version 430 + +in vec4 vertexColor; +out vec4 fragmentColor; + +in vec2 texCoord; +uniform bool enableShading; + +uniform int vertexType; + +uniform sampler2D objTexture; // C++ method +//layout(location=2, rgba32f) readonly uniform image2D objTexture; // k4a method + +void main() +{ + if (vertexColor.a == 0.0f) + { + discard; + } + + if (vertexType == 1) + { + // Object mode + vec2 coord = texCoord; + fragmentColor = texture2D(objTexture, coord); + } + else if (vertexType == 2) + { + // Point cloud mode + fragmentColor = vertexColor; + } + + if (fragmentColor.a == 0.0f) + { + discard; + } +} diff --git a/src/point_cloud/shaders/point_cloud_vertex_shader.glsl b/src/point_cloud/shaders/point_cloud_vertex_shader.glsl new file mode 100644 index 000000000..dfc080bc5 --- /dev/null +++ b/src/point_cloud/shaders/point_cloud_vertex_shader.glsl @@ -0,0 +1,212 @@ +// This is taken from Microsoft's MIT-licensed k4a libraries. +// From the file `tools/k4aviewer/graphics/shaders/k4apointcloudshaders.h` +#version 430 + +// NB: It appears that locations need to be wide enough apart, or the colors bleed across locations. +layout(location=0) in vec4 inColor; +//layout(location=10) in vec4 inColor1; // TODO: Get rid of this! + +// For use with objects. +// Might restructure the entire program. +in vec3 position; +in vec3 normal; + +in vec2 vTextureCoord; +out vec2 texCoord; + +out vec4 vertexColor; + +// Transformation matrices +uniform mat4 model; +uniform mat4 view; +uniform mat4 projection; + +// 1 = object, 2 = pointCloud +uniform int vertexType; + +// NB: This is not a scalable approach. But it might work for now. +// 0 = camera0, 1 = camera1 +uniform int pointCloudTextureToUse; + +layout(rgba32f, binding=0) readonly uniform image2D pointCloudTexture0; +layout(rgba32f, binding=1) readonly uniform image2D pointCloudTexture1; + +uniform bool enableShading; + +// bool GetPoint3d(in vec2 pointCloudSize, in ivec2 point2d, out vec3 point3d) +// { +// if (point2d.x < 0 || point2d.x >= pointCloudSize.x || +// point2d.y < 0 || point3d.y >= pointCloudSize.y) +// { +// return false; +// } +// +// point3d = imageLoad(pointCloudTexture0, point2d).xyz; +// if (point3d.z <= 0) +// { +// return false; +// } +// +// return true; +// } + +void main() +{ + ivec2 pointCloudSize0 = imageSize(pointCloudTexture0); + int pointCloudLength0 = pointCloudSize0.x * pointCloudSize0.y; + + vec3 vertexPosition; + vec4 colorOut; + + // We're having to multiplex on gl_VertexID, and I don't think we get double the range for two + // cameras. This effectively 'downsamples' each camera. We need to find a way to double this. + // I hate drawing like this + //if (gl_VertexID < pointCloudLength0) { // TODO - this is the one to enable + if (gl_VertexID < 10) { + // Camera #0 + ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize0.x, gl_VertexID / pointCloudSize0.x); + + vertexPosition = imageLoad(pointCloudTexture0, currentDepthPixelCoordinates).xyz; + + } else { + // Camera #1 + ivec2 pointCloudSize1 = imageSize(pointCloudTexture1); + int pointCloudLength1 = pointCloudSize1.x * pointCloudSize1.y; + + ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize1.x, gl_VertexID / pointCloudSize1.x); + + // // Let's move the model away a bit. + // vec3 originalPosition = imageLoad(pointCloudTexture1, currentDepthPixelCoordinates).xyz; + // vertexPosition = vec3( + // originalPosition.x - 1.5, + // originalPosition.y, + // originalPosition.z + // ); + + vertexPosition = imageLoad(pointCloudTexture1, currentDepthPixelCoordinates).xyz; + + vertexPosition = position; + } + + if (view[0][0] > 0.5) { + vertexPosition.x -= 2.0; + vertexPosition.y -= 2.0; + } + + vertexColor = colorOut; + texCoord = vTextureCoord; + + if (vertexType == 1) { + // Object mode + + vertexColor = colorOut; + texCoord = vTextureCoord; + + } else if (vertexType == 2) { + // Point cloud mode + + if (pointCloudTextureToUse == 0) { + ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize0.x, gl_VertexID / pointCloudSize0.x); + + vertexPosition = imageLoad(pointCloudTexture0, currentDepthPixelCoordinates).xyz; + + vertexPosition.x += 1.0; // Camera 0 looks best on the left + + colorOut = vec4( + inColor.r, + inColor.g, + inColor.b, + 255 + ); + + } else if (pointCloudTextureToUse == 1) { + ivec2 pointCloudSize1 = imageSize(pointCloudTexture1); + int pointCloudLength1 = pointCloudSize1.x * pointCloudSize1.y; + + ivec2 currentDepthPixelCoordinates = ivec2(gl_VertexID % pointCloudSize1.x, gl_VertexID / pointCloudSize1.x); + + vertexPosition = imageLoad(pointCloudTexture1, currentDepthPixelCoordinates).xyz; + + vertexPosition.x -= 1.0; // Camera 1 looks best on the right + + colorOut = vec4( + inColor.r, + inColor.g, + inColor.b, + 255 + ); + } + + if (vertexPosition.z > 1.5) { + // Depth threshold + // This leads directly to pixel discard + colorOut.a = 0.0; + } + + vertexColor = colorOut; + } + + gl_Position = projection * view * model * vec4(vertexPosition, 1); + + // Pass along the 'invalid pixel' flag as the alpha channel + // + // if (vertexPosition.z == 0.0f) + // { + // vertexColor.a = 0.0f; + // } + + // NB: This is affecting the second camera. Disabling for now. + // if (enableShading) + // { + // // Compute the location of the closest neighbor pixel to compute lighting + // // + // vec3 closestNeighbor = vertexPosition; + // + // // If no neighbors have data, default to 1 meter behind point. + // // + // closestNeighbor.z += 1.0f; + // + // vec3 outPoint; + // if (GetPoint3d(pointCloudSize, currentDepthPixelCoordinates - ivec2(1, 0), outPoint)) + // { + // if (closestNeighbor.z > outPoint.z) + // { + // closestNeighbor = outPoint; + // } + // } + // if (GetPoint3d(pointCloudSize, currentDepthPixelCoordinates + ivec2(1, 0), outPoint)) + // { + // if (closestNeighbor.z > outPoint.z) + // { + // closestNeighbor = outPoint; + // } + // } + // if (GetPoint3d(pointCloudSize, currentDepthPixelCoordinates - ivec2(0, 1), outPoint)) + // { + // if (closestNeighbor.z > outPoint.z) + // { + // closestNeighbor = outPoint; + // } + // } + // if (GetPoint3d(pointCloudSize, currentDepthPixelCoordinates + ivec2(0, 1), outPoint)) + // { + // if (closestNeighbor.z > outPoint.z) + // { + // closestNeighbor = outPoint; + // } + // } + // + // vec3 lightPosition = vec3(0, 0, 0); + // float occlusion = length(vertexPosition - closestNeighbor) * 20.0f; + // float diffuse = 1.0f - clamp(occlusion, 0.0f, 0.6f); + // + // float distance = length(lightPosition - vertexPosition); + // + // // Attenuation term for light source that covers distance up to 50 meters + // // http://wiki.ogre3d.org/tiki-index.php?page=-Point+Light+Attenuation + // // + // float attenuation = 1.0 / (1.0 + 0.09 * distance + 0.032 * distance * distance); + // + // vertexColor = vec4(attenuation * diffuse * vertexColor.rgb, vertexColor.a); + // } +} diff --git a/src/point_cloud/shaders/simple_vertex_shader.glsl b/src/point_cloud/shaders/simple_vertex_shader.glsl new file mode 100644 index 000000000..3c0928cc1 --- /dev/null +++ b/src/point_cloud/shaders/simple_vertex_shader.glsl @@ -0,0 +1,23 @@ +//#version 140 +#version 420 + +uniform mat4 view; +uniform mat4 projection; +in vec4 inColor; // UNUSED + +layout(rgba32f, binding=0) readonly uniform image2D pointCloudTexture0; // UNUSED +layout(rgba32f, binding=1) readonly uniform image2D pointCloudTexture1; // UNUSED + +in vec3 position; +in vec3 normal; + +uniform bool enableShading; // UNUSED + +out vec3 v_position; +//out vec3 v_normal; + +void main() { + v_position = position; + //v_normal = normal; + gl_Position = projection * view * vec4(v_position * 0.005, 1.0); +} \ No newline at end of file diff --git a/src/point_cloud/util.rs b/src/point_cloud/util.rs new file mode 100644 index 000000000..bd21ccc5e --- /dev/null +++ b/src/point_cloud/util.rs @@ -0,0 +1,69 @@ +use crate::point_cloud::pixel_structs::{DepthPixel, BgraPixel}; +use imgui::sys::igColorConvertHSVtoRGB; +use k4a_sys_temp as k4a_sys; + +/// Ranges of values we can expect from the depth sensor, depending on its current configuration. +pub struct ValueRange { + /// Lowest (nearest) value reported from the depth sensor in millimeters. + pub near: DepthPixel, + /// Highest (furthest) value reported from the depth sensor in millimeters. + pub far: DepthPixel, +} + +/// Gets the range of values that we expect to see from the depth camera +/// when using a given depth mode, in millimeters +pub fn get_depth_mode_range(depth_mode: k4a_sys::k4a_depth_mode_t) -> Option { + match depth_mode { + k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_2X2BINNED => Some(ValueRange { near: 500, far: 5800 }), + k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_NFOV_UNBINNED => Some(ValueRange { near: 500, far: 4000 }), + k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_2X2BINNED => Some(ValueRange { near: 250, far: 3000 }), + k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_WFOV_UNBINNED => Some(ValueRange { near: 250, far: 2500 }), + k4a_sys::k4a_depth_mode_t_K4A_DEPTH_MODE_PASSIVE_IR => None, // TODO: Return error + _ => None, // TODO: Return error + } +} + +/// This is a port of Microsoft's `ColorizeBlueToRed` in the open source / MIT-licensed libk4a +pub fn colorize_depth_blue_to_red(depth_pixel: DepthPixel, value_range: &ValueRange) -> BgraPixel { + let pixel_max = libc::uint8_t::max_value(); + + let mut result = BgraPixel { blue: 0, green: 0, red: 0, alpha: pixel_max }; + + if depth_pixel == 0 { + // If the pixel is actually zero and not just below the min value, make it black. + return result; + } + + let clamped_value = depth_pixel + .min(value_range.far) + .max(value_range.near); + + // Normalize to [0, 1] + let mut hue = (clamped_value - value_range.near) as f32 / (value_range.far - value_range.near) as f32; + + // The 'hue' coordinate in HSV is a polar coordinate, so it 'wraps'. + // Purple starts after blue and is close enough to red to be a bit unclear, + // so we want to go from blue to red. Purple starts around .6666667, + // so we want to normalize to [0, .6666667]. + let range = 2.0f32 / 3.0; + hue *= range; + + // We want blue to be close and red to be far, so we need to reflect the + // hue across the middle of the range. + hue = range - hue; + + let mut red = 0.0f32; + let mut green= 0.0f32; + let mut blue = 0.0f32; + + unsafe { + igColorConvertHSVtoRGB(hue, 1.0, 1.0, &mut red, &mut green, &mut blue); + } + + let pixel_max = pixel_max as f32; + result.red = (red * pixel_max) as u8; + result.green = (green * pixel_max) as u8; + result.blue = (blue * pixel_max) as u8; + + result +} \ No newline at end of file diff --git a/src/point_cloud/viewer_image.rs b/src/point_cloud/viewer_image.rs new file mode 100644 index 000000000..3f9b491eb --- /dev/null +++ b/src/point_cloud/viewer_image.rs @@ -0,0 +1,199 @@ +use std::fmt::Formatter; +use std::ptr::null; + +use gl; +use gl::types::*; +use libc::uint8_t; + +use crate::opengl::wrapper::buffer::Buffer; +use crate::opengl::wrapper::other_misc_wrapper::OpenGlError; +use crate::opengl::wrapper::other_misc_wrapper::gl_get_error; +use crate::opengl::wrapper::texture::Texture; +use crate::point_cloud::pixel_structs::BgraPixel; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug)] +pub enum ViewerImageError { + OpenGlError(OpenGlError), +} + +impl From for ViewerImageError { + fn from(error: OpenGlError) -> Self { + ViewerImageError::OpenGlError(error) + } +} + +impl std::fmt::Display for ViewerImageError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let description = match self { + ViewerImageError::OpenGlError(inner) => { + format!("Visualizer OpenGL error: {}", inner) + }, + }; + write!(f, "{}", description) + } +} + +impl std::error::Error for ViewerImageError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + // Generic error, no backtrace. + None + } +} + + +#[derive(Clone, Debug)] +pub struct ImageDimensions { + pub width: u32, + pub height: u32, +} + +#[derive(Clone, Debug)] +pub struct ViewerImage { + dimensions: ImageDimensions, + format: GLenum, + + texture: Texture, + texture_buffer: Buffer, + + texture_buffer_size: GLuint, +} + +// GLenum format = GL_BGRA (default), but overridden as gl::RGBA +// GLenum internalFormat = GL_RGBA8 (default) +impl ViewerImage { + /// private ctor + fn new( + dimensions: ImageDimensions, + format: GLenum + ) -> Result { + let texture_buffer_size = (dimensions.width * dimensions.height) as GLuint + * get_format_pixel_element_count(format); + + let texture = Texture::new_initialized(); + let texture_buffer = Buffer::new_initialized(); + + unsafe { + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, texture_buffer.id()); + gl::BufferData(gl::PIXEL_UNPACK_BUFFER, texture_buffer_size as isize, null(), gl::STREAM_DRAW); + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, 0); + } + + Ok(Self { + dimensions, + format, + texture_buffer_size, + texture, + texture_buffer, + }) + } + + pub fn create( + width: u32, + height: u32, + ) -> Result + { + let dimensions = ImageDimensions { width, height }; + let format = gl::RGBA; + let mut viewer_image = ViewerImage::new(dimensions, format)?; + + unsafe { + gl::BindTexture(gl::TEXTURE_2D, viewer_image.texture.id()); + + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::LINEAR as i32); + gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::LINEAR as i32); + gl::PixelStorei(gl::UNPACK_ROW_LENGTH, 0); + + gl::TexStorage2D( + gl::TEXTURE_2D, + 1, + gl::RGBA8, + width as i32, + height as i32, + ); + + viewer_image.update_texture(None)?; + } + + Ok(viewer_image) + } + + pub fn texture_id(&self) -> GLuint { + self.texture.id() + } + + pub unsafe fn update_texture(&mut self, data: Option<&[uint8_t]>) -> Result<()> { + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, self.texture_buffer.id()); + gl::BindTexture(gl::TEXTURE_2D, self.texture.id()); + + let _cleanup_guard = CleanupGuard {}; + + let buffer = gl::MapBufferRange( + gl::PIXEL_UNPACK_BUFFER, + 0, + self.texture_buffer_size as isize, + gl::MAP_WRITE_BIT | gl::MAP_INVALIDATE_BUFFER_BIT, + ) as *mut BgraPixel; // TODO: It's actually RGBA, not BGRA. + + if buffer as usize == 0 { + return Err(ViewerImageError::OpenGlError(gl_get_error().err().unwrap())); + } + + // TODO COPY DATA + match data { + Some(_) => { + unreachable!("TODO") + }, + None => { + unsafe { + *buffer = std::mem::zeroed(); + } + }, + } + + let result = gl::UnmapBuffer(gl::PIXEL_UNPACK_BUFFER); + + if result == 0 { + return Err(ViewerImageError::OpenGlError(gl_get_error().err().unwrap())); + } + + gl::TexSubImage2D( + gl::TEXTURE_2D, + 0, + 0, + 0, + self.dimensions.width as i32, + self.dimensions.height as i32, + self.format, + gl::UNSIGNED_BYTE, + null(), + ); + + gl_get_error() + .map_err(|err| ViewerImageError::OpenGlError(err)) + } +} + +pub fn get_format_pixel_element_count(format: GLenum) -> GLuint { + match format { + gl::RED => 1, + gl::RG => 2, + gl::RGB | gl::BGR => 3, + gl::RGBA | gl::BGRA => 4, + _ => unreachable!("bad get_format_pixel_element_count") // TODO: Ugh. + } +} + +// TODO: Dedup +struct CleanupGuard {} + +impl Drop for CleanupGuard { + fn drop(&mut self) { + println!("Running CleanupGuard"); + unsafe { + gl::BindBuffer(gl::PIXEL_UNPACK_BUFFER, 0); + } + } +} + diff --git a/src/webcam.rs b/src/webcam.rs new file mode 100644 index 000000000..8302a14bb --- /dev/null +++ b/src/webcam.rs @@ -0,0 +1,108 @@ +use crate::opengl::wrapper::other_misc_wrapper::{gl_get_error, OpenGlError}; +use gl::types::*; +use gl; +use std::ffi::c_void; +use std::fmt::Formatter; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::Path; +use std::{fmt, io}; + +/// WebcamWriter uses Webcamoid's akvcam kernel driver to stream output to a /dev/video* +/// file. Their wiki contains all of the setup instructions, and I'm checked in some example +/// configs into this repo. +/// +/// https://github.com/webcamoid/akvcam +pub struct WebcamWriter { + /// Internal byte buffer to pull from OpenGL texture and flush to file. + buffer: Vec, + buffer_size: usize, + /// File to flush raw RGB24 bytes to. + file: File, +} + +impl WebcamWriter { + + /// Create a WebcamWriter that outputs to the supplied file. + /// The file handle is created and owned by WebcamWriter. + pub fn open_file(filename: &str, width: usize, height: usize, bit_depth: usize) + -> Result + { + let path = Path::new(filename); + let file = OpenOptions::new() + .read(true) + .truncate(true) + .write(true) + .open(path)?; + + let buffer_size = width * height * bit_depth; + + info!("Webcam writer buffer size: {} (width: {}, height: {}, bit depth: {}", + buffer_size, width, height, bit_depth); + + let mut buffer = Vec::with_capacity(buffer_size); + buffer.resize(buffer_size, 0); + + Ok(Self { + buffer, + buffer_size, + file, + }) + } + + // TODO: Make it easy to update for new textures of different sizes (dynamic buffer resize) + /// Grab the current texture held by `texture_id` and immediately write it to file. + pub fn write_current_frame_to_file(&mut self, texture_id: GLuint) -> Result<(), WebcamError> { + let typed_buffer = self.buffer.as_mut_ptr() as *mut c_void; + + unsafe { + gl::BindTexture(gl::TEXTURE_2D, texture_id); + gl::GetnTexImage(gl::TEXTURE_2D, 0, gl::RGB, gl::UNSIGNED_BYTE, self.buffer_size as i32, typed_buffer); + + gl_get_error()?; + + gl::BindTexture(gl::TEXTURE_2D, 0); + } + + self.file.write_all(&self.buffer)?; + self.file.flush()?; + + Ok(()) + } +} + +/// Errors from this module. +#[derive(Debug)] +pub enum WebcamError { + IoError(io::Error), + OpenGlError(OpenGlError), +} + +impl fmt::Display for WebcamError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let explain = match self { + WebcamError::IoError(e) => format!("IoError: {:?}", e), + WebcamError::OpenGlError(e) => format!("OpenGL error: {:?}", e), + }; + write!(f, "{}", explain) + } +} + +impl std::error::Error for WebcamError { + fn source(&self) -> Option<&(dyn std::error::Error +'static)> { + None // Generic error, no backtrace. + } +} + +impl From for WebcamError { + fn from(error: io::Error) -> Self { + WebcamError::IoError(error) + } +} + +impl From for WebcamError { + fn from(error: OpenGlError) -> Self { + WebcamError::OpenGlError(error) + } +} + diff --git a/src/zeromq/color.rs b/src/zeromq/color.rs new file mode 100644 index 000000000..7c68ec626 --- /dev/null +++ b/src/zeromq/color.rs @@ -0,0 +1,104 @@ +use std::time::{UNIX_EPOCH, SystemTime}; +use rand::distributions::Uniform; +use rand::Rng; + +#[derive(Copy, Clone, Debug)] +pub enum Color { + Black, + White, + Red, + Green, + Blue, + Yellow, // Red + Green + Teal, // Green + Blue + Purple, // Red + Blue + Custom { r: u8, g: u8, b: u8, a: u8 }, +} + +impl Color { + pub fn get_time_based_color() -> Self { + let start = SystemTime::now(); + let timestamp = start + .duration_since(UNIX_EPOCH) + .expect("Time should work."); + + let seconds = timestamp.as_secs(); + + match seconds % 10 { + 0 | 1 => Color::Purple, //Color::Black, + 2 | 3 => Color::Blue, + 4 | 5 => Color::Red, + 6 | 7 => Color::Green, + 8 | 9 => Color::White, + _ => Color::White, + } + } + + pub fn get_random_rgb_color() -> Self { + let mut rng = rand::thread_rng(); + let range = Uniform::new_inclusive(0u8, 255u8); + + let r = rng.sample(&range); + let g = rng.sample(&range); + let b = rng.sample(&range); + + Color::Custom { r, g, b, a: 255 } + } + + pub fn get_u8_r(&self) -> u8 { + match &self { + Color::Black => 0, + Color::White => 255, + Color::Red => 255, + Color::Green => 0, + Color::Blue => 0, + Color::Yellow => 255, + Color::Teal => 0, + Color::Purple => 255, + Color::Custom {r, g, b, a} => *r, + } + } + + pub fn get_u8_g(&self) -> u8 { + match &self { + Color::Black => 0, + Color::White => 255, + Color::Red => 0, + Color::Green => 255, + Color::Blue => 0, + Color::Yellow => 255, + Color::Teal => 255, + Color::Purple => 0, + Color::Custom {r, g, b, a} => *g, + } + } + + pub fn get_u8_b(&self) -> u8 { + match &self { + Color::Black => 0, + Color::White => 255, + Color::Red => 0, + Color::Green => 0, + Color::Blue => 255, + Color::Yellow => 0, + Color::Teal => 255, + Color::Purple => 255, + Color::Custom {r, g, b, a} => *b, + } + } + + pub fn get_u8_a(&self) -> u8 { + match &self { + Color::Black => 255, + Color::White => 255, + Color::Red => 255, + Color::Green => 255, + Color::Blue => 255, + Color::Yellow => 255, + Color::Teal => 255, + Color::Purple => 255, + Color::Custom {r, g, b, a} => *a, + } + } +} + diff --git a/src/zeromq/cpu_calculate_point_cloud.rs b/src/zeromq/cpu_calculate_point_cloud.rs new file mode 100644 index 000000000..844cbe1c2 --- /dev/null +++ b/src/zeromq/cpu_calculate_point_cloud.rs @@ -0,0 +1,150 @@ +use anyhow::anyhow; +use cgmath::num_traits::Float; +use crate::AnyhowResult; +use crate::zeromq::color::Color; +use crate::zeromq::point::Point; +use k4a_sys_temp as k4a_sys; +use kinect::Image; +use kinect::ImageFormat; +use std::mem::size_of; + +pub struct PointCloudResult { + pub point_cloud_image: Image, + pub point_count: u32, +} + +/// This has been adapted from libk4a sources (MIT-licensed). +pub fn calculate_point_cloud(depth_image: &Image, xy_table_image: &Image) -> AnyhowResult { + let width = depth_image.get_width_pixels(); + let height = depth_image.get_height_pixels(); + + let point_cloud_image= Image::create( + ImageFormat::Custom, + width as u32, + height as u32, + width as u32 * size_of::() as u32, + )?; + + let point_cloud_data = point_cloud_image.get_buffer() as *mut k4a_sys::k4a_float3_t; + let xy_table_data = xy_table_image.get_buffer() as *mut k4a_sys::k4a_float2_t; + let depth_data = depth_image.get_buffer() as *mut u16; // uint16_t + + let depth_data_length = (width * height) as isize; + + let mut point_count = 0; + + for i in 0 .. depth_data_length { + unsafe { + // TODO: This is missing `isnan` checks. + // if (depth_data[i] != 0 && !isnan(xy_table_data[i].xy.x) && !isnan(xy_table_data[i].xy.y)) + if (*depth_data.offset(i)) != 0 { + (*point_cloud_data.offset(i)).xyz.x = (*xy_table_data.offset(i)).xy.x * ((*depth_data.offset(i)) as f32); + (*point_cloud_data.offset(i)).xyz.y = (*xy_table_data.offset(i)).xy.y * ((*depth_data.offset(i)) as f32); + (*point_cloud_data.offset(i)).xyz.z = (*depth_data.offset(i)) as f32; + point_count += 1; + } else { + (*point_cloud_data.offset(i)).xyz.x = f32::nan(); + (*point_cloud_data.offset(i)).xyz.y = f32::nan(); + (*point_cloud_data.offset(i)).xyz.z = f32::nan(); + } + } + } + + Ok(PointCloudResult { + point_cloud_image, + point_count, + }) +} + +// Directly return as a vector. +pub fn calculate_point_cloud2(depth_image: &Image, xy_table_image: &Image, color: Color) -> AnyhowResult> { + let width = depth_image.get_width_pixels(); + let height = depth_image.get_height_pixels(); + + { + let xy_width = xy_table_image.get_width_pixels(); + let xy_height = xy_table_image.get_height_pixels(); + if width != xy_width || height != xy_height { + return Err(anyhow!("Depth image ({}x{}) and XY table ({}x{}) dimensions are not equal!", + width, height, xy_width, xy_height)); + } + } + + let xy_table_data = xy_table_image.get_buffer() as *mut k4a_sys::k4a_float2_t; + let depth_data = depth_image.get_buffer() as *mut u16; // uint16_t + + let depth_data_length = (width * height) as isize; + + let mut points = Vec::new(); + + for i in 0 .. depth_data_length { + unsafe { + // TODO: This is missing `isnan` checks. + // if (depth_data[i] != 0 && !isnan(xy_table_data[i].xy.x) && !isnan(xy_table_data[i].xy.y)) + if (*depth_data.offset(i)) != 0 { + let x = (*xy_table_data.offset(i)).xy.x * ((*depth_data.offset(i)) as f32); + let y = (*xy_table_data.offset(i)).xy.y * ((*depth_data.offset(i)) as f32); + let z = (*depth_data.offset(i)) as f32; + points.push(Point::at(x, y, z, color)); + } + } + } + + Ok(points) +} + +#[repr(C, packed)] +#[derive(Copy, Clone)] +struct ColorCameraPoint { + pub b: u8, + pub g: u8, + pub r: u8, + pub a: u8, +} + +// Directly return as a vector. +pub fn calculate_point_cloud3(depth_image: &Image, xy_table_image: &Image, color_image: &Image) -> AnyhowResult> { + let width = depth_image.get_width_pixels(); + let height = depth_image.get_height_pixels(); + + { + let xy_width = xy_table_image.get_width_pixels(); + let xy_height = xy_table_image.get_height_pixels(); + if width != xy_width || height != xy_height { + return Err(anyhow!("Depth image ({}x{}) and XY table ({}x{}) dimensions are not equal!", + width, height, xy_width, xy_height)); + } + } + + let xy_table_data = xy_table_image.get_buffer() as *mut k4a_sys::k4a_float2_t; + let depth_data = depth_image.get_buffer() as *mut u16; // uint16_t + let color_data = color_image.get_buffer() as *mut ColorCameraPoint; + + let depth_data_length = (width * height) as isize; + + let mut points = Vec::new(); + + for i in 0 .. depth_data_length { + unsafe { + // TODO: This is missing `isnan` checks. + // if (depth_data[i] != 0 && !isnan(xy_table_data[i].xy.x) && !isnan(xy_table_data[i].xy.y)) + if (*depth_data.offset(i)) != 0 { + let x = (*xy_table_data.offset(i)).xy.x * ((*depth_data.offset(i)) as f32); + let y = (*xy_table_data.offset(i)).xy.y * ((*depth_data.offset(i)) as f32); + let z = (*depth_data.offset(i)) as f32; + let color = (*color_data.offset(i)) as ColorCameraPoint; + + let point_color = Color::Custom { + r: color.r, + b: color.b, + g: color.g, + a: 255 + }; + + points.push(Point::at(x, y, z, point_color)); + } + } + } + + Ok(points) +} diff --git a/src/zeromq/cpu_transformation.rs b/src/zeromq/cpu_transformation.rs new file mode 100644 index 000000000..b8eecefc0 --- /dev/null +++ b/src/zeromq/cpu_transformation.rs @@ -0,0 +1,72 @@ +use crate::AnyhowResult; +use anyhow::anyhow; +use k4a_sys_temp; +use kinect::{Transformation, Image, Calibration, ImageFormat}; +use std::mem::size_of; + +/// Transform Depth images into Color Camera space. +pub struct DepthTransformer { + transformation: Transformation, + output_width: u32, + output_height: u32, + output_stride_bytes: u32, +} + +impl DepthTransformer { + + pub fn new(calibration: &Calibration) -> Self { + let transformation = Transformation::from_calibration(&calibration); + + // TODO(bt): Temporary hardcoding for K4A_COLOR_RESOLUTION_2160P + let output_width = transformation.color_resolution.width as u32; + let output_height = transformation.color_resolution.height as u32; + //let output_width = 3840; //transformation.color_resolution.width as u32; + //let output_height = 2160; //transformation.color_resolution.height as u32; + let output_stride_bytes = output_width * size_of::() as u32; // NB: Depth pixel + + Self { + transformation, + output_width, + output_height, + output_stride_bytes, + } + } + + pub fn transform(&self, depth_image: &Image) -> AnyhowResult { + let transform_h = self.transformation.get_handle(); + let depth_image_h = depth_image.get_handle(); + + println!("Width, height, stride: {}, {}, {}", self.output_width, self.output_height, self.output_stride_bytes); + + let output_image = Image::create( + ImageFormat::Depth16, + self.output_width, + self.output_height, + self.output_stride_bytes, + ).map_err(|e| anyhow!("Can't create the image: {:?}", e))?; + + println!("Image created"); + + let output_image_h = output_image.get_handle(); + println!("About to transform..."); + + let result = unsafe { + k4a_sys_temp::k4a_transformation_depth_image_to_color_camera( + transform_h, + depth_image_h, + output_image_h, + ) + }; + println!("Transformed"); + + if result != k4a_sys_temp::k4a_buffer_result_t_K4A_BUFFER_RESULT_SUCCEEDED { + return Err(anyhow!("Depth transformation failed.")); + } + + Ok(output_image) + } + + // TODO + //pub fn transform_in_place(&mut self) { + //} +} diff --git a/src/zeromq/debug.rs b/src/zeromq/debug.rs new file mode 100644 index 000000000..3f1ca6155 --- /dev/null +++ b/src/zeromq/debug.rs @@ -0,0 +1,38 @@ +use cgmath::num_traits::Float; +use crate::zeromq::point::Point; + +/// Print the minima and maxima along each dimension. +pub fn print_pointcloud_maxima(points: &Vec) { + let mut max_x = f32::min_value(); + let mut min_x = f32::max_value(); + let mut max_y = f32::min_value(); + let mut min_y = f32::max_value(); + let mut max_z = f32::min_value(); + let mut min_z = f32::max_value(); + + for pt in points.iter() { + if pt.x < min_x { + min_x = pt.x; + } + if pt.x > max_x { + max_x = pt.x; + } + if pt.y < min_y { + min_y = pt.y; + } + if pt.y > max_y { + max_y = pt.y; + } + if pt.z < min_z { + min_z = pt.z; + } + if pt.z > max_z { + max_z = pt.z; + } + } + + // minmax x : -2612.5303, 4902.1655 | y: -3491.938, 1000.92487 | z: 163, 5494 + // minmax x : -2615.739, 4918.527 | y: -3507.2483, 1007.1275 | z: 163, 5518 + println!("minmax x : {}, {} | y: {}, {} | z: {}, {}", + min_x, max_x, min_y, max_y, min_z, max_z); +} diff --git a/src/zeromq/message.rs b/src/zeromq/message.rs new file mode 100644 index 000000000..3c7bdde07 --- /dev/null +++ b/src/zeromq/message.rs @@ -0,0 +1,67 @@ +use crate::zeromq::point::Point; +use byteorder::{WriteBytesExt, LittleEndian}; +use std::io::Write; + +/* + + +/// Denotes the command that sends the variable-length data +const POINT_DATA_BEGIN_PAYLOAD_COMMAND : u32 = 2; + +/// Denotes the command that sends the variable-length data +const POINT_DATA_CONTINUE_PAYLOAD_COMMAND : u32 = 3; + +/// The maximum number of points to send per ZeroMQ "packet". +/// This is also defined in C++, so it needs to be adjusted in multiple places. +const MAX_SEND_POINTS_PER_PACKET : usize = 3000; + +/// Streaming points +struct StreamingPointsPacket { + is_first_point_packet: bool, + points: Vec, +} + +impl StreamingPointsPacket { + fn from_all_points(points: Vec) -> Self { + Self { + is_first_point_packet: true, + points, + } + } + + /// Over the wire, this looks like: + /// + /// | command | 4 bytes (u32) | + /// | point_count | 4 bytes (u32) | + /// | points[] ... | [16 bytes] ... | + /// + fn encode(&self) -> Vec { + let point_bytes = Point::size_bytes() * MAX_SEND_POINTS_PER_PACKET; + + let mut buf = Vec::with_capacity(4 + point_bytes); + + if self.is_first_point_packet { + buf.write_u32::(POINT_DATA_BEGIN_PAYLOAD_COMMAND); // COMMAND # + } else { + buf.write_u32::(POINT_DATA_CONTINUE_PAYLOAD_COMMAND); // COMMAND # + } + + //let subset = if points.len() > MAX_SEND_POINTS_PER_PACKET { + // points.drain(0..MAX_SEND_POINTS_PER_PACKET).collect::>() + //} else { + // points.drain(0..points.len()).collect::>() + //}; + + buf.write_u32::(self.points.len() as u32); // LENGTH + + for point in self.points.iter() { + let bytes = point.to_bytes(); + buf.write_all(&bytes); + } + + buf + } +} + + +*/ diff --git a/src/zeromq/mod.rs b/src/zeromq/mod.rs new file mode 100644 index 000000000..3b250bfe4 --- /dev/null +++ b/src/zeromq/mod.rs @@ -0,0 +1,7 @@ +pub mod cpu_calculate_point_cloud; +pub mod color; +pub mod debug; +pub mod message; +pub mod point; +pub mod cpu_transformation; +pub mod xy_table; \ No newline at end of file diff --git a/src/zeromq/point.rs b/src/zeromq/point.rs new file mode 100644 index 000000000..56e6f52bd --- /dev/null +++ b/src/zeromq/point.rs @@ -0,0 +1,78 @@ +use byteorder::{WriteBytesExt, LittleEndian, BigEndian}; +use bytes::{BytesMut, BufMut}; +use crate::zeromq::color::Color; +use rand::Rng; +use rand::distributions::Uniform; + +#[derive(Debug)] +pub struct Point { + // Position + pub x: f32, + pub y: f32, + pub z: f32, + // Color + pub r: u8, + pub g: u8, + pub b: u8, + pub a: u8, +} + +impl Point { + + pub fn size_bytes() -> usize { + 16 // 3xf32, 4xu8 + } + + pub fn at(x: f32, y: f32, z: f32, color: Color) -> Self { + Point { + x, + y, + z, + r: color.get_u8_r(), + g: color.get_u8_g(), + b: color.get_u8_b(), + a: color.get_u8_a(), + } + } + + pub fn at_random_range(min: f32, max: f32, color: Color) -> Self { + let mut rng = rand::thread_rng(); + let range = Uniform::new_inclusive(min, max); + + let x = rng.sample(&range); + let y = rng.sample(&range); + let z = rng.sample(&range); + + Point { + x, + y, + z, + r: color.get_u8_r(), + g: color.get_u8_g(), + b: color.get_u8_b(), + a: color.get_u8_a(), + } + } + + pub fn to_bytes(&self) -> Vec { + let mut buf = Vec::with_capacity(32); + buf.write_f32::(self.x); + buf.write_f32::(self.y); + buf.write_f32::(self.z); + buf.write_u8(self.r); + buf.write_u8(self.g); + buf.write_u8(self.b); + buf.write_u8(self.a); + buf + } + + pub fn location_string(&self) -> String { + format!("{}, {}, {}", self.x, self.y, self.z) + } + + pub fn debug_string(&self) -> String { + format!("(location: {}, {}, {}); (color: {}, {}, {}, alpha: {})", + self.x, self.y, self.z, + self.r, self.g, self.b, self.a) + } +} \ No newline at end of file diff --git a/src/zeromq/xy_table.rs b/src/zeromq/xy_table.rs new file mode 100644 index 000000000..7cadc79dc --- /dev/null +++ b/src/zeromq/xy_table.rs @@ -0,0 +1,93 @@ +use crate::AnyhowResult; +use k4a_sys_temp as k4a_sys; +use kinect::{Calibration, Image, ImageFormat}; +use std::mem::size_of; +use std::os::raw::c_int; + +/** + * Creates an XY depth lookup table. (Taken from MIT-licensed `fastpointcloud.exe` in the K4A libs; doc below) + * + * The Azure Kinect Fastpointcloud example computes a 3d point cloud from a depth map. The example precomputes a lookup table + * by storing x- and y-scale factors for every pixel. At runtime, the 3d X-coordinate of a pixel in millimeters is derived + * by multiplying the pixel's depth value with the corresponding x-scale factor. The 3d Y-coordinate is obtained by + * multiplying with the y-scale factor. + * + * This method represents an alternative to calling k4a_transformation_depth_image_to_point_cloud() and lends itself + * to efficient implementation on the GPU. + */ +pub fn create_xy_table_from_depth_calibration(calibration: &Calibration) -> AnyhowResult { + let width = calibration.depth_camera_resolution_width(); + let height = calibration.depth_camera_resolution_height(); + create_xy_table(calibration, width, height) +} + +pub fn create_xy_table_from_color_calibration(calibration: &Calibration) -> AnyhowResult { + let width = calibration.color_camera_resolution_width(); + let height = calibration.color_camera_resolution_height(); + create_xy_table(calibration, width, height) +} + +pub fn create_xy_table(calibration: &Calibration, width: i32, height: i32) -> AnyhowResult { + let image = Image::create( + ImageFormat::Custom, + width as u32, + height as u32, + width as u32 * size_of::() as u32, + )?; + + let typed_buffer = image.get_buffer() as *mut k4a_sys::k4a_float2_t; + + let mut p = k4a_sys::k4a_float2_t { + xy: k4a_sys::k4a_float2_t__xy { + x: 0.0, + y: 0.0, + } + }; + + let mut ray = k4a_sys::k4a_float3_t { + xyz: k4a_sys::k4a_float3_t__xyz { + x: 0.0, + y: 0.0, + z: 0.0, + } + }; + + let mut idx = 0; + + for y in 0 .. height { + p.xy.y = y as f32; + + for x in 0 .. width { + p.xy.x = x as f32; + + let mut valid: c_int = -1; + let result = unsafe { + k4a_sys::k4a_calibration_2d_to_3d( + &calibration.0, + &p, // source point 2d + 1.0, // source depth mm + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_DEPTH, // source camera + k4a_sys::k4a_calibration_type_t_K4A_CALIBRATION_TYPE_DEPTH, // target camera + &mut ray, // target point3d mm + &mut valid // set to 1 when valid result, 0 when coordinate is not valid + ) + }; + + if valid == 1 { + unsafe { + (*typed_buffer.offset(idx)).xy.x = ray.xyz.x; + (*typed_buffer.offset(idx)).xy.y = ray.xyz.y; + } + } else { + unsafe { + (*typed_buffer.offset(idx)).xy.x = 0.0; + (*typed_buffer.offset(idx)).xy.y = 0.0; + } + } + + idx += 1; + } + } + + Ok(image) +} \ No newline at end of file diff --git a/temp/old_protocol.txt b/temp/old_protocol.txt new file mode 100644 index 000000000..ade430c97 --- /dev/null +++ b/temp/old_protocol.txt @@ -0,0 +1,60 @@ + let mut reconnect = false; + let mut fail_count = 0; + +/* + let color = get_color(); + let point = Point::at_random_range(-1000.0f32, 1000.0f32, color); + let bytes = point.to_bytes(); + //println!("Point : {}", point.debug_string()); + + if reconnect { + socket = reconnect_socket(&context, socket, SOCKET_ADDRESS); + reconnect = false; + messaging_state = MessagingState::Sending; + } + + match messaging_state { + MessagingState::Sending => { + //println!("Sending request..."); + //let result = socket.send("hello world!", 0); + let result = socket.send(&bytes, 0); + + match result { + Ok(_) => { + //println!("Sent!"); + //thread::sleep(Duration::from_millis(250)); + messaging_state = MessagingState::Receiving; + }, + Err(e) => { + //eprintln!("Send Error ({}): {:?}", e.to_raw(), e); + //thread::sleep(Duration::from_millis(250)); + fail_count += 1; + }, + } + + }, + MessagingState::Receiving => { + //println!("Awaiting response..."); + let result = socket.recv_bytes(DONTWAIT); + + match result { + Ok(_) => { + //println!("Response received!"); + messaging_state = MessagingState::Sending; + }, + Err(e) => { + //eprintln!("Recv Error ({}): {:?}", e.to_raw(), e); + //thread::sleep(Duration::from_millis(250)); + fail_count += 1; + }, + } + + }, + } + + if fail_count > 5 { + reconnect = true; + fail_count = 0; + //thread::sleep(Duration::from_millis(2000)); + } +*/