From 30dc91e5a03da29f3394c04d3ed6c73c8a2df2b5 Mon Sep 17 00:00:00 2001 From: Evan Pratten Date: Sat, 29 May 2021 13:35:33 -0400 Subject: [PATCH] video streaming --- README.md | 2 ++ libodm/Cargo.toml | 2 ++ libodm/build.rs | 1 - libodm/cpp/listener.cc | 28 ----------------- libodm/cpp/wrapper.cc | 16 ++++++---- libodm/src/image.rs | 56 ++++++++++++++++++++++++++++++--- libodm/src/leapmotion/device.rs | 48 +++++++++++++++++++--------- libodm/src/leapmotion/ffi.rs | 1 + libodm/src/lib.rs | 3 +- libodm/src/stereo/mod.rs | 9 ++++++ odm/src/main.rs | 13 ++++++-- 11 files changed, 120 insertions(+), 59 deletions(-) delete mode 100644 libodm/cpp/listener.cc create mode 100644 libodm/src/stereo/mod.rs diff --git a/README.md b/README.md index b2d2067..7b6513d 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,5 @@ Point cloud streams from Leap Motion cameras clang libclang-dev libopencv-dev ``` + +https://developer-archive.leapmotion.com/documentation/v2/cpp/index.html \ No newline at end of file diff --git a/libodm/Cargo.toml b/libodm/Cargo.toml index 06919f1..5f079b6 100644 --- a/libodm/Cargo.toml +++ b/libodm/Cargo.toml @@ -10,6 +10,8 @@ build = "build.rs" [dependencies] libc = "0.2.95" log = "0.4.14" +glam = "0.15.2" +opencv = "0.53.0" [build-dependencies] cc = "1.0" diff --git a/libodm/build.rs b/libodm/build.rs index 190fb3c..6e5e4e9 100644 --- a/libodm/build.rs +++ b/libodm/build.rs @@ -14,5 +14,4 @@ fn main() { // Make cargo auto-rebuild these files println!("cargo:rerun-if-changed=libodm/build.rs"); println!("cargo:rerun-if-changed=libodm/cpp/wrapper.cc"); - println!("cargo:rerun-if-changed=libodm/cpp/listener.cc"); } diff --git a/libodm/cpp/listener.cc b/libodm/cpp/listener.cc deleted file mode 100644 index 8082e8a..0000000 --- a/libodm/cpp/listener.cc +++ /dev/null @@ -1,28 +0,0 @@ -class LeapEventListener : public Listener -{ -public: - virtual void onConnect(const Controller &); - virtual void onDisconnect(const Controller &); - virtual void onFrame(const Controller &); -}; - -void LeapEventListener::onConnect(const Controller &controller) -{ - // std::cout << "LeapMotion Controller: Connected" << std::endl; -} - -//Not dispatched when running in a debugger -void LeapEventListener::onDisconnect(const Controller &controller) -{ - // std::cout << "LeapMotion Controller: Disconnected" << std::endl; -} - -void LeapEventListener::onFrame(const Controller &controller) -{ - // std::cout << "New frame available" << std::endl; - // Frame frame = controller.frame(); - - // // Get the camera images - // images = frame.images(); - -} \ No newline at end of file diff --git a/libodm/cpp/wrapper.cc b/libodm/cpp/wrapper.cc index 164e6d7..d087577 100644 --- a/libodm/cpp/wrapper.cc +++ b/libodm/cpp/wrapper.cc @@ -7,6 +7,7 @@ using namespace Leap; extern "C" void beginEventLoop(); extern "C" bool isControllerCreated(); extern "C" void endEventLoop(); +extern "C" void updateFrame(); extern "C" bool imageExists(); extern "C" int getImageHeight(); extern "C" int getImageWidth(); @@ -17,11 +18,9 @@ extern "C" const unsigned char *getImageRight(); //---- Start Globals ----// Controller *controller = nullptr; -// ImageList images; +Frame *frame = nullptr; //---- End Globals ----// -#include "listener.cc" - //---- Start Public Function Impls ----// void beginEventLoop() @@ -34,9 +33,6 @@ void beginEventLoop() // Set device policy controller->setPolicyFlags(Controller::POLICY_IMAGES); - // Set up event handling - // LeapEventListener listener; - // controller->addListener(listener); } } @@ -48,6 +44,14 @@ void endEventLoop() } } +void updateFrame() +{ + // free(frame); + // Frame f = controller->frame(); + // frame = (Frame*) malloc(sizeof(f)); + // memccpy(frame, f); +} + bool isControllerCreated() { return controller != nullptr; } bool imageExists() { return controller->frame().images().count() == 2; } int getImageHeight() { return controller->frame().images()[0].height(); } diff --git a/libodm/src/image.rs b/libodm/src/image.rs index 15ae7ab..0fca6d2 100644 --- a/libodm/src/image.rs +++ b/libodm/src/image.rs @@ -1,10 +1,56 @@ -#[derive(Debug)] -pub struct Image {} +use glam::UVec2; +use opencv::{core::{Image2D, Mat, Mat_AUTO_STEP, Size, CV_8U}, highgui::{imshow, named_window, wait_key}}; + +pub struct Image { + size: UVec2, + raw_data: *const u8, + img: Mat, +} impl Image { - pub fn new(width: i32, height: i32, raw_data: &[u8]) -> Self { - Self { - + + /// Create a new image from raw data + pub fn new_raw(width: i32, height: i32, raw_data: *const u8) -> Result { + // Build a CV size + let size = Size::new(width, height); + + let image; + unsafe { + image = + Mat::new_size_with_data(size, CV_8U, raw_data as *mut libc::c_void, Mat_AUTO_STEP)?; } + + Ok(Self { + size: UVec2::new(width as u32, height as u32), + raw_data, + img: image, + }) + } + + /// Get the image size + pub fn get_size(&self) -> &UVec2 { + &self.size + } + + /// Get the internal OpenCV image + pub fn get_image(&self) -> &Mat { + &self.img + } + + /// Show this image in a debug window + pub fn debug_show(&self){ + + // Create a debug window + imshow("Debug", &self.img).unwrap(); + wait_key(0).unwrap(); + } } + +// impl Drop for Image { +// fn drop(&mut self) { +// unsafe { +// libc::free(self.raw_data as *mut libc::c_void); +// } +// } +// } diff --git a/libodm/src/leapmotion/device.rs b/libodm/src/leapmotion/device.rs index 029b231..b56b297 100644 --- a/libodm/src/leapmotion/device.rs +++ b/libodm/src/leapmotion/device.rs @@ -2,6 +2,8 @@ use std::time::{Duration, SystemTime}; +use opencv::highgui::{imshow, wait_key}; + use crate::image::Image; use super::ffi; @@ -17,10 +19,12 @@ pub enum DeviceError { /// An error caused when no data is being sent by the device NoData, + + /// An error cause when image data cannot be decoded + DecodeError, } /// Represents a single data frame generated by the LeapMotion device -#[derive(Debug)] pub struct DeviceFrame { /// Number of bytes per pixel pub bytes_per_pixel: u8, @@ -32,6 +36,14 @@ pub struct DeviceFrame { pub right_camera: Image, } +impl DeviceFrame { + pub fn debug_show(&self){ + imshow("Left Camera", self.left_camera.get_image()).unwrap(); + imshow("Right Camera", self.right_camera.get_image()).unwrap(); + wait_key(25).unwrap(); + } +} + /// Represents a LeapMotion device, and provides safe wrappers around its FFI #[derive(Debug)] pub struct LeapMotionDevice { @@ -69,6 +81,7 @@ impl LeapMotionDevice { // Handle device connect unsafe { + ffi::updateFrame(); if ffi::isControllerCreated() && ffi::imageExists() { break; } @@ -100,6 +113,13 @@ impl LeapMotionDevice { /// Get the latest frame from the device pub fn get_frame(&mut self) -> Result { + // Update the frame data from the controller + unsafe{ + if ffi::isControllerCreated(){ + ffi::updateFrame(); + } + } + if !self.has_frame() { return Err(DeviceError::NoData); } @@ -108,30 +128,28 @@ impl LeapMotionDevice { let image_width; let image_height; let bytes_per_pixel; - let left_image_raw; - let right_image_raw; + let mut left_image_raw; + let mut right_image_raw; unsafe { image_width = ffi::getImageWidth(); image_height = ffi::getImageHeight(); bytes_per_pixel = ffi::getImageBPP(); - left_image_raw = std::slice::from_raw_parts( - ffi::getImageLeft(), - (image_width * image_height * bytes_per_pixel) as usize, - ); - right_image_raw = std::slice::from_raw_parts( - ffi::getImageRight(), - (image_width * image_height * bytes_per_pixel) as usize, - ); + left_image_raw = ffi::getImageLeft(); + right_image_raw = ffi::getImageRight(); } // Build two images - let left_image = Image::new(image_width, image_height, left_image_raw); - let right_image = Image::new(image_width, image_height, right_image_raw); + let left_image = Image::new_raw(image_width, image_height, left_image_raw); + let right_image = Image::new_raw(image_width, image_height, right_image_raw); + + if left_image.is_err() || right_image.is_err() { + return Err(DeviceError::DecodeError); + } Ok(DeviceFrame { bytes_per_pixel: bytes_per_pixel as u8, - left_camera: left_image, - right_camera: right_image, + left_camera: left_image.unwrap(), + right_camera: right_image.unwrap(), }) } } diff --git a/libodm/src/leapmotion/ffi.rs b/libodm/src/leapmotion/ffi.rs index fd41286..180f87f 100644 --- a/libodm/src/leapmotion/ffi.rs +++ b/libodm/src/leapmotion/ffi.rs @@ -3,6 +3,7 @@ extern { pub fn beginEventLoop(); pub fn isControllerCreated() -> bool; pub fn endEventLoop(); + pub fn updateFrame(); pub fn imageExists() -> bool; pub fn getImageHeight() -> libc::c_int; pub fn getImageWidth()-> libc::c_int; diff --git a/libodm/src/lib.rs b/libodm/src/lib.rs index 0fe478b..e3c6a9b 100644 --- a/libodm/src/lib.rs +++ b/libodm/src/lib.rs @@ -1,4 +1,5 @@ #![feature(static_nobundle)] pub mod leapmotion; -pub mod image; \ No newline at end of file +pub mod image; +pub mod stereo; \ No newline at end of file diff --git a/libodm/src/stereo/mod.rs b/libodm/src/stereo/mod.rs new file mode 100644 index 0000000..f1d0460 --- /dev/null +++ b/libodm/src/stereo/mod.rs @@ -0,0 +1,9 @@ +// use crate::image::Image; +// use opencv::stereo::MatchQuasiDense; + + +// pub fn stereo_from_images(left: &Image, right: &Image) { + +// MatchQuasiDense + +// } \ No newline at end of file diff --git a/odm/src/main.rs b/odm/src/main.rs index f9960f4..47393b7 100644 --- a/odm/src/main.rs +++ b/odm/src/main.rs @@ -6,12 +6,19 @@ use libodm::leapmotion::device::LeapMotionDevice; mod cli; fn main() { - start_fern(); - + // Init a device - let device = LeapMotionDevice::create_device(Duration::from_secs(1)).unwrap(); + let mut device = LeapMotionDevice::create_device(Duration::from_secs(4)).unwrap(); log::info!("Got data"); + log::info!("Press CTRL+C to exit"); + loop { + // Read a single frame + let frame = device.get_frame().unwrap(); + + // Show data + frame.debug_show(); + } }