video streaming

README.md

@@ -4,3 +4,5 @@ Point cloud streams from Leap Motion cameras
 clang libclang-dev libopencv-dev
 
 ```
+
+https://developer-archive.leapmotion.com/documentation/v2/cpp/index.html

libodm/Cargo.toml

@@ -10,6 +10,8 @@ build = "build.rs"
 [dependencies]
 libc = "0.2.95"
 log = "0.4.14"
+glam = "0.15.2"
+opencv = "0.53.0"
 
 [build-dependencies]
 cc = "1.0"

libodm/build.rs

@@ -14,5 +14,4 @@ fn main() {
     // Make cargo auto-rebuild these files
     println!("cargo:rerun-if-changed=libodm/build.rs");
     println!("cargo:rerun-if-changed=libodm/cpp/wrapper.cc");
-    println!("cargo:rerun-if-changed=libodm/cpp/listener.cc");
 }

libodm/cpp/listener.cc

@@ -1,28 +0,0 @@
-class LeapEventListener : public Listener
-{
-public:
-    virtual void onConnect(const Controller &);
-    virtual void onDisconnect(const Controller &);
-    virtual void onFrame(const Controller &);
-};
-
-void LeapEventListener::onConnect(const Controller &controller)
-{
-    // std::cout << "LeapMotion Controller: Connected" << std::endl;
-}
-
-//Not dispatched when running in a debugger
-void LeapEventListener::onDisconnect(const Controller &controller)
-{
-    // std::cout << "LeapMotion Controller: Disconnected" << std::endl;
-}
-
-void LeapEventListener::onFrame(const Controller &controller)
-{
-    // std::cout << "New frame available" << std::endl;
-    // Frame frame = controller.frame();
-
-    // // Get the camera images
-    // images = frame.images();
-
-}

libodm/cpp/wrapper.cc

@@ -7,6 +7,7 @@ using namespace Leap;
 extern "C" void beginEventLoop();
 extern "C" bool isControllerCreated();
 extern "C" void endEventLoop();
+extern "C" void updateFrame();
 extern "C" bool imageExists();
 extern "C" int getImageHeight();
 extern "C" int getImageWidth();
@@ -17,11 +18,9 @@ extern "C" const unsigned char *getImageRight();
 
 //---- Start Globals ----//
 Controller *controller = nullptr;
-// ImageList images;
+Frame *frame = nullptr;
 //---- End Globals ----//
 
-#include "listener.cc"
-
 //---- Start Public Function Impls ----//
 
 void beginEventLoop()
@@ -34,9 +33,6 @@ void beginEventLoop()
         // Set device policy
         controller->setPolicyFlags(Controller::POLICY_IMAGES);
 
-        // Set up event handling
-        // LeapEventListener listener;
-        // controller->addListener(listener);
     }
 }
 
@@ -48,6 +44,14 @@ void endEventLoop()
     }
 }
 
+void updateFrame()
+{
+    // free(frame);
+    // Frame f = controller->frame();
+    // frame = (Frame*) malloc(sizeof(f));
+    // memccpy(frame, f);
+}
+
 bool isControllerCreated() { return controller != nullptr; }
 bool imageExists() { return controller->frame().images().count() == 2; }
 int getImageHeight() { return controller->frame().images()[0].height(); }

libodm/src/image.rs

@@ -1,10 +1,56 @@
-#[derive(Debug)]
+use glam::UVec2;
-pub struct Image {}
+use opencv::{core::{Image2D, Mat, Mat_AUTO_STEP, Size, CV_8U}, highgui::{imshow, named_window, wait_key}};
+
+pub struct Image {
+    size: UVec2,
+    raw_data: *const u8,
+    img: Mat,
+}
 
 impl Image {
-    pub fn new(width: i32, height: i32, raw_data: &[u8]) -> Self {
+
-        Self {
+    /// Create a new image from raw data
-            
+    pub fn new_raw(width: i32, height: i32, raw_data: *const u8) -> Result<Self, opencv::Error> {
+        // Build a CV size
+        let size = Size::new(width, height);
+
+        let image;
+        unsafe {
+            image =
+                Mat::new_size_with_data(size, CV_8U, raw_data as *mut libc::c_void, Mat_AUTO_STEP)?;
         }
+
+        Ok(Self {
+            size: UVec2::new(width as u32, height as u32),
+            raw_data,
+            img: image,
+        })
+    }
+
+    /// Get the image size
+    pub fn get_size(&self) -> &UVec2 {
+        &self.size
+    }
+
+    /// Get the internal OpenCV image
+    pub fn get_image(&self) -> &Mat {
+        &self.img
+    }
+
+    /// Show this image in a debug window
+    pub fn debug_show(&self){
+
+        // Create a debug window
+        imshow("Debug", &self.img).unwrap();
+        wait_key(0).unwrap();
+
     }
 }
+
+// impl Drop for Image {
+//     fn drop(&mut self) {
+//         unsafe {
+//             libc::free(self.raw_data as *mut libc::c_void);
+//         }
+//     }
+// }

libodm/src/leapmotion/device.rs

@@ -2,6 +2,8 @@
 
 use std::time::{Duration, SystemTime};
 
+use opencv::highgui::{imshow, wait_key};
+
 use crate::image::Image;
 
 use super::ffi;
@@ -17,10 +19,12 @@ pub enum DeviceError {
 
     /// An error caused when no data is being sent by the device
     NoData,
+
+    /// An error cause when image data cannot be decoded
+    DecodeError,
 }
 
 /// Represents a single data frame generated by the LeapMotion device
-#[derive(Debug)]
 pub struct DeviceFrame {
     /// Number of bytes per pixel
     pub bytes_per_pixel: u8,
@@ -32,6 +36,14 @@ pub struct DeviceFrame {
     pub right_camera: Image,
 }
 
+impl DeviceFrame {
+    pub fn debug_show(&self){
+        imshow("Left Camera", self.left_camera.get_image()).unwrap();
+        imshow("Right Camera", self.right_camera.get_image()).unwrap();
+        wait_key(25).unwrap();
+    }
+}
+
 /// Represents a LeapMotion device, and provides safe wrappers around its FFI
 #[derive(Debug)]
 pub struct LeapMotionDevice {
@@ -69,6 +81,7 @@ impl LeapMotionDevice {
 
             // Handle device connect
             unsafe {
+                ffi::updateFrame();
                 if ffi::isControllerCreated() && ffi::imageExists() {
                     break;
                 }
@@ -100,6 +113,13 @@ impl LeapMotionDevice {
 
     /// Get the latest frame from the device
     pub fn get_frame(&mut self) -> Result<DeviceFrame, DeviceError> {
+        // Update the frame data from the controller
+        unsafe{
+            if ffi::isControllerCreated(){
+                ffi::updateFrame();
+            }
+        }
+
         if !self.has_frame() {
             return Err(DeviceError::NoData);
         }
@@ -108,30 +128,28 @@ impl LeapMotionDevice {
         let image_width;
         let image_height;
         let bytes_per_pixel;
-        let left_image_raw;
+        let mut left_image_raw;
-        let right_image_raw;
+        let mut right_image_raw;
         unsafe {
             image_width = ffi::getImageWidth();
             image_height = ffi::getImageHeight();
             bytes_per_pixel = ffi::getImageBPP();
-            left_image_raw = std::slice::from_raw_parts(
+            left_image_raw = ffi::getImageLeft();
-                ffi::getImageLeft(),
+            right_image_raw = ffi::getImageRight();
-                (image_width * image_height * bytes_per_pixel) as usize,
-            );
-            right_image_raw = std::slice::from_raw_parts(
-                ffi::getImageRight(),
-                (image_width * image_height * bytes_per_pixel) as usize,
-            );
         }
 
         // Build two images
-        let left_image = Image::new(image_width, image_height, left_image_raw);
+        let left_image = Image::new_raw(image_width, image_height, left_image_raw);
-        let right_image = Image::new(image_width, image_height, right_image_raw);
+        let right_image = Image::new_raw(image_width, image_height, right_image_raw);
+
+        if left_image.is_err() || right_image.is_err() {
+            return Err(DeviceError::DecodeError);
+        }
 
         Ok(DeviceFrame {
             bytes_per_pixel: bytes_per_pixel as u8,
-            left_camera: left_image,
+            left_camera: left_image.unwrap(),
-            right_camera: right_image,
+            right_camera: right_image.unwrap(),
         })
     }
 }

libodm/src/leapmotion/ffi.rs

@@ -3,6 +3,7 @@ extern {
     pub fn beginEventLoop();
     pub fn isControllerCreated() -> bool;
     pub fn endEventLoop();
+    pub fn updateFrame();
     pub fn imageExists() -> bool;
     pub fn getImageHeight() -> libc::c_int;
     pub fn getImageWidth()-> libc::c_int;

libodm/src/lib.rs

@@ -1,4 +1,5 @@
 #![feature(static_nobundle)]
 
 pub mod leapmotion;
-pub mod image;
+pub mod image;
+pub mod stereo;

libodm/src/stereo/mod.rs

@@ -0,0 +1,9 @@
+// use crate::image::Image;
+// use opencv::stereo::MatchQuasiDense;
+
+
+// pub fn stereo_from_images(left: &Image, right: &Image) {
+
+//     MatchQuasiDense
+
+// }

odm/src/main.rs

@@ -6,12 +6,19 @@ use libodm::leapmotion::device::LeapMotionDevice;
 mod cli;
 
 fn main() {
-
     start_fern();
-    
+
     // Init a device
-    let device = LeapMotionDevice::create_device(Duration::from_secs(1)).unwrap();
+    let mut device = LeapMotionDevice::create_device(Duration::from_secs(4)).unwrap();
 
     log::info!("Got data");
 
+    log::info!("Press CTRL+C to exit");
+    loop {
+        // Read a single frame
+        let frame = device.get_frame().unwrap();
+
+        // Show data
+        frame.debug_show();
+    }
 }