public RealSenseEngine()
{
m_session = PXCMSession.CreateInstance();
if (m_session == null)
{
throw new RealSenseEngineException("Failed to create a Session");
}
m_senseManager = m_session.CreateSenseManager();
if (m_senseManager == null)
{
throw new RealSenseEngineException("Failed to create a SenseManager");
}
pxcmStatus res = m_senseManager.EnableTouchlessController();
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Enable Touchless Controller");
}
var handler = new PXCMSenseManager.Handler();
res = m_senseManager.Init(handler);
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Init Handler");
}
// getting touchless controller
m_touchlessController = m_senseManager.QueryTouchlessController();
if (m_touchlessController == null)
{
throw new RealSenseEngineException("Failed to Query Touchless Controller");
}
}
private void Uninitialize()
{
if (this.controller != null)
{
this.controller.UnsubscribeEvent(OnTouchlessControllerUXEvent);
this.controller.Dispose();
this.controller = null;
}
if (this.session != null)
{
this.session.Dispose();
this.session = null;
}
if (this.senseManager != null)
{
this.senseManager.Close();
this.senseManager.Dispose();
this.senseManager = null;
}
this.initialized = false;
}
public RealSenseEngine()
{
m_session = PXCMSession.CreateInstance();
if (m_session == null)
{
throw new RealSenseEngineException("Failed to create a Session");
}
m_senseManager = m_session.CreateSenseManager();
if (m_senseManager == null)
{
throw new RealSenseEngineException("Failed to create a SenseManager");
}
pxcmStatus res = m_senseManager.EnableTouchlessController();
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Enable Touchless Controller");
}
var handler = new PXCMSenseManager.Handler();
res = m_senseManager.Init(handler);
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Init Handler");
}
// getting touchless controller
m_touchlessController = m_senseManager.QueryTouchlessController();
if (m_touchlessController == null)
{
throw new RealSenseEngineException("Failed to Query Touchless Controller");
}
}
private void StartRealSense()
{
Console.WriteLine("Starting Touchless Controller");
pxcmStatus rc;
// creating Sense Manager
psm = PXCMSenseManager.CreateInstance();
Console.WriteLine("Creating SenseManager: " + psm == null ? "failed" : "success");
if (psm == null)
{
MessageBox.Show("Failed to create SenseManager!", "Failed", MessageBoxButton.OK, MessageBoxImage.Error);
Environment.Exit(-1);
}
// work from file if a filename is given as command line argument
string[] args = Environment.GetCommandLineArgs();
if (args.Length > 1)
{
psm.captureManager.SetFileName(args[1], false);
}
// Enable touchless controller in the multimodal pipeline
rc = psm.EnableTouchlessController(null);
Console.WriteLine("Enabling Touchless Controller: " + rc.ToString());
if (rc != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
MessageBox.Show("Failed to enable touchless controller!", "Failed", MessageBoxButton.OK, MessageBoxImage.Error);
Environment.Exit(-1);
}
// initialize the pipeline
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
rc = psm.Init(handler);
Console.WriteLine("Initializing the pipeline: " + rc.ToString());
if (rc != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
MessageBox.Show("Failed to initialize the pipeline!", "Failed", MessageBoxButton.OK, MessageBoxImage.Error);
Environment.Exit(-1);
}
// getting touchless controller
ptc = psm.QueryTouchlessController();
if (ptc == null)
{
MessageBox.Show("Failed to get touchless controller!", "Failed", MessageBoxButton.OK, MessageBoxImage.Error);
Environment.Exit(-1);
}
ptc.SubscribeEvent(new PXCMTouchlessController.OnFiredUXEventDelegate(OnTouchlessControllerUXEvent));
}
// RS
private void STButton_Click(object sender, RoutedEventArgs e) {
EXButton.IsEnabled = true;
STButton.IsEnabled = false;
// init the sense manager
_senseManager = PXCMSenseManager.CreateInstance();
// enable hand analysis in the multimodal pipeline
_senseManager.EnableTouchlessController();
// init the pipeline
if (_senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR) {
MessageBox.Show("RealSense Camera init failed");
}
// get an instance of the touchless control
_touchlessController = _senseManager.QueryTouchlessController();
// register fo ux event
_touchlessController.SubscribeEvent(OnTouchlessControllerUXEventHandler); //event
_touchlessController.SubscribeAlert(OnFiredAlertEventHandler); //alert
// configuration
pxcmStatus rc;
PXCMTouchlessController.ProfileInfo pInfo;
rc = _touchlessController.QueryProfile(out pInfo);
pInfo.config = PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Zoom
| PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Selection
| PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Back;
rc = _touchlessController.SetProfile(pInfo);
// configure hand module
_hand = _senseManager.QueryHand();
_handConfig = _hand.CreateActiveConfiguration();
_handConfig.EnableGesture("thumb_up", true); // true is importment
_handConfig.EnableGesture("v_sign", true);
_handConfig.SubscribeGesture(OnFiredGestureEventHandler);
_handConfig.ApplyChanges();
// processing loop
_processingThread = new Thread(new ThreadStart(ProcessingThread));
_processingThread.Start();
}
private void InitTouchlessController()
{
this.senseManager = PXCMSenseManager.CreateInstance();
if (this.senseManager == null)
{
throw new Exception("Could not create Sense Manager.");
}
this.session = this.senseManager.session;
pxcmStatus sts = this.senseManager.EnableTouchlessController(null);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Can't get Touchless Controller");
}
sts = this.senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, this.width, this.height, FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Could not enable Color Stream.");
}
sts = this.senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, this.width, this.height, FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Could not enable Depth Stream.");
}
if (!this.senseManager.IsConnected())
{
return;
}
var handler = new PXCMSenseManager.Handler();
sts = this.senseManager.Init(handler);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Initialization failed. " + sts.ToString());
}
this.controller = this.senseManager.QueryTouchlessController();
if (this.controller == null)
{
throw new Exception("Can't get Touchless Controller");
}
this.controller.SubscribeEvent(new PXCMTouchlessController.OnFiredUXEventDelegate(OnTouchlessControllerUXEvent));
this.controller.AddGestureActionMapping("swipeLeft", PXCMTouchlessController.Action.Action_LeftKeyPress, OnFiredAction);
PXCMTouchlessController.ProfileInfo pinfo;
//this.controller.ClearAllGestureActionMappings();
sts = this.controller.QueryProfile(out pinfo);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Can't get Touchless Controller profile");
}
IntPtr pointer = this.controller.QueryNativePointer();
for (int i = 0; i < FInConfiguration.SliceCount; i++)
{
pinfo.config |= FInConfiguration[i];
}
sts = this.controller.SetProfile(pinfo);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Can't set Touchless Controller profile");
}
this.senseManager.StreamFrames(false);
this.initialized = true;
}
public void SetConfiguration(PXCMTouchlessController.ProfileInfo.Configuration configuration)
{
PXCMTouchlessController.ProfileInfo pInfo;
pxcmStatus res = m_touchlessController.ClearAllGestureActionMappings();
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Clear All Gesture Action Mappings. {0}", res);
}
res = m_touchlessController.QueryProfile(out pInfo);
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Query Profile. {0}", res);
}
pInfo.config = configuration;
res = m_touchlessController.SetProfile(pInfo);
if (res != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new RealSenseEngineException("Failed to Set Profile. {0}", res);
}
}
// touchless event handler
private void OnTouchlessControllerUXEventHandler(PXCMTouchlessController.UXEventData uxEventData) {
switch (uxEventData.type) {
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorVisible:
{
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = "Got your...hand"));
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorNotVisible:
{
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = "No hand detected>3<"));
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorMove:
{
double winHeight = SystemParameters.PrimaryScreenHeight; ;// 1080;
double winWidth = SystemParameters.PrimaryScreenWidth;// 1920;
int mousePos_x = (int)(winWidth * Convert.ToDouble(uxEventData.position.x));
int mousePos_y = (int)(winHeight * Convert.ToDouble(uxEventData.position.y));
SetPosition(mousePos_x, mousePos_y);
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_StartZoom:
{
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = "Zoom Start"));
webBrowser1.Dispatcher.Invoke(new Action(() => { webBrowser1.InvokeScript("startZoom", new object[] { }); }));
_rs_Old_Z = uxEventData.position.z;
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Zoom:
// handle continued zoom
double zoomDelta = _rs_Old_Z - uxEventData.position.z;
if (Math.Abs(zoomDelta) > 0.05) {
_rs_Old_Z = uxEventData.position.z;
var zoomVar = zoomDelta > 0 ? 1 : -1;
_mapZoom = _mapZoom + zoomVar;
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = "Zooming"));
// call js script from WPF
webBrowser1.Dispatcher.Invoke(new Action(() => { webBrowser1.InvokeScript("processZoomInOut", new object[] { zoomVar }); }));
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_EndZoom:
// handle end zoom
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = "Zoom End"));
webBrowser1.Dispatcher.Invoke(new Action(() => { webBrowser1.InvokeScript("endZoom", new object[] { }); }));
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Select:
MouseSimulator.ClickLeftMouseButton();
RSInfoText.Dispatcher.Invoke(new Action(() => RSInfoText.Text = " mouse left button clicked"));
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_GotoStart:
// open the steet view
if (_isInMap) {
webBrowser1.Dispatcher.Invoke(new Action(() => { webBrowser1.InvokeScript("deleteLayersandMarkers", new object[] { }); })); // it is so ugly to exit sv
}
else {
webBrowser1.Dispatcher.Invoke(new Action(() => { webBrowser1.InvokeScript("gobacktoMap", new object[] { }); })); // it is so ugly to exit sv
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Back:
break;
}
}
// alert event
private void OnFiredAlertEventHandler(PXCMTouchlessController.AlertData data) {
Dispatcher.BeginInvoke((Action)(() => {
switch (data.type) {
case PXCMTouchlessController.AlertData.AlertType.Alert_TooClose:
Popwind_ShowInfo("Too Close");
break;
case PXCMTouchlessController.AlertData.AlertType.Alert_TooFar:
Popwind_ShowInfo("Too Far");
break;
case PXCMTouchlessController.AlertData.AlertType.Alert_NoAlerts:
break;
}
}));
}
private void OnFiredAlertDelegate(PXCMTouchlessController.AlertData data)
{
Dispatcher.BeginInvoke((Action)(() =>
{
switch (data.type)
{
case PXCMTouchlessController.AlertData.AlertType.Alert_TooClose:
log.Text += "Your hand is too close.\n";
break;
case PXCMTouchlessController.AlertData.AlertType.Alert_TooFar:
log.Text += "Your hand is too far away.\n";
break;
case PXCMTouchlessController.AlertData.AlertType.Alert_NoAlerts:
break;
}
}));
}
private void OnFiredUxEventDelegate(PXCMTouchlessController.UXEventData data)
{
var posX = data.position.x;
var posY = data.position.y;
var horizontal = CursorMapping.HorizontalMappingConstant;
var vertical = CursorMapping.VeriticalMappingContant;
var Y = (int)(posY * vertical);
var X = (int)(posX * horizontal);
Dispatcher.BeginInvoke((Action)(() =>
{
switch (data.type)
{
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorVisible:
log.Text += "Recognized hand input. Tracking...\n";
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorNotVisible:
log.Text += "Lost tracked hand...\n";
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_CursorMove:
labelX.Dispatcher.Invoke(
new Action(() => labelX.Content = "x " + data.position.x.ToString()));
labelY.Dispatcher.Invoke(
new Action(() => labelY.Content = "y " + data.position.y.ToString()));
labelZ.Dispatcher.Invoke(
new Action(() => labelZ.Content = "z " + data.position.z.ToString()));
if (gameMode.IsChecked == true)
{
int sens = (int)sensitivity.Value;
// generating user input rectangle
if (posX > 0.5 && posX < 0.7)
{
SetCursorPos(horizontal / 2 - sens / 2 + (int)(posX / 2 * sens), vertical / 2);
} else if (posX > 0.8)
{
SetCursorPos(horizontal / 2 - sens / 2 + (int)(posX * sens), vertical / 2);
} else if (posY > 0.6)
{
SetCursorPos(horizontal / 2, vertical / 2 - sens / 2 + (int)(posY * sens));
} else if (posY < 0.4)
{
SetCursorPos(horizontal / 2, vertical / 2 - sens / 2 + (int)(posY * sens));
}
//SetCursorPos((horizontal / 2 - sens / 2) + (int)(posX * sens), (vertical / 2 - sens / 2) + (int)(posY * sens));
}
else
{
SetCursorPos(X, Y);
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Select:
log.Text += "Found gesture 'Select'\n";
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Scroll:
Thread.Sleep(10);
log.Text += "Found gesture 'Scroll'\n";
if (MouseClick == true)
{
LeftUp(X, Y);
MouseClick = false;
} else if (MouseClick == false)
{
LeftDown(X, Y);
MouseClick = true;
}
break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_EndScroll:
log.Text += "Found gesture 'EndScroll'\n";
break;
default:
break;
}
}));
}
private void releaseInstances()
{
if (handModule != null)
{
handModule.Dispose();
handModule = null;
}
if (touchController != null)
{
touchController.Dispose();
touchController = null;
}
if (manager != null)
{
manager.Close();
manager.Dispose();
manager = null;
}
started = false;
}
private void OnTouchlessControllerUxEvent(PXCMTouchlessController.UXEventData data)
{
if (data.type == UXEventT.UXEvent_CursorMove)
{
this.handData.Update();
if (this.handData.QueryNumberOfHands() < 2)
{
PXCMHandData.IHand hand;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, 0, out hand);
if (hand != null)
{
PXCMHandData.FingerData thumbFinger, indexFinger, middleFinger, ringFinger, pinkyFinger;
hand.QueryFingerData(PXCMHandData.FingerType.FINGER_THUMB, out thumbFinger);
hand.QueryFingerData(PXCMHandData.FingerType.FINGER_INDEX, out indexFinger);
hand.QueryFingerData(PXCMHandData.FingerType.FINGER_MIDDLE, out middleFinger);
hand.QueryFingerData(PXCMHandData.FingerType.FINGER_RING, out ringFinger);
hand.QueryFingerData(PXCMHandData.FingerType.FINGER_PINKY, out pinkyFinger);
bool allowMovement;
if (touchReserved
&& indexFinger.foldedness < dragSensitivity
&& middleFinger.foldedness < dragSensitivity
&& pinkyFinger.foldedness > 60)
{
allowMovement = true;
}
else
{
if (touchReserved)
{
TouchInjection.TouchUp();
touchReserved = false;
}
allowMovement =
indexFinger.foldedness > 60
&& middleFinger.foldedness > 75
&& ringFinger.foldedness > 75
&& pinkyFinger.foldedness > 60;
}
PXCMHandData.ExtremityData leftData;
PXCMHandData.ExtremityData rightData;
hand.QueryExtremityPoint(PXCMHandData.ExtremityType.EXTREMITY_LEFTMOST, out leftData);
hand.QueryExtremityPoint(PXCMHandData.ExtremityType.EXTREMITY_RIGHTMOST, out rightData);
// Hand width less than 100px
bool allowScroll = Math.Abs(leftData.pointImage.x - rightData.pointImage.x) < 100;
if (allowMovement || allowScroll)
{
// Calc coordinates ============================================
int x, y; // point position
// X axis
if (data.position.x <= 0)
{
x = 0;
}
else if (data.position.x >= 1)
{
x = AreaWidth;
}
else
{
x = (int) (data.position.x * AreaWidth);
}
// Y axis
if (data.position.y <= 0)
{
y = 0;
}
else if (data.position.y >= 1)
{
y = AreaHeight;
}
else
{
y = (int) (data.position.y * AreaHeight);
}
// Calc coordinates ============================================
if (touchReserved)
{
// Drag touch contact
TouchInjection.UpdateTouchPos(x, y);
}
else if (allowScroll)
{
if (scrollStarted)
{
int xDistance = x - scrollCurrPosX,
yDistance = y - scrollCurrPosY,
xDistanceAbs = Math.Abs(xDistance),
yDistanceAbs = Math.Abs(yDistance);
if (xDistanceAbs > scrollSensitivity || yDistanceAbs > scrollSensitivity)
{
bool isXAxis = xDistanceAbs > yDistanceAbs;
if (isXAxis)
{
TouchInjection.Scroll(CenterPx, CenterPy, xDistance * 4, false);
}
else
{
TouchInjection.Scroll(CenterPx, CenterPy, invertVerticalScroll * (yDistance * 4), true);
}
scrollCurrPosX = x;
scrollCurrPosY = y;
}
}
else
{
scrollCurrPosX = x;
scrollCurrPosY = y;
TouchInjectionCore.TouchInjector.InitializeTouchInjection();
scrollStarted = true;
}
}
else
{
if (scrollStarted)
{
MouseInjection.ReturnPointer(AreaWidth, AreaHeight);
scrollStarted = false;
}
// Move cursor to position
MouseInjection.SetCursorPos(x, y);
if (!MouseInjection.IsCursorVisible())
{
MouseInjection.ReturnPointer(AreaWidth, AreaHeight);
}
}
}
}
}
}
}
private void InitRealSense()
{
this.releaseInstances();
errorManager = new ErrorManager();
this.manager = PXCMSenseManager.CreateInstance();
if (manager == null)
{
errorManager.InitError("PXCMSenseManager::Create");
}
// Init hand module
errorManager.RunWithStatusCheck(() => manager.EnableHand(), "PXCMHandModule");
this.handModule = this.manager.QueryHand();
this.handData = this.handModule.CreateOutput();
// config hand module
var hconfig = this.handModule.CreateActiveConfiguration();
hconfig.EnableTrackedJoints(true);
//Gestures
hconfig.DisableAllGestures();
//alerts========================================================
hconfig.DisableAllAlerts();
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_CALIBRATED);
//hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_DETECTED);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
// For hand tracking
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS);
// For user info
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TOO_FAR);
hconfig.SubscribeAlert(OnAlertFired);
//alerts=========================================================
hconfig.ApplyChanges();
hconfig.Dispose();
// Init touchless controller:: Move tracking
errorManager.RunWithStatusCheck(() => manager.EnableTouchlessController(), "TouchlessController");
this.touchController = this.manager.QueryTouchlessController();
// On processed frame callback
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler {
onModuleProcessedFrame = OnFrameProcessed
};
// UX events callback
this.touchController.SubscribeEvent(OnTouchlessControllerUxEvent);
// Init Sense manager
errorManager.RunWithStatusCheck(() => manager.Init(handler), "PXCMSenseManager::Init");
errorManager.RunWithStatusCheck(() => manager.StreamFrames(false), "PXCMSenseManager::StreamFrames");
if (this.manager.captureManager == null || this.manager.captureManager.device == null)
{
throw new NoDeviceException();
}
// Query scene info
PXCMCapture.Sample sample = null;
do {
sample = manager.QuerySample();
}
while (sample == null || sample.depth == null || sample.depth.info == null);
sceneXCenter = sample.depth.info.width / 2;
sceneYCenter = sample.depth.info.height / 2;
//Set delay before close gesture ability
Task.Factory.StartNew(() =>
{
Thread.Sleep(3000);
delayForCloseAchived = true;
this.handData.Update();
handInsideBorders = this.handData.QueryNumberOfHands() > 0;
});
// Hand tracking started.
started = true;
}
