private bool getGestureState(PXCMHandData.BodySideType pSide) { var numGestures = mHand.QueryFiredGesturesNumber(); for (int j = 0; j < numGestures; ++j) { PXCMHandData.GestureData gestData; if (mHand.QueryFiredGestureData(j, out gestData) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMHandData.IHand hand; if (mHand.QueryHandDataById(gestData.handId, out hand) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { var gestName = gestData.name; if (gestName.Contains("cursor_click")) { if (hand.QueryBodySide() == pSide) { return(true); } } } } } return(false); }
// Update is called once per frame void Update() { /* Make sure PXCMSenseManager Instance is Initialized */ if (SenseToolkitManager.Instance.SenseManager == null) { return; } /* Wait until any frame data is available true(aligned) false(unaligned) */ if (SenseToolkitManager.Instance.SenseManager.AcquireFrame(false, 0) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve am instance of hand tracking Module */ handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand(); if (handAnalyzer != null) { /* Retrieve an instance of hand tracking Data */ PXCMHandData _outputData = handAnalyzer.CreateOutput(); if (_outputData != null) { _outputData.Update(); //Retrieve joint data, gesture recognition data and alert notification data //refer to next section //AcquireFrame /* Retrieve Gesture Data */ PXCMHandData.GestureData _gestureData; for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++) { if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { //Display the gestures: explained in rendering the frame section if ((_gestureData.name == "spreadfingers") || (_gestureData.name == "tap") || (_gestureData.name == "swipe")) { CursorController.isHandClicked = false; } else if ((_gestureData.name == "fist") || (_gestureData.name == "full_pinch") || (_gestureData.name == "thumb_down") || (_gestureData.name == "thumb_up") || (_gestureData.name == "two_fingers_pinch_open") || (_gestureData.name == "v_sign")) { CursorController.isHandClicked = true; } else if (_gestureData.name == "wave") { transform.position = new Vector3(0, 0, 0); } } } } } /* Realease the frame to process the next frame */ SenseToolkitManager.Instance.SenseManager.ReleaseFrame(); }
/* Displaying current frame gestures */ private void DisplayGesture(PXCMHandData handAnalysis, int frameNumber) { int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber(); string gestureStatusLeft = string.Empty; string gestureStatusRight = string.Empty; if (firedGesturesNumber == 0) { return; } for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMHandData.IHand handData; if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } PXCMHandData.BodySideType bodySideType = handData.QueryBodySide(); if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { gestureStatusLeft += "Left Hand Gesture: " + gestureData.name; NextPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval()); PreviousPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval()); FirstPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval()); EndPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval()); } else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { gestureStatusRight += "Right Hand Gesture: " + gestureData.name; NextPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval()); PreviousPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval()); FirstPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval()); EndPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval()); } } } if (gestureStatusLeft == String.Empty) { form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen); } else { form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen); } }
/* Displaying current frame gestures */ private void DisplayGesture(PXCMHandData handAnalysis,int frameNumber) { int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber(); string gestureStatusLeft = string.Empty; string gestureStatusRight = string.Empty; if (firedGesturesNumber == 0) { return; } for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMHandData.IHand handData; if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR) return; PXCMHandData.BodySideType bodySideType = handData.QueryBodySide(); if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { gestureStatusLeft += "Left Hand Gesture: " + gestureData.name; } else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { gestureStatusRight += "Right Hand Gesture: " + gestureData.name; } } } if (gestureStatusLeft == String.Empty) form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen); else form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen); }
// Update is called once per frame void Update() { handData = SenseToolkitManager.Instance.HandDataOutput; if (handData != null) { handData.Update(); for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++) { if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayGestures(gestureData); } } PXCMHandData.AlertData alertData; for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++) { if (handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { ProcessAlerts(alertData); } } } }
void handleGestures() { PXCMHandData.GestureData gestureData; for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++) { if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (gestureCooldown <= 0) { if (gestureData.name == "v_sign") { showColour = !showColour; gestureCooldown = 2.5f; } else if (gestureData.name == "thumb_down") { nextEffect(); gestureCooldown = 2.5f; } } } } }
// Update is called once per frame void Update() { /* Make sure SenseManager Instance is valid */ if (sm == null) { return; } /* Wait until any frame data is available */ if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve hand tracking Module Instance */ handAnalyzer = sm.QueryHand(); if (handAnalyzer != null) { /* Retrieve hand tracking Data */ PXCMHandData _handData = handAnalyzer.CreateOutput(); if (_handData != null) { _handData.Update(); /* Retrieve Gesture Data to manipulate GUIText */ PXCMHandData.GestureData gestureData; for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++) { if (_handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayGestures(gestureData); } } /* Retrieve Alert Data to manipulate GUIText */ PXCMHandData.AlertData alertData; for (int i = 0; i < _handData.QueryFiredAlertsNumber(); i++) { if (_handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { ProcessAlerts(alertData); } } /* Retrieve all joint Data */ for (int i = 0; i < _handData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _iHand; if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR) { for (int j = 0; j < MaxJoints; j++) { if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i][j]) != pxcmStatus.PXCM_STATUS_NO_ERROR) { jointData[i][j] = null; } } if (!handList.ContainsKey(_iHand.QueryUniqueId())) { handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide()); } } } /* Smoothen and Display the Data - Joints and Bones*/ DisplayJoints(); } handAnalyzer.Dispose(); } sm.ReleaseFrame(); RotateCam(); }
// Update is called once per frame void Update() { // For testing only. //handRenderer.queryLeftHand2DCoordinates (); _outputData = SenseToolkitManager.Instance.HandDataOutput; if (_outputData != null) { _outputData.Update(); /* Retrieve Hand Joints*/ joints = new PXCMHandData.JointData[2, PXCMHandData.NUMBER_OF_JOINTS]; for (int i = 0; i < _outputData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _handData; _outputData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _handData); //Debug.Log(_handData.IsCalibrated().ToString()); //if(_handData!=null){ //else { Debug.Log ("No data"); continue; } for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { _handData.QueryTrackedJoint((PXCMHandData.JointType)j, out joints[i, j]); } handIds[i] = _handData.QueryUniqueId(); bodySides[i] = _handData.QueryBodySide(); if (!_handData.IsCalibrated()) { if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { isleftCalibrated = false; } else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { isrightCalibrated = false; } } else { if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { isleftCalibrated = true; } else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { isrightCalibrated = true; } } } handRenderer.DisplaySmoothenedJoints(_outputData, joints, handIds, bodySides, _outputData.QueryNumberOfHands()); /* Retrieve Alert Data */ /* * PXCMHandData.AlertData _alertData; * for (int i = 0; i < _outputData.QueryFiredAlertsNumber(); i++) * if (_outputData.QueryFiredAlertData(i, out _alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) * handRenderer.DisplayAlerts(_alertData); */ /* Retrieve Gesture Data */ PXCMHandData.GestureData _gestureData; for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++) { if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { handRenderer.DisplayGestures(_gestureData); } } if (_outputData.QueryNumberOfHands() == 0) { handRenderer.makeNull(); } } else { handRenderer.makeNull(); } // For testing //handRenderer.DisplayGest (); displayHandGestures(); }
/* Displaying current frame gestures */ private int DisplayGesture(PXCMHandData handAnalysis) { int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber(); string gestureStatusLeft = string.Empty; string gestureStatusRight = string.Empty; if (firedGesturesNumber == 0) { return -1; } for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMHandData.IHand handData; if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR) return -1; PXCMHandData.BodySideType bodySideType = handData.QueryBodySide(); if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { gestureStatusLeft += gestureData.name; } else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { gestureStatusRight += gestureData.name; } } } int value = -1; if (gestureStatusLeft == "spreadfingers" && gestureStatusRight == "spreadfingers") { value = 0; } else if (gestureStatusLeft == "spreadfingers" && gestureStatusRight == "fist") { value = 1; } else if (gestureStatusLeft == "fist" && gestureStatusRight == "spreadfingers") { value = 2; } else if (gestureStatusLeft == "fist" && gestureStatusRight == "fist") { value = 3; } if (value != -1) { //form.UpdateInfo("Gesture: " + value + "\n", Color.SeaGreen); return value; } return -1; }
private static void ProcessGestures(PXCMHandData handData) { // Processing gestures // Querying how many gestures were detected int firedGesturesNumber = handData.QueryFiredGesturesNumber(); // Querying which gestures were detected for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { // Getting the gesture name String gestureName = gestureData.name; // Getting the gesture state PXCMHandData.GestureStateType gestureState = gestureData.state; // Getting the gesture timestamp long timestamp = gestureData.timeStamp; // Getting the hand information about which hand fired the gesture PXCMHandData.IHand handInfo; if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Getting which hand fired the gesture PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide(); if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START) { Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand); Console.WriteLine("--------------------"); if (gestureName.CompareTo(GESTURE_FIST) == 0) { if (currentGesture.CompareTo(GESTURE_FIST) != 0) { //SendMqttMessage("surfboard2/relay?0"); String target = (String) iot_form.comboTarget.SelectedItem; String hclosed = (String) iot_form.comboHandsClosed.SelectedItem; //if (target.Equals("")) target = "*"; SendMqttMessage(target + "/" + hclosed); SendMqttMessage(hclosed); currentGesture = GESTURE_FIST; } } else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0) { if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0) { String target = (String)iot_form.comboTarget.SelectedItem; String hopen = (String)iot_form.comboGive5Action.SelectedItem; //if (target.Equals("")) target = "*"; SendMqttMessage(hopen); SendMqttMessage(target + "/" + hopen); //System.Diagnostics.Process.Start("http://www.globalcode.com.br"); //SendMqttMessage("surfboard2/relay?1"); currentGesture = GESTURE_SPREADFINGERS; } } else if (gestureName.CompareTo(GESTURE_CLICK) == 0) { if (currentGesture.CompareTo(GESTURE_CLICK) != 0) { SendMqttMessage("green?255"); currentGesture = GESTURE_CLICK; } } else if (gestureName.CompareTo(GESTURE_VSIGN) == 0) { if (currentGesture.CompareTo(GESTURE_VSIGN) != 0) { //System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U"); SendMqttMessage("green?0"); keepLooping = false; currentGesture = GESTURE_VSIGN; } } } } } }
private static void ProcessGestures(PXCMHandData handData) { // Processing gestures // Querying how many gestures were detected int firedGesturesNumber = handData.QueryFiredGesturesNumber(); // Querying which gestures were detected for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { // Getting the gesture name String gestureName = gestureData.name; // Getting the gesture state PXCMHandData.GestureStateType gestureState = gestureData.state; // Getting the gesture timestamp long timestamp = gestureData.timeStamp; // Getting the hand information about which hand fired the gesture PXCMHandData.IHand handInfo; if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Getting which hand fired the gesture PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide(); if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START) { Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand); Console.WriteLine("--------------------"); if (gestureName.CompareTo(GESTURE_FIST) == 0) { if (currentGesture.CompareTo(GESTURE_FIST) != 0) { //SendMqttMessage("surfboard2/relay?0"); String target = (String)iot_form.comboTarget.SelectedItem; String hclosed = (String)iot_form.comboHandsClosed.SelectedItem; //if (target.Equals("")) target = "*"; //SendMqttMessage(target + "/" + hclosed); SendMqttMessage(hclosed); currentGesture = GESTURE_FIST; } } else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0) { if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0) { String target = (String)iot_form.comboTarget.SelectedItem; String hopen = (String)iot_form.comboGive5Action.SelectedItem; //if (target.Equals("")) target = "*"; SendMqttMessage(hopen); //SendMqttMessage(target + "/" + hopen); //System.Diagnostics.Process.Start("http://www.globalcode.com.br"); //SendMqttMessage("surfboard2/relay?1"); currentGesture = GESTURE_SPREADFINGERS; } } else if (gestureName.CompareTo(GESTURE_CLICK) == 0) { if (currentGesture.CompareTo(GESTURE_CLICK) != 0) { SendMqttMessage("green?255"); currentGesture = GESTURE_CLICK; } } else if (gestureName.CompareTo(GESTURE_VSIGN) == 0) { if (currentGesture.CompareTo(GESTURE_VSIGN) != 0) { //System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U"); SendMqttMessage("green?0"); keepLooping = false; currentGesture = GESTURE_VSIGN; } } } } } }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する CanvasHandParts.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 指の関節を列挙する for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { PXCMHandData.JointData jointData; sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } AddEllipse(CanvasHandParts, new Point(jointData.positionImage.x, jointData.positionImage.y), 5, Brushes.Green); } } // 認識したジェスチャーの数を取得する var numOfGestures = handData.QueryFiredGesturesNumber(); for (int i = 0; i < numOfGestures; i++) { // 認識したジェスチャーを取得する PXCMHandData.GestureData gesture; var sts = handData.QueryFiredGestureData(i, out gesture); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // ジェスチャーをした手を取得する PXCMHandData.IHand hand; sts = handData.QueryHandDataById(gesture.handId, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // どちらの手でジェスチャーしたのか var side = hand.QueryBodySide(); if (side == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { ++leftGestureCount; } else { ++rightGestureCount; } } TextLeftGesture.Text = string.Format("Left gesture : {0}", leftGestureCount); TextRightGesture.Text = string.Format("Right gesture : {0}", rightGestureCount); }