private bool getGestureState(PXCMHandData.BodySideType pSide)
{
var numGestures = mHand.QueryFiredGesturesNumber();
for (int j = 0; j < numGestures; ++j)
{
PXCMHandData.GestureData gestData;
if (mHand.QueryFiredGestureData(j, out gestData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand hand;
if (mHand.QueryHandDataById(gestData.handId, out hand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
var gestName = gestData.name;
if (gestName.Contains("cursor_click"))
{
if (hand.QueryBodySide() == pSide)
{
return(true);
}
}
}
}
}
return(false);
}
// Update is called once per frame
void Update()
{
/* Make sure PXCMSenseManager Instance is Initialized */
if (SenseToolkitManager.Instance.SenseManager == null)
{
return;
}
/* Wait until any frame data is available true(aligned) false(unaligned) */
if (SenseToolkitManager.Instance.SenseManager.AcquireFrame(false, 0) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve am instance of hand tracking Module */
handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve an instance of hand tracking Data */
PXCMHandData _outputData = handAnalyzer.CreateOutput();
if (_outputData != null)
{
_outputData.Update();
//Retrieve joint data, gesture recognition data and alert notification data
//refer to next section
//AcquireFrame
/* Retrieve Gesture Data */
PXCMHandData.GestureData _gestureData;
for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++)
{
if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
//Display the gestures: explained in rendering the frame section
if ((_gestureData.name == "spreadfingers") || (_gestureData.name == "tap") ||
(_gestureData.name == "swipe"))
{
CursorController.isHandClicked = false;
}
else if ((_gestureData.name == "fist") || (_gestureData.name == "full_pinch") ||
(_gestureData.name == "thumb_down") || (_gestureData.name == "thumb_up") ||
(_gestureData.name == "two_fingers_pinch_open") || (_gestureData.name == "v_sign"))
{
CursorController.isHandClicked = true;
}
else if (_gestureData.name == "wave")
{
transform.position = new Vector3(0, 0, 0);
}
}
}
}
}
/* Realease the frame to process the next frame */
SenseToolkitManager.Instance.SenseManager.ReleaseFrame();
}
/* Displaying current frame gestures */
private void DisplayGesture(PXCMHandData handAnalysis, int frameNumber)
{
int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber();
string gestureStatusLeft = string.Empty;
string gestureStatusRight = string.Empty;
if (firedGesturesNumber == 0)
{
return;
}
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand handData;
if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
PXCMHandData.BodySideType bodySideType = handData.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
gestureStatusLeft += "Left Hand Gesture: " + gestureData.name;
NextPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval());
PreviousPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval());
FirstPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval());
EndPageGesture.Check(gestureData.name, Hand.LeftHand, form.GetInterval());
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
gestureStatusRight += "Right Hand Gesture: " + gestureData.name;
NextPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval());
PreviousPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval());
FirstPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval());
EndPageGesture.Check(gestureData.name, Hand.RightHand, form.GetInterval());
}
}
}
if (gestureStatusLeft == String.Empty)
{
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen);
}
else
{
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen);
}
}
/* Displaying current frame gestures */
private void DisplayGesture(PXCMHandData handAnalysis,int frameNumber)
{
int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber();
string gestureStatusLeft = string.Empty;
string gestureStatusRight = string.Empty;
if (firedGesturesNumber == 0)
{
return;
}
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand handData;
if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return;
PXCMHandData.BodySideType bodySideType = handData.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
gestureStatusLeft += "Left Hand Gesture: " + gestureData.name;
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
gestureStatusRight += "Right Hand Gesture: " + gestureData.name;
}
}
}
if (gestureStatusLeft == String.Empty)
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen);
else
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen);
}
// Update is called once per frame
void Update()
{
handData = SenseToolkitManager.Instance.HandDataOutput;
if (handData != null)
{
handData.Update();
for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++)
{
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayGestures(gestureData);
}
}
PXCMHandData.AlertData alertData;
for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++)
{
if (handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ProcessAlerts(alertData);
}
}
}
}
void handleGestures()
{
PXCMHandData.GestureData gestureData;
for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++)
{
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (gestureCooldown <= 0)
{
if (gestureData.name == "v_sign")
{
showColour = !showColour;
gestureCooldown = 2.5f;
}
else if (gestureData.name == "thumb_down")
{
nextEffect();
gestureCooldown = 2.5f;
}
}
}
}
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
{
return;
}
/* Wait until any frame data is available */
if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
/* Retrieve Gesture Data to manipulate GUIText */
PXCMHandData.GestureData gestureData;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
{
if (_handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayGestures(gestureData);
}
}
/* Retrieve Alert Data to manipulate GUIText */
PXCMHandData.AlertData alertData;
for (int i = 0; i < _handData.QueryFiredAlertsNumber(); i++)
{
if (_handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ProcessAlerts(alertData);
}
}
/* Retrieve all joint Data */
for (int i = 0; i < _handData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
for (int j = 0; j < MaxJoints; j++)
{
if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i][j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
jointData[i][j] = null;
}
}
if (!handList.ContainsKey(_iHand.QueryUniqueId()))
{
handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide());
}
}
}
/* Smoothen and Display the Data - Joints and Bones*/
DisplayJoints();
}
handAnalyzer.Dispose();
}
sm.ReleaseFrame();
RotateCam();
}
// Update is called once per frame
void Update()
{
// For testing only.
//handRenderer.queryLeftHand2DCoordinates ();
_outputData = SenseToolkitManager.Instance.HandDataOutput;
if (_outputData != null)
{
_outputData.Update();
/* Retrieve Hand Joints*/
joints = new PXCMHandData.JointData[2, PXCMHandData.NUMBER_OF_JOINTS];
for (int i = 0; i < _outputData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _handData;
_outputData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _handData);
//Debug.Log(_handData.IsCalibrated().ToString());
//if(_handData!=null){
//else { Debug.Log ("No data"); continue; }
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
_handData.QueryTrackedJoint((PXCMHandData.JointType)j, out joints[i, j]);
}
handIds[i] = _handData.QueryUniqueId();
bodySides[i] = _handData.QueryBodySide();
if (!_handData.IsCalibrated())
{
if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
isleftCalibrated = false;
}
else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
isrightCalibrated = false;
}
}
else
{
if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
isleftCalibrated = true;
}
else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
isrightCalibrated = true;
}
}
}
handRenderer.DisplaySmoothenedJoints(_outputData, joints, handIds, bodySides, _outputData.QueryNumberOfHands());
/* Retrieve Alert Data */
/*
* PXCMHandData.AlertData _alertData;
* for (int i = 0; i < _outputData.QueryFiredAlertsNumber(); i++)
* if (_outputData.QueryFiredAlertData(i, out _alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
* handRenderer.DisplayAlerts(_alertData);
*/
/* Retrieve Gesture Data */
PXCMHandData.GestureData _gestureData;
for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++)
{
if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
handRenderer.DisplayGestures(_gestureData);
}
}
if (_outputData.QueryNumberOfHands() == 0)
{
handRenderer.makeNull();
}
}
else
{
handRenderer.makeNull();
}
// For testing
//handRenderer.DisplayGest ();
displayHandGestures();
}
/* Displaying current frame gestures */
private int DisplayGesture(PXCMHandData handAnalysis)
{
int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber();
string gestureStatusLeft = string.Empty;
string gestureStatusRight = string.Empty;
if (firedGesturesNumber == 0)
{
return -1;
}
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand handData;
if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return -1;
PXCMHandData.BodySideType bodySideType = handData.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
gestureStatusLeft += gestureData.name;
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
gestureStatusRight += gestureData.name;
}
}
}
int value = -1;
if (gestureStatusLeft == "spreadfingers" && gestureStatusRight == "spreadfingers")
{
value = 0;
}
else if (gestureStatusLeft == "spreadfingers" && gestureStatusRight == "fist")
{
value = 1;
}
else if (gestureStatusLeft == "fist" && gestureStatusRight == "spreadfingers")
{
value = 2;
}
else if (gestureStatusLeft == "fist" && gestureStatusRight == "fist")
{
value = 3;
}
if (value != -1)
{
//form.UpdateInfo("Gesture: " + value + "\n", Color.SeaGreen);
return value;
}
return -1;
}
private static void ProcessGestures(PXCMHandData handData)
{
// Processing gestures
// Querying how many gestures were detected
int firedGesturesNumber = handData.QueryFiredGesturesNumber();
// Querying which gestures were detected
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Getting the gesture name
String gestureName = gestureData.name;
// Getting the gesture state
PXCMHandData.GestureStateType gestureState = gestureData.state;
// Getting the gesture timestamp
long timestamp = gestureData.timeStamp;
// Getting the hand information about which hand fired the gesture
PXCMHandData.IHand handInfo;
if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Getting which hand fired the gesture
PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide();
if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START)
{
Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand);
Console.WriteLine("--------------------");
if (gestureName.CompareTo(GESTURE_FIST) == 0)
{
if (currentGesture.CompareTo(GESTURE_FIST) != 0)
{
//SendMqttMessage("surfboard2/relay?0");
String target = (String) iot_form.comboTarget.SelectedItem;
String hclosed = (String) iot_form.comboHandsClosed.SelectedItem;
//if (target.Equals("")) target = "*";
SendMqttMessage(target + "/" + hclosed);
SendMqttMessage(hclosed);
currentGesture = GESTURE_FIST;
}
}
else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0)
{
if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0)
{
String target = (String)iot_form.comboTarget.SelectedItem;
String hopen = (String)iot_form.comboGive5Action.SelectedItem;
//if (target.Equals("")) target = "*";
SendMqttMessage(hopen);
SendMqttMessage(target + "/" + hopen);
//System.Diagnostics.Process.Start("http://www.globalcode.com.br");
//SendMqttMessage("surfboard2/relay?1");
currentGesture = GESTURE_SPREADFINGERS;
}
}
else if (gestureName.CompareTo(GESTURE_CLICK) == 0)
{
if (currentGesture.CompareTo(GESTURE_CLICK) != 0)
{
SendMqttMessage("green?255");
currentGesture = GESTURE_CLICK;
}
}
else if (gestureName.CompareTo(GESTURE_VSIGN) == 0)
{
if (currentGesture.CompareTo(GESTURE_VSIGN) != 0)
{
//System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U");
SendMqttMessage("green?0");
keepLooping = false;
currentGesture = GESTURE_VSIGN;
}
}
}
}
}
}
private static void ProcessGestures(PXCMHandData handData)
{
// Processing gestures
// Querying how many gestures were detected
int firedGesturesNumber = handData.QueryFiredGesturesNumber();
// Querying which gestures were detected
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Getting the gesture name
String gestureName = gestureData.name;
// Getting the gesture state
PXCMHandData.GestureStateType gestureState = gestureData.state;
// Getting the gesture timestamp
long timestamp = gestureData.timeStamp;
// Getting the hand information about which hand fired the gesture
PXCMHandData.IHand handInfo;
if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Getting which hand fired the gesture
PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide();
if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START)
{
Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand);
Console.WriteLine("--------------------");
if (gestureName.CompareTo(GESTURE_FIST) == 0)
{
if (currentGesture.CompareTo(GESTURE_FIST) != 0)
{
//SendMqttMessage("surfboard2/relay?0");
String target = (String)iot_form.comboTarget.SelectedItem;
String hclosed = (String)iot_form.comboHandsClosed.SelectedItem;
//if (target.Equals("")) target = "*";
//SendMqttMessage(target + "/" + hclosed);
SendMqttMessage(hclosed);
currentGesture = GESTURE_FIST;
}
}
else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0)
{
if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0)
{
String target = (String)iot_form.comboTarget.SelectedItem;
String hopen = (String)iot_form.comboGive5Action.SelectedItem;
//if (target.Equals("")) target = "*";
SendMqttMessage(hopen);
//SendMqttMessage(target + "/" + hopen);
//System.Diagnostics.Process.Start("http://www.globalcode.com.br");
//SendMqttMessage("surfboard2/relay?1");
currentGesture = GESTURE_SPREADFINGERS;
}
}
else if (gestureName.CompareTo(GESTURE_CLICK) == 0)
{
if (currentGesture.CompareTo(GESTURE_CLICK) != 0)
{
SendMqttMessage("green?255");
currentGesture = GESTURE_CLICK;
}
}
else if (gestureName.CompareTo(GESTURE_VSIGN) == 0)
{
if (currentGesture.CompareTo(GESTURE_VSIGN) != 0)
{
//System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U");
SendMqttMessage("green?0");
keepLooping = false;
currentGesture = GESTURE_VSIGN;
}
}
}
}
}
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
CanvasHandParts.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 指の関節を列挙する
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
PXCMHandData.JointData jointData;
sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
AddEllipse(CanvasHandParts,
new Point(jointData.positionImage.x, jointData.positionImage.y),
5, Brushes.Green);
}
}
// 認識したジェスチャーの数を取得する
var numOfGestures = handData.QueryFiredGesturesNumber();
for (int i = 0; i < numOfGestures; i++)
{
// 認識したジェスチャーを取得する
PXCMHandData.GestureData gesture;
var sts = handData.QueryFiredGestureData(i, out gesture);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// ジェスチャーをした手を取得する
PXCMHandData.IHand hand;
sts = handData.QueryHandDataById(gesture.handId, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// どちらの手でジェスチャーしたのか
var side = hand.QueryBodySide();
if (side == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
++leftGestureCount;
}
else
{
++rightGestureCount;
}
}
TextLeftGesture.Text = string.Format("Left gesture : {0}", leftGestureCount);
TextRightGesture.Text = string.Format("Right gesture : {0}", rightGestureCount);
}
