/// <summary>
/// End of gesture movement callback.
/// </summary>
/// <param name='data'>
/// Data of the gesture gathered.
/// </param>
public void endOfGestureCallback(GestureData data)
{
GestureUtils.EnumGestures gResult;
Debug.Log("Gesture ended!");
List<Vector2> pointList = new List<Vector2>();
GestureData normalizedData = data.getNormalizedGestureData(CANVAS_SIZE);
GestureFrame previousFrame = (GestureFrame)normalizedData.frames[0];
pointList.Add(new Vector2(previousFrame.position.x, previousFrame.position.y));
for (int i = 1; i < normalizedData.frames.Count; i++) {
GestureFrame currentFrame = (GestureFrame)normalizedData.frames[i];
if ((Math.Abs(currentFrame.position.x - previousFrame.position.x) < NORMAL_FRAME_VARIATION) &&
(Math.Abs(currentFrame.position.y - previousFrame.position.y) < NORMAL_FRAME_VARIATION)) {
pointList.Add(new Vector2(currentFrame.position.x, currentFrame.position.y));
previousFrame = currentFrame;
}
}
//At gesture end, draw the normalized data.
Vector2 previousPoint = pointList[0];
for (int i = 1; i < pointList.Count; i++) {
Vector2 currentPoint = pointList[i];
DrawLine(canvas, previousPoint.x, previousPoint.y, currentPoint.x, currentPoint.y, Color.black);
previousPoint = currentPoint;
}
canvas.Apply();
//The checker method only receives canvas if you want the points used drawn on the screen,
//can just receive a points list otherwise.
gResult = gu.GestureChecker(pointList, canvas, CANVAS_SIZE);
switch(gResult) {
case GestureUtils.EnumGestures.DownZigZag:
Debug.Log("Downwards Zig Zag!");
status = "Downwards Zig Zag!";
break;
case GestureUtils.EnumGestures.RightZigZag:
Debug.Log("Rightwards Zig Zag!");
status = "Rightwards Zig Zag!";
break;
case GestureUtils.EnumGestures.Square:
Debug.Log("Square!");
status = "Square!";
break;
default:
Debug.Log("No Gesture found.");
status = "Nothing.";
break;
}
}
public int AddToQueue(GestureData data)
{
//Decrement the semaphore to make sure the spot is available
this.requestGestureDataSem.WaitOne();
lock (_threadLock)
{
dataQueue.Enqueue(data);
}
//Increament the semaphore to indicate there is work to do
int previousCount = handleRequests.Release();
return previousCount;
}
private void OnClick(GestureData gestureData)
{
int unitUid = _order.units[_order.units.Count - 1];
var map = BattleProcedure.CurSession.Map;
var mapGrid = map.GetMapGridByScreenPos(gestureData.touchPos);
if (mapGrid != null)
{
var goal = mapGrid.Coord;
_path = map.FindPath2Goal(unitUid, goal);
if (_path.Count > _count + 1)
{
_path.RemoveRange(_count + 1, _path.Count - _count - 1);
}
map.ShowPath(_path, true);
_order.paths.Add(_path);
bFinish = true;
}
}
public void SendSpellToServer(RecognitionResult result, GestureData gestureData)
{
Debug.Log("SendSpellToServer");
if (gestureData != null)
{
SpellCastToJson values = new SpellCastToJson();
if (!testMode)
{
values.match_id = SessionData.Match.MatchID;
}
values.gestureData = gestureData;
/*foreach (GestureLine line in gestureData.lines)
* {
* foreach (Vector2 point in line.points)
* {
* values.x.Add(point.x);
* values.y.Add(point.y);
* }
* }*/
if (socket)
{
if (result.gesture == attackSpell)
{
JSONObject dataObj = new JSONObject(values.SaveToString());
//Debug.Log(dataObj);
socket.Emit("player:cast:spell", dataObj);
}
else if (result.gesture == defendSpell)
{
Debug.Log("defend");
JSONObject dataObj = new JSONObject(values.SaveToString());
//Debug.Log(dataObj);
socket.Emit("player:defend:spell", dataObj);
}
}
}
}
//检测姿势是否完成
private static void CheckPoseComplete(ref GestureData gestureData, float timestamp, Vector3 jointPos, bool isInPose, float durationToComplete)
{
if (isInPose)
{
float timeLeft = timestamp - gestureData.timestamp;
gestureData.progress = durationToComplete > 0f ? Mathf.Clamp01(timeLeft / durationToComplete) : 1.0f;
if (timeLeft >= durationToComplete)
{
gestureData.timestamp = timestamp;
gestureData.jointPos = jointPos;
gestureData.state++;
gestureData.complete = true;
}
}
else
{
SetGestureCancelled(ref gestureData);
}
}
void Start()
{
shootingGestureData = new GestureData {
downPoint = new Vector3(0, 0, 0),
dragPoint = new Vector3(0, 0, 0),
index = -1
};
movementGestureData = new GestureData {
downPoint = new Vector3(0, 0, 0),
dragPoint = new Vector3(0, 0, 0),
index = -1
};
TextAsset[] gesturesXml = Resources.LoadAll <TextAsset>("GestureSet/10-stylus-MEDIUM/");
foreach (TextAsset gestureXml in gesturesXml)
{
trainingSet.Add(GestureIO.ReadGestureFromXML(gestureXml.text));
}
}
private void OnLongTap(GestureData data)
{
if (selectUnit != null)
{
Vector3 tapMapPos;
MapCoordinates tapMapCoord;
//检测 长按的地图位是否包含有效单位
if (GameOperation.Instance.MapPosSelector(data.pos, out tapMapPos))
{
tapMapCoord = MapCoordinates.FromPosition(tapMapPos);
if (selectPath.Count > 0)
{
MapCoordinates pathEnd = selectPath[selectPath.Count - 1];
var addPath = MapManager.Instance.FindPath(pathEnd, tapMapCoord);
var astarPath = MapManager.Instance.FindPath(selectUnit.pos, tapMapCoord);
if ((selectPath.Count + addPath.Count - 1) > astarPath.Count)
{
selectPath = astarPath;
}
else
{
for (int i = 1; i < addPath.Count; ++i)
{
selectPath.Add(addPath[i]);
}
}
}
else
{
var astarPath = MapManager.Instance.FindPath(selectUnit.pos, tapMapCoord);
selectPath = astarPath;
}
}
}
}
public void Process(Vector2 position)
{
if (!_processor.InputZone.Contains(position))
{
Finish();
}
else if (_currentDirection == Directions.None)
{
float distance = Vector2.Distance(_initialPosition, position);
if (distance > _minimumAmplitude)
{
GestureData data = GetGestureData(_initialPosition, position);
_currentDirection = data.direction;
// handle gesture via interface
if (_processor != null)
{
_processor.ProcessGesture(data, distance);
}
}
}
}
//=========================== event ================================
private void OnLongTapDown(GestureData data)
{
Vector3 tapMapPos;
MapCoordinates tapMapCoord;
Unit unit;
//检测 长按的地图位是否包含有效单位
if (GameOperation.Instance.MapPosSelector(data.pos, out tapMapPos))
{
tapMapCoord = MapCoordinates.FromPosition(tapMapPos);
if (MapManager.Instance.FindUnit(tapMapCoord, out unit))
{
selectUnit = unit;
SwitchOperationMode(BattleOperationMode.SelectPath);
}
else
{
return;
}
}
;
}
void Update()
{
if (Input.GetKeyDown(KeyCode.Escape))
{
OnEsc?.Invoke(KeyCode.Escape);
}
if (Input.touchCount == 0)
{
_isMoving = false;
}
else
{
foreach (var touch in Input.touches)
{
//register new input
if (touch.phase == TouchPhase.Began)
{
//prevent UI taps
if (EventSystem.current.IsPointerOverGameObject(touch.fingerId))
{
continue;
}
//prevent taps while drag
if (_isMoving)
{
continue;
}
GestureData gestureData = new GestureData();
gestureData.fingerId = touch.fingerId;
gestureData.Type = GestureType.None;
gestureData.startPosition = touch.position;
gestureData.endPosition = touch.position;
gestureData.deltaPosition = Vector2.zero;
gestureData.time = 0;
_gestures.Add(gestureData);
++count;
}
else //process existing input
{
int index = FindTouchIndexById(touch.fingerId);
if (index == -1)
{
continue;
}
_gestures[index].endPosition = touch.position;
_gestures[index].deltaPosition = touch.deltaPosition;
_gestures[index].time += touch.deltaTime;
float distance = (_gestures[index].endPosition - _gestures[index].startPosition).magnitude;
if (touch.phase == TouchPhase.Moved)
{
if (distance > minDistance || _isMoving)
{
_isMoving = true;
if (_gestures[index].time > maxTime)
{
_gestures[index].Type = GestureType.Drag;
OnDrag.Invoke(_gestures[index]);
}
}
}
else if (touch.phase == TouchPhase.Ended)
{
if (distance < minDistance && _gestures[index].time < maxTime)
{
OnTap.Invoke(_gestures[index]);
}
else if (distance > minDistance && _gestures[index].time < maxTime)
{
_isMoving = false;
OnSwipe.Invoke(_gestures[index]);
}
else if (distance > minDistance && _gestures[index].time > maxTime)
{
_isMoving = false;
OnDrag.Invoke(_gestures[index]);
}
_gestures.RemoveAt(index);
--count;
}
} //end proccess existing input
}
}
}
// estimate the next state and completeness of the gesture
public static void CheckForGesture(long userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
if (gestureData.complete)
{
return;
}
switch (gestureData.gesture)
{
case Gestures.MyRaiseUpRight:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[head]) //右手和头部能被追踪到
{
Hip = jointsPos[head].y;
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[head] && jointsPos[rightHandIndex].y - jointsPos[head].y > 0.15f; //右手比头高一定的距离，这个距离几乎能适应所有年龄断的人
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
}
break;
case Gestures.MyRaiseUpLeft:
switch (gestureData.state)
{
case 0:
if (jointsTracked[leftHandIndex] && jointsTracked[head]) //右手和头部能被追踪到
{
Hip = jointsPos[head].y;
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[head] && jointsPos[leftHandIndex].y - jointsPos[head].y > 0.15f; //右手比头高一定的距离，这个距离几乎能适应所有年龄断的人
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
}
break;
case Gestures.Myjump:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[AnkleLeftIndex] && jointsTracked[AnkleRightIndex]) //不用脚跟，采用脚踝
{
ankleLeft_y = jointsPos[AnkleLeftIndex].y;
SetGestureJoint(ref gestureData, timestamp, AnkleRightIndex, jointsPos[AnkleRightIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f) //不用脚跟，采用脚踝
{
bool isInPose = jointsTracked[AnkleLeftIndex] && jointsTracked[AnkleRightIndex] &&
(jointsPos[AnkleRightIndex].y - gestureData.jointPos.y) > 0.1f && (jointsPos[AnkleLeftIndex].y - ankleLeft_y) > 0.1f;//脚踝高度差10cm
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.MySquat:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[hipCenterIndex])
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.2f && (jointsPos[hipCenterIndex].y - jointsPos[footRightIndex].y) < 0.4f;//臀部下降0.2m,并且和臀部的距离差值降低.
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.Walk:
switch (gestureData.state)
{
case 0: //走路的第一阶段，左脚右脚一高一低
if (jointsTracked[footRightIndex] && jointsTracked[footleftIndex] &&
Mathf.Abs(jointsPos[footRightIndex].y - jointsPos[footleftIndex].y) > 0.15f)
{
SetGestureJoint(ref gestureData, timestamp, footRightIndex, jointsPos[footRightIndex]);
gestureData.progress = 0.5f; //过程时间为0.5s（不知道有什么用）
}
break;
case 1:
if ((timestamp - gestureData.timestamp) < 0.5f) //如果两个姿势的时间差不超过1秒
{
bool isInPose = jointsTracked[footRightIndex] && jointsTracked[footleftIndex] &&
Mathf.Abs(jointsPos[footRightIndex].y - jointsPos[footleftIndex].y) < 0.03f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.MyMoveLeft:
switch (gestureData.state)
{
case 0:
if (jointsTracked[hipCenterIndex])
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.1f;
}
break;
case 1:
if (timestamp - gestureData.timestamp < 1.0f)
{
bool isInPose = (jointsTracked[hipCenterIndex]) && (gestureData.jointPos.x - jointsPos[hipCenterIndex].x > 0.15f);
;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.MyMoveRight:
switch (gestureData.state)
{
case 0:
if (jointsTracked[hipCenterIndex])
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.1f;
}
break;
case 1:
if (timestamp - gestureData.timestamp < 1.0f)
{
bool isInPose = (jointsTracked[hipCenterIndex]) && (gestureData.jointPos.x - jointsPos[hipCenterIndex].x < -0.15f);
;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
SetGestureCancelled(ref gestureData);
}
break;
}
break;
}
}
private static void CheckPoseComplete(ref GestureData gestureData, float timestamp, Vector3 jointPos, bool isInPose, float durationToComplete)
{
if(isInPose)
{
float timeLeft = timestamp - gestureData.timestamp;
gestureData.progress = durationToComplete > 0f ? Mathf.Clamp01(timeLeft / durationToComplete) : 1.0f;
if(timeLeft >= durationToComplete)
{
gestureData.timestamp = timestamp;
gestureData.jointPos = jointPos;
gestureData.state++;
gestureData.complete = true;
}
}
else
{
SetGestureCancelled(ref gestureData);
}
}
private static void SetGestureJoint(ref GestureData gestureData, float timestamp, int joint, Vector3 jointPos)
{
gestureData.joint = joint;
gestureData.jointPos = jointPos;
gestureData.timestamp = timestamp;
gestureData.state++;
}
/**
@brief Get the details of the fired gesture with the given index.
@param[in] index - the zero-based index of the requested fired gesture.
@param[out] gestureData - the information for the fired gesture.
@note The gesture index must be between 0 and [QueryFiredGesturesNumber() - 1]
@return PXCM_STATUS_NO_ERROR - operation succeeded.
@return PXCM_STATUS_PARAM_UNSUPPORTED - invalid input parameter.
@see GestureData
@see QueryFiredGesturesNumber
*/
public pxcmStatus QueryFiredGestureData(Int32 index, out GestureData gestureData)
{
return QueryFiredGestureDataINT(instance, index, out gestureData);
}
internal static Boolean IsGestureFiredByHandINT(IntPtr instance, String gestureName, Int32 handID, out GestureData gestureData)
{
gestureData = new GestureData();
return(PXCMHandData_IsGestureFiredByHand(instance, gestureName, handID, gestureData));
}
private static extern Boolean PXCMHandData_IsGestureFiredByHand(IntPtr instance, String gestureName, Int32 handID, [Out] GestureData gestureData);
internal static pxcmStatus QueryFiredGestureDataINT(IntPtr instance, Int32 index, out GestureData gestureData)
{
gestureData = new GestureData();
return(PXCMHandData_QueryFiredGestureData(instance, index, gestureData));
}
private static extern pxcmStatus PXCMHandData_QueryFiredGestureData(IntPtr instance, Int32 index, [Out] GestureData gestureData);
// estimate the next state and completeness of the gesture
public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
if(gestureData.complete)
return;
switch(gestureData.gesture)
{
// check for RaiseHand
case Gestures.RaiseHand:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for Psi
case Gestures.Psi:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for Wave
case Gestures.Wave:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture - phase 2
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f;
if(isInPose)
{
gestureData.timestamp = timestamp;
gestureData.state++;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
case 2: // gesture phase 3 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
}
break;
// check for Click
case Gestures.Click:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, as this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, as this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
break;
case 1: // gesture - phase 2
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
//(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
//(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f;
if(isInPose)
{
gestureData.timestamp = timestamp;
gestureData.jointPos = jointsPos[gestureData.joint];
gestureData.state++;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
case 2: // gesture phase 3 = complete
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
//(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
//(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
}
break;
// check for SweepLeft
case Gestures.SweepLeft:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
// else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
// (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, leftHandIndex);
// gestureData.jointPos = jointsPos[leftHandIndex];
// gestureData.progress = 0.5f;
// }
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.2f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
}
break;
// check for SweepRight
case Gestures.SweepRight:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
// (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, rightHandIndex);
// gestureData.jointPos = jointsPos[rightHandIndex];
// gestureData.progress = 0.5f;
// }
// else
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.2f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
gestureData.state = 0;
gestureData.progress = 0f;
}
break;
}
break;
// check for RightHandCursor
case Gestures.RightHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[rightHandIndex])
{
gestureData.joint = rightHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[rightHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.5f;
}
break;
}
break;
// check for LeftHandCursor
case Gestures.LeftHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[leftHandIndex])
{
gestureData.joint = leftHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[leftHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.5f;
}
break;
}
break;
}
}
private static void SetWheelRotation(uint userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos)
{
float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y);
gestureData.screenPos.z = angle;
}
/// <summary>
/// End of gesture movement callback.
/// </summary>
/// <param name='data'>
/// Data of the gesture gathered.
/// </param>
public void endOfGestureCallback(GestureData data)
{
GestureUtils.EnumGestures gResult;
Debug.Log("Gesture ended!");
List <Vector2> pointList = new List <Vector2>();
GestureData normalizedData = data.getNormalizedGestureData(CANVAS_SIZE);
GestureFrame previousFrame = (GestureFrame)normalizedData.frames[0];
pointList.Add(new Vector2(previousFrame.position.x, previousFrame.position.y));
for (int i = 1; i < normalizedData.frames.Count; i++)
{
GestureFrame currentFrame = (GestureFrame)normalizedData.frames[i];
if ((Math.Abs(currentFrame.position.x - previousFrame.position.x) < NORMAL_FRAME_VARIATION) &&
(Math.Abs(currentFrame.position.y - previousFrame.position.y) < NORMAL_FRAME_VARIATION))
{
pointList.Add(new Vector2(currentFrame.position.x, currentFrame.position.y));
previousFrame = currentFrame;
}
}
//At gesture end, draw the normalized data.
Vector2 previousPoint = pointList[0];
for (int i = 1; i < pointList.Count; i++)
{
Vector2 currentPoint = pointList[i];
DrawLine(canvas, previousPoint.x, previousPoint.y, currentPoint.x, currentPoint.y, Color.black);
previousPoint = currentPoint;
}
canvas.Apply();
//The checker method only receives canvas if you want the points used drawn on the screen,
//can just receive a points list otherwise.
gResult = gu.GestureChecker(pointList, canvas, CANVAS_SIZE);
switch (gResult)
{
case GestureUtils.EnumGestures.DownZigZag:
Debug.Log("Downwards Zig Zag!");
status = "Downwards Zig Zag!";
break;
case GestureUtils.EnumGestures.RightZigZag:
Debug.Log("Rightwards Zig Zag!");
status = "Rightwards Zig Zag!";
break;
case GestureUtils.EnumGestures.Square:
Debug.Log("Square!");
status = "Square!";
break;
default:
Debug.Log("No Gesture found.");
status = "Nothing.";
break;
}
}
public int ProcessNewGestureData(GestureData newGestureData)
{
int result = (int)ResultCodes.Success;
do
{
SkeletonData trackedSkeleton = (from skeletons in newGestureData.GetSkeletonFrame().Skeletons
where skeletons.TrackingState == SkeletonTrackingState.Tracked
select skeletons).FirstOrDefault();
if (trackedSkeleton == null)
{
result = (int)ResultCodes.NullTrackedSkeleton;
break;
}
GestureBase.NuiElement nuiElement = new GestureBase.NuiElement();
nuiElement.SetSkeletonFrame(newGestureData.GetSkeletonFrame());
nuiElement.SetTimeStamp(DateTime.Now);
if (legsGesture != null) legsGesture.AddToQueue(nuiElement);
if (mouseGesture != null) mouseGesture.AddToQueue(nuiElement);
if (frontbackGesture != null) frontbackGesture.AddToQueue(nuiElement);
if (jumpGesture != null) jumpGesture.AddToQueue(nuiElement);
if (shootGesture != null) shootGesture.AddToQueue(nuiElement);
if (!this.stopVisualThreads)
{
TransformedSkeletonDrawing.SkeletalDrawingData newDrawingData = new TransformedSkeletonDrawing.SkeletalDrawingData();
newDrawingData.SetSkeletalFrame(newGestureData.GetSkeletonFrame());
this.drawSkeleton.AddToQueue(newDrawingData);
}
} while (false);
return result;
}
/**
@brief Return whether the specified gesture is fired for a specific hand in the current frame, and if so retrieve its data.
@param[in] gestureName - the name of the gesture to be checked.
@param[in] handID - the ID of the hand whose alert should be retrieved.
@param[out] gestureData - the information for the fired gesture.
@return true if the gesture was fired, false otherwise.
@see GestureData
*/
public Boolean IsGestureFiredByHand(String gestureName, Int32 handID, out GestureData gestureData)
{
return IsGestureFiredByHandINT(instance, gestureName, handID, out gestureData);
}
public GestureInfos(GestureData data)
{
this.data = data;
}
private static void SetZoomFactor(uint userId, ref GestureData gestureData, float initialZoom, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
Vector3 vectorZooming = jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
if(gestureData.tagFloat == 0f || gestureData.userId != userId)
{
gestureData.tagFloat = 0.5f; // this is 100%
}
float distZooming = vectorZooming.magnitude;
gestureData.screenPos.z = initialZoom + (distZooming / gestureData.tagFloat);
}
void LateUpdate()
{
#if UNITY_IOS || UNITY_ANDROID
if (curState == OperationState.Normal)
{
var touchs = new List <Touch>();
//筛选有效touch
for (int i = 0; i < Input.touches.Length; ++i)
{
if (Input.touches[i].phase == TouchPhase.Began && EventSystem.current.IsPointerOverGameObject(Input.touches[i].fingerId))
{
ignoreTouchs.Add(Input.touches[i].fingerId);
}
if (IsTouchIgnored(Input.touches[i]) == -1)
{
touchs.Add(Input.touches[i]);
}
if (Input.touches[i].phase == TouchPhase.Ended)
{
int index = IsTouchIgnored(Input.touches[i]);
if (index > 0)
{
ignoreTouchs.RemoveAt(index);
}
}
}
//double tap check
if (doubleTapActive)
{
doubleTapElapsed += Time.deltaTime;
if (doubleTapElapsed > doubleClickInterval)
{
doubleTapActive = false;
doubleTapElapsed = 0;
}
}
int lastTouchNum = curTouchNum;
curTouchNum = touchs.Count;
switch (curTouchNum)
{
case 0:
holdElapesd = 0;
break;
case 1:
if (lastTouchNum == 0 && touchs[0].phase == TouchPhase.Began)
{
//初次点击
tapPos_1 = touchs[0].position;
holdElapesd = 0;
}
else if (lastTouchNum == 1)
{
if (touchs[0].phase == TouchPhase.Moved)
{
//Swipe
var data = new GestureData();
data.type = GestureType.Swipe;
data.offset = touchs[0].position - tapPos_1;
if (onSwipe != null)
{
onSwipe.Invoke(data);
}
tapPos_1 = touchs[0].position;
}
else if (touchs[0].phase == TouchPhase.Stationary)
{
holdElapesd += Time.deltaTime;
}
else if (touchs[0].phase == TouchPhase.Ended)
{
if (holdElapesd <= this.clickInterval)
{
var data = new GestureData();
//TODO data
if (onSingleTap != null)
{
onSingleTap.Invoke(data);
}
holdElapesd = 0;
if (doubleTapActive)
{
if (doubleTapElapsed <= doubleClickInterval)
{
var doubleData = new GestureData();
//TODO data
if (onDoubleTap != null)
{
onDoubleTap.Invoke(doubleData);
}
doubleTapActive = false;
}
}
else
{
doubleTapActive = true;
doubleTapElapsed = 0;
}
}
}
}
break;
case 2:
default:
if (lastTouchNum < 2)
{
//首次记录位置
tapPos_1 = touchs[0].position;
tapPos_2 = touchs[1].position;
}
if (touchs[0].phase == TouchPhase.Moved || touchs[1].phase == TouchPhase.Moved)
{
//Pinch
var data = new GestureData();
data.type = GestureType.Pinch;
data.value = Vector2.Distance(touchs[0].position, touchs[1].position) - Vector2.Distance(tapPos_1, tapPos_2);
if (onPinch != null)
{
onPinch.Invoke(data);
}
tapPos_1 = touchs[0].position;
tapPos_2 = touchs[1].position;
}
break;
}
}
else
{
if (Input.GetTouch(selectFingerId).phase == TouchPhase.Moved)
{
Vector3 pos = Input.GetTouch(selectFingerId).position;
if (pos.y > cardExcuteBound)
{
switch (curState)
{
case OperationState.HoldCard:
switch (selectCardType)
{
case CardExcuteType.SelectPos:
SwitchOperationState(OperationState.SelectTarget);
break;
}
break;
}
}
else
{
SwitchOperationState(OperationState.HoldCard);
}
}
if (Input.GetTouch(selectFingerId).phase == TouchPhase.Ended)
{
Vector3 pos = Input.GetTouch(selectFingerId).position;
if (pos.y > cardExcuteBound)
{
var data = new GestureData();
switch (curState)
{
case OperationState.HoldCard:
if (onReleaseCard != null)
{
onReleaseCard.Invoke(data);
}
break;
case OperationState.SelectTarget:
if (onReleaseCard != null)
{
onReleaseCard.Invoke(data);
}
break;
}
}
SwitchOperationState(OperationState.Normal);
}
}
#endif
#if UNITY_STANDALONE_WIN
//double tap check
if (doubleTapActive)
{
doubleTapElapsed += Time.deltaTime;
if (doubleTapElapsed > doubleClickInterval)
{
doubleTapActive = false;
doubleTapElapsed = 0;
}
}
//check valid
if (Input.GetMouseButtonUp(0))
{
ignoreMouse = false;
}
if (Input.GetMouseButtonDown(0))
{
if (EventSystem.current.IsPointerOverGameObject())
{
ignoreMouse = true;
}
else
{
//初次点击
tapPos_0 = Input.mousePosition;
tapPos_1 = Input.mousePosition;
holdElapesd = 0;
var data = new GestureData();
data.pos = tapPos_0;
if (onSingleDown != null)
{
onSingleDown.Invoke(data);
}
}
}
if (!ignoreMouse)
{
if (Input.GetMouseButton(0))
{
Vector2 curPos = Input.mousePosition;
//longTap
var dist = Vector2.Distance(curPos, tapPos_0);
if (holdElapesd > this.longClickInterval)
{
var longTapData = new GestureData();
longTapData.pos = tapPos_0;
if (dist <= clickOffsetBound && longTapActive == false)
{
longTapActive = true;
if (onLongTapDown != null)
{
onLongTapDown.Invoke(longTapData);
}
}
longTapData.pos = curPos;
if (onLongTap != null)
{
onLongTap.Invoke(longTapData);
}
}
//Swipe
var data = new GestureData();
data.type = GestureType.Swipe;
data.offset = (curPos - tapPos_1) * MouseSwipeFactor;
if (onSwipe != null)
{
onSwipe.Invoke(data);
}
tapPos_1 = Input.mousePosition;
holdElapesd += Time.deltaTime;
}
if (Input.GetMouseButtonUp(0))
{
if (holdElapesd <= this.clickInterval && Vector2.Distance(tapPos_0, Input.mousePosition) <= clickOffsetBound)
{
var data = new GestureData();
//TODO data
if (onSingleTap != null)
{
onSingleTap.Invoke(data);
}
holdElapesd = 0;
if (doubleTapActive)
{
if (doubleTapElapsed <= doubleClickInterval)
{
var doubleData = new GestureData();
//TODO data
if (onDoubleTap != null)
{
onDoubleTap.Invoke(doubleData);
}
doubleTapActive = false;
}
}
else
{
doubleTapActive = true;
doubleTapElapsed = 0;
}
}
if (longTapActive)
{
longTapActive = false;
if (onLongTapUp != null)
{
GestureData data = new GestureData();
this.onLongTapUp.Invoke(data);
}
}
}
}
if (Input.GetAxis("Mouse ScrollWheel") != 0)
{
//Pinch
var data = new GestureData();
data.type = GestureType.Pinch;
data.value = Input.GetAxis("Mouse ScrollWheel") * MouseSwipeFactor;
if (onPinch != null)
{
if (onPinch != null)
{
onPinch.Invoke(data);
}
}
}
#endif
}
private static void SetScreenPos(uint userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
Vector3 handPos = jointsPos[rightHandIndex];
Vector3 elbowPos = jointsPos[rightElbowIndex];
Vector3 shoulderPos = jointsPos[rightShoulderIndex];
bool calculateCoords = false;
if(gestureData.joint == rightHandIndex)
{
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex])
{
calculateCoords = true;
}
}
else if(gestureData.joint == leftHandIndex)
{
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex])
{
handPos = jointsPos[leftHandIndex];
elbowPos = jointsPos[leftElbowIndex];
shoulderPos = jointsPos[leftShoulderIndex];
calculateCoords = true;
}
}
if(calculateCoords)
{
if(gestureData.tagFloat == 0f || gestureData.userId != userId)
{
// get length from shoulder to hand (screen range)
Vector3 shoulderToElbow = elbowPos - shoulderPos;
Vector3 elbowToHand = handPos - elbowPos;
gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude);
}
Vector3 shoulderToHand = handPos - shoulderPos;
gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat);
gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat);
//Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat));
}
}
/**
* @brief Get the details of the fired gesture with the given index.
*
* @param[in] index - the zero-based index of the requested fired gesture.
* @param[out] gestureData - the information for the fired gesture.
*
* @note The gesture index must be between 0 and [QueryFiredGesturesNumber() - 1]
*
* @return PXCM_STATUS_NO_ERROR - operation succeeded.
* @return PXCM_STATUS_PARAM_UNSUPPORTED - invalid input parameter.
*
* @see GestureData
* @see QueryFiredGesturesNumber
*/
public pxcmStatus QueryFiredGestureData(Int32 index, out GestureData gestureData)
{
return(QueryFiredGestureDataINT(instance, index, out gestureData));
}
/// <summary>
/// Detects the movement and builds the frame list when movement is detected.
/// </summary>
/// <param name='frame'>
/// Leap frame.
/// </param>
public void DetectMovement(Frame frame)
{
GestureFrame gestureFrame = new GestureFrame(frame.Pointables[0].TipPosition, frame.Pointables[0].TipVelocity, frame.Timestamp);
this.detectionHeap.Add(gestureFrame);
Vector3 acceleration = Vector3.zero;
Vector3 delta = Vector3.zero;
Vector3 movementWork = Vector3.zero;
Vector3 kinecticEnergy = Vector3.zero;
float movementTotalWork;
float kinecticEnergyTotal;
if (this.detectionHeap.Count > this.maximumHeapSize)
{
this.detectionHeap.RemoveAt(0);
GestureFrame firstFrame = (GestureFrame)this.detectionHeap[0];
GestureFrame lastFrame = (GestureFrame)this.detectionHeap[this.maximumHeapSize - 1];
float deltaTime = lastFrame.timestamp - firstFrame.timestamp;
acceleration.x = (lastFrame.velocity.x - firstFrame.velocity.x) / deltaTime;
acceleration.y = (lastFrame.velocity.y - firstFrame.velocity.y) / deltaTime;
acceleration.z = (lastFrame.velocity.z - firstFrame.velocity.z) / deltaTime;
this.movementDirection.x = acceleration.x > 0 ? 1 : -1;
this.movementDirection.y = acceleration.y > 0 ? 1 : -1;
this.movementDirection.z = acceleration.z > 0 ? 1 : -1;
delta.x = (lastFrame.position.x - firstFrame.position.x);
delta.y = (lastFrame.position.y - firstFrame.position.y);
delta.z = (lastFrame.position.z - firstFrame.position.z);
/*
* http://en.wikipedia.org/wiki/Work_(physics)
* Work calculation, using the acceleration previously calculated as the force to move the finger from one point to another
*/
movementWork.x = delta.x * acceleration.x;
movementWork.y = delta.y * acceleration.y;
movementWork.z = delta.z * acceleration.z;
/*
* http://en.wikipedia.org/wiki/Kinetic_energy
* Measures the pointable kinectic energy to detect when gesture have ended.
*/
kinecticEnergy.x = Mathf.Pow(lastFrame.velocity.x, 2) * 0.5f * 1.0e-6f;
kinecticEnergy.y = Mathf.Pow(lastFrame.velocity.y, 2) * 0.5f * 1.0e-6f;
kinecticEnergy.z = Mathf.Pow(lastFrame.velocity.z, 2) * 0.5f * 1.0e-6f;
kinecticEnergyTotal = kinecticEnergy.x + kinecticEnergy.y;
if (Mathf.Abs(movementWork.x) > this.minimumRelevantWorkThreshold)
{
this.movementTotalWork.x += movementWork.x;
}
if (Mathf.Abs(movementWork.y) > this.minimumRelevantWorkThreshold)
{
this.movementTotalWork.y += movementWork.y;
}
if (Mathf.Abs(movementWork.z) > this.minimumRelevantWorkThreshold)
{
this.movementTotalWork.z += movementWork.z;
}
if (!this.performingMovement && lastFrame.timestamp - this.lastStoppedTime > 50000.0f)
{
this.movementTotalWork.x = 0.0f;
this.movementTotalWork.y = 0.0f;
this.movementTotalWork.z = 0.0f;
this.lastStoppedTime = lastFrame.timestamp;
}
movementTotalWork = this.movementTotalWork.x + this.movementTotalWork.y;
if (movementTotalWork > this.minimumStartMovementTotalWork && !this.performingMovement)
{
this.movementTotalWork.x = 0;
this.movementTotalWork.y = 0;
this.movementTotalWork.z = 0;
this.performingMovement = true;
if (!this.performingGesture)
{
this.performingGesture = true;
this.gestureData = new GestureData();
if (this.callbackInstance != null)
{
this.callbackInstance.beginOfGestureCallback();
}
}
}
else if (this.performingMovement && kinecticEnergyTotal < this.minimumEndMovementKinectEnergy)
{
this.movementTotalWork.x = 0.0f;
this.movementTotalWork.y = 0.0f;
this.movementTotalWork.z = 0.0f;
this.performingMovement = false;
this.lastStoppedTime = this.lastEndMovementTime = lastFrame.timestamp;
}
if (this.performingGesture)
{
this.gestureData.appendFrame(gestureFrame);
}
float timeSinceEndMovement = lastFrame.timestamp - this.lastEndMovementTime;
if (!this.performingMovement && this.performingGesture && timeSinceEndMovement > 500000)
{
if (this.callbackInstance != null)
{
this.callbackInstance.endOfGestureCallback(this.gestureData);
}
this.performingGesture = false;
}
}
}
/**
* @brief Check whether a gesture was fired and if so return its details.
*
* @param[in] gestureName - the name of the gesture to be checked.
* @param[out] gestureData - the information for the fired gesture.
*
* @return true if the gesture was fired, false otherwise.
*
* @see GestureData
*/
public Boolean IsGestureFired(String gestureName, out GestureData gestureData)
{
return(IsGestureFiredINT(instance, gestureName, out gestureData));
}
/**
* @brief Return whether the specified gesture is fired for a specific hand in the current frame, and if so retrieve its data.
* @param[in] gestureName - the name of the gesture to be checked.
* @param[in] handID - the ID of the hand whose alert should be retrieved.
* @param[out] gestureData - the information for the fired gesture.
*
* @return true if the gesture was fired, false otherwise.
* @see GestureData
*/
public Boolean IsGestureFiredByHand(String gestureName, Int32 handID, out GestureData gestureData)
{
return(IsGestureFiredByHandINT(instance, gestureName, handID, out gestureData));
}
/// <summary>
/// Gets the normalized gesture data.
/// </summary>
/// <returns>
/// The normalized gesture data.
/// </returns>
/// <param name='resolution'>
/// Resolution to normalize to.
/// </param>
public GestureData getNormalizedGestureData(float resolution)
{
var dx = (this.maximumX - this.minimumX) * 1.0f;
var dy = (this.maximumY - this.minimumY) * 1.0f;
if (dx > dy)
dy = dx;
else
dx = dy;
GestureData normalizedData = new GestureData();
for (var i = 0; i < this.frames.Count; i++) {
GestureFrame frame = (GestureFrame)this.frames[i];
float x = (frame.position.x * 1.0f - this.minimumX) / dx * resolution;
float y = resolution - (frame.position.y * 1.0f - this.minimumY) / dy * resolution;
GestureFrame normalizedFrame = new GestureFrame(new Vector(x, y, 0.0f), frame.velocity, frame.timestamp);
normalizedData.appendFrame(normalizedFrame);
}
return normalizedData;
}
private void OnLoneTapUp(GestureData data)
{
selectUnit = null;
selectPath.Clear();
SwitchOperationMode(BattleOperationMode.Normal);
}
public ClassificationAlgorithm(GestureData gestureData)
{
Train(gestureData);
}
void Update()
{
if (handModelManager.leapProvider)
{
Frame frame = handModelManager.leapProvider.CurrentFrame;
if (Input.GetKeyDown(KeyCode.Space) && experienceState == ExperienceState.Idle)
{
typeBox_white.SetActive(false);
typeBox_blue.SetActive(true);
experienceState = ExperienceState.Chatting;
}
else if (Input.GetKeyDown(KeyCode.Space) && experienceState == ExperienceState.Chatting)
{
typeBox_white.SetActive(true);
typeBox_blue.SetActive(false);
experienceState = ExperienceState.Idle;
}
else if (Input.GetKeyDown(KeyCode.Backspace) /* && experienceState == ExperienceState.Chatting*/)
{
RemoveWordsMark(Sign.Hello, Sign.My, Sign.Name, Sign.Sana, Sign.What, Sign.Your, Sign.Nice, Sign.Meet, Sign.You, Sign.How, Sign.Good);
}
else if (Input.GetKeyDown(KeyCode.R) && experienceState == ExperienceState.Idle)
{
StartCoroutine(CountingDown());
experienceState = ExperienceState.Recording;
}
if (frame.Hands.Count > 0)
{
var calibration = GetDeltaRotMaterix();
List <Hand> hands = frame.Hands;
foreach (var hand in hands)
{
if (hand.IsLeft)
{
//Debug.Log(experienceState.ToString());
if (GetGesture(hand, Quote.l_Dirs, calibration) && experienceState == ExperienceState.Idle)
{
typeBox_white.SetActive(false);
typeBox_blue.SetActive(true);
experienceState = ExperienceState.Chatting;
}
if (Input.GetKeyDown(KeyCode.Alpha2) || isCapturing)
{
captured_L_Dirs = new Vector3[6];
Debug.Log("Capturing left hand...");
Capture(hand, out captured_L_Dirs);
}
}
if (hand.IsRight)
{
if (GetGesture(hand, Quote.r_Dirs, calibration) && experienceState == ExperienceState.Chatting)
{
typeBox_white.SetActive(true);
typeBox_blue.SetActive(false);
experienceState = ExperienceState.Idle;
}
switch (experienceState)
{
case ExperienceState.Chatting:
Translate(hand, calibration);
break;
case ExperienceState.Recording:
break;
case ExperienceState.Idle:
DetectSentencesAndPlay();
break;
default:
break;
}
if (Input.GetKeyDown(KeyCode.Alpha1) || isCapturing)
{
captured_R_Dirs = new Vector3[6];
Debug.Log("Capturing left hand...");
Capture(hand, out captured_R_Dirs);
}
}
}
}
}
if (isCapturing)
{
gestureBase[highlightedSentence] = new GestureData(captured_R_Dirs, captured_L_Dirs, false);
}
var typeBoxText = "";
foreach (var gesture in gestureBase)
{
if (gesture.Value.isMarked)
{
typeBoxText += gesture.Key.ToString() + ", ";
}
}
typeBox.text = typeBoxText;
isCapturing = false;
captured_L_Dirs = captured_R_Dirs = null;
}
protected abstract void Train(GestureData data);
// estimate the next state and completeness of the gesture
public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
if(gestureData.complete)
return;
switch(gestureData.gesture)
{
// check for RaiseRightHand
case Gestures.RaiseRightHand:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for RaiseLeftHand
case Gestures.RaiseLeftHand:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for Psi
case Gestures.Psi:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for Stop
case Gestures.Stop:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0f &&
jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0f &&
jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration);
break;
}
break;
// check for Wave
case Gestures.Wave:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture - phase 2
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f;
if(isInPose)
{
gestureData.timestamp = timestamp;
gestureData.state++;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
case 2: // gesture phase 3 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Click
case Gestures.Click:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
break;
case 1: // gesture - phase 2
// if((timestamp - gestureData.timestamp) < 1.0f)
// {
// bool isInPose = gestureData.joint == rightHandIndex ?
// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f :
// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f;
//
// if(isInPose)
// {
// gestureData.timestamp = timestamp;
// gestureData.jointPos = jointsPos[gestureData.joint];
// gestureData.state++;
// gestureData.progress = 0.7f;
// }
// else
// {
// // check for stay-in-place
// Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
// isInPose = distVector.magnitude < 0.05f;
//
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration);
// }
// }
// else
{
// check for stay-in-place
Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
bool isInPose = distVector.magnitude < 0.05f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration);
// SetGestureCancelled(gestureData);
}
break;
// case 2: // gesture phase 3 = complete
// if((timestamp - gestureData.timestamp) < 1.0f)
// {
// bool isInPose = gestureData.joint == rightHandIndex ?
// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f :
// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f;
//
// if(isInPose)
// {
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
// }
// }
// else
// {
// // cancel the gesture
// SetGestureCancelled(ref gestureData);
// }
// break;
}
break;
// check for SweepLeft
case Gestures.SweepLeft:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
// else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
// (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, leftHandIndex);
// gestureData.jointPos = jointsPos[leftHandIndex];
// gestureData.progress = 0.5f;
// }
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.2f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SweepRight
case Gestures.SweepRight:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
// (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, rightHandIndex);
// gestureData.jointPos = jointsPos[rightHandIndex];
// gestureData.progress = 0.5f;
// }
// else
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.2f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SweepUp
case Gestures.SweepUp:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) >= -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) >= -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].y - gestureData.jointPos.y) > 0.2f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].y - gestureData.jointPos.y) > 0.2f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SweepDown
case Gestures.SweepDown:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) >= 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) >= 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < -0.1f &&
(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < -0.2f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < -0.1f &&
(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < -0.2f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for RightHandCursor
case Gestures.RightHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f)
{
gestureData.joint = rightHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[rightHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.7f;
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for LeftHandCursor
case Gestures.LeftHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f)
{
gestureData.joint = leftHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[leftHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.7f;
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ZoomOut
case Gestures.ZoomOut:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
float distZoomOut = ((Vector3)(jointsPos[rightHandIndex] - jointsPos[leftHandIndex])).magnitude;
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
distZoomOut < 0.2f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f ||
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f);
if(isInPose)
{
SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ZoomIn
case Gestures.ZoomIn:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
float distZoomIn = ((Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]).magnitude;
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
distZoomIn >= 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagFloat = distZoomIn;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if((timestamp - gestureData.timestamp) < 1.0f)
{
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f ||
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f);
if(isInPose)
{
SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Wheel
case Gestures.Wheel:
Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distWheel = vectorWheel.magnitude;
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
distWheel > 0.2f && distWheel < 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = vectorWheel;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f ||
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f &&
Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f);
if(isInPose)
{
SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel);
gestureData.timestamp = timestamp;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.Move:
switch(gestureData.state)
{
case 0:
if(jointsTracked[leftKneeIndex] && jointsTracked[rightKneeIndex] && Math.Abs(jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z) >0.1f)
{
SetGestureJoint(ref gestureData,timestamp,leftKneeIndex,jointsPos[leftKneeIndex]);
}
break;
case 1:
bool isInPose = jointsTracked[leftKneeIndex] && jointsTracked[rightKneeIndex] && Math.Abs(jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z) >0.1f;
if(isInPose)
{
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
gestureData.screenPos.z = jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z;
}
else
{
SetGestureCancelled(ref gestureData);
}
break;
}
break;
}
}
public abstract double ComputeError(GestureData data);
public static void Clear()
{
Data = new GestureData();
}
public MultivariateAlgorithm(GestureData input) : base(input)
{
}
/**
@brief Check whether a gesture was fired and if so return its details.
@param[in] gestureName - the name of the gesture to be checked.
@param[out] gestureData - the information for the fired gesture.
@return true if the gesture was fired, false otherwise.
@see GestureData
*/
public Boolean IsGestureFired(String gestureName, out GestureData gestureData)
{
return IsGestureFiredINT(instance, gestureName, out gestureData);
}
//设置姿势取消
private static void SetGestureCancelled(ref GestureData gestureData)
{
gestureData.state = 0;
gestureData.progress = 0f;
gestureData.cancelled = true;
}
private static void SetGestureCancelled(ref GestureData gestureData)
{
gestureData.state = 0;
gestureData.progress = 0f;
gestureData.cancelled = true;
}
internal static pxcmStatus QueryFiredGestureDataINT(IntPtr instance, Int32 index, out GestureData gestureData)
{
gestureData = new GestureData();
return PXCMHandData_QueryFiredGestureData(instance, index, gestureData);
}
private static void SetScreenPos(uint userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
Vector3 handPos = jointsPos[rightHandIndex];
// Vector3 elbowPos = jointsPos[rightElbowIndex];
// Vector3 shoulderPos = jointsPos[rightShoulderIndex];
bool calculateCoords = false;
if(gestureData.joint == rightHandIndex)
{
if(jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/)
{
calculateCoords = true;
}
}
else if(gestureData.joint == leftHandIndex)
{
if(jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/)
{
handPos = jointsPos[leftHandIndex];
// elbowPos = jointsPos[leftElbowIndex];
// shoulderPos = jointsPos[leftShoulderIndex];
calculateCoords = true;
}
}
if(calculateCoords)
{
// if(gestureData.tagFloat == 0f || gestureData.userId != userId)
// {
// // get length from shoulder to hand (screen range)
// Vector3 shoulderToElbow = elbowPos - shoulderPos;
// Vector3 elbowToHand = handPos - elbowPos;
// gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude);
// }
if(jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] &&
jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex])
{
Vector3 neckToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex];
Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex];
gestureData.tagVector2.x = rightToLeft.x; // * 1.2f;
gestureData.tagVector2.y = neckToHips.y; // * 1.2f;
if(gestureData.joint == rightHandIndex)
{
gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
else
{
gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
}
// Vector3 shoulderToHand = handPos - shoulderPos;
// gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat);
// gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat);
if(gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0)
{
Vector3 relHandPos = handPos - gestureData.tagVector;
gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x);
gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y);
}
//Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat));
}
}
protected void SetScreenPos(long userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
Vector3 handPos = jointsPos[rightHandIndex];
bool calculateCoords = false;
if(gestureData.joint == rightHandIndex)
{
if(jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/)
{
calculateCoords = true;
}
}
else if(gestureData.joint == leftHandIndex)
{
if(jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/)
{
handPos = jointsPos[leftHandIndex];
calculateCoords = true;
}
}
if(calculateCoords)
{
if(jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] &&
jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex])
{
Vector3 shoulderToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex];
Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex];
gestureData.tagVector2.x = rightToLeft.x; // * 1.2f;
gestureData.tagVector2.y = shoulderToHips.y; // * 1.2f;
if(gestureData.joint == rightHandIndex)
{
gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
else
{
gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
}
if(gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0)
{
Vector3 relHandPos = handPos - gestureData.tagVector;
gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x);
gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y);
}
}
}
// private static void SetWheelRotation(uint userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos)
// {
// float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y);
// gestureData.screenPos.z = angle;
// }
// estimate the next state and completeness of the gesture
public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
if(gestureData.complete)
return;
float bandSize = (jointsPos[shoulderCenterIndex].y - jointsPos[hipCenterIndex].y);
float gestureTop = jointsPos[shoulderCenterIndex].y + bandSize / 2;
float gestureBottom = jointsPos[shoulderCenterIndex].y - bandSize;
float gestureRight = jointsPos[rightHipIndex].x;
float gestureLeft = jointsPos[leftHipIndex].x;
switch(gestureData.gesture)
{
// check for RaiseRightHand
case Gestures.RaiseRightHand:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration);
break;
}
break;
// check for RaiseLeftHand
case Gestures.RaiseLeftHand:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration);
break;
}
break;
// check for Psi
case Gestures.Psi:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration);
break;
}
break;
// check for Tpose
case Gestures.Tpose:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.07f
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration);
break;
}
break;
// check for Stop
case Gestures.Stop:
switch(gestureData.state)
{
case 0: // gesture detection
if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.4f &&
(jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.4f &&
(jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = (gestureData.joint == rightHandIndex) ?
(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.4f &&
(jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) :
(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.4f &&
(jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f);
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration);
break;
}
break;
// check for Wave
case Gestures.Wave:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture - phase 2
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f;
if(isInPose)
{
gestureData.timestamp = timestamp;
gestureData.state++;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
case 2: // gesture phase 3 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Click
case Gestures.Click:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
break;
case 1: // gesture - phase 2
// if((timestamp - gestureData.timestamp) < 1.0f)
// {
// bool isInPose = gestureData.joint == rightHandIndex ?
// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f :
// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f;
//
// if(isInPose)
// {
// gestureData.timestamp = timestamp;
// gestureData.jointPos = jointsPos[gestureData.joint];
// gestureData.state++;
// gestureData.progress = 0.7f;
// }
// else
// {
// // check for stay-in-place
// Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
// isInPose = distVector.magnitude < 0.05f;
//
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration);
// }
// }
// else
{
// check for stay-in-place
Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
bool isInPose = distVector.magnitude < 0.05f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.ClickStayDuration);
// SetGestureCancelled(gestureData);
}
break;
// case 2: // gesture phase 3 = complete
// if((timestamp - gestureData.timestamp) < 1.0f)
// {
// bool isInPose = gestureData.joint == rightHandIndex ?
// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f :
// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f;
//
// if(isInPose)
// {
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
// }
// }
// else
// {
// // cancel the gesture
// SetGestureCancelled(ref gestureData);
// }
// break;
}
break;
// check for SwipeLeft
case Gestures.SwipeLeft:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f &&
// (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
// gestureData.progress = 0.5f;
// }
if(jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].x <= gestureRight && jointsPos[rightHandIndex].x > gestureLeft)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.1f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
// bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
// (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.15f;
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].x < gestureLeft;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else if(jointsPos[rightHandIndex].x <= gestureRight)
{
float gestureSize = gestureRight - gestureLeft;
gestureData.progress = gestureSize > 0.01f ? (gestureRight - jointsPos[rightHandIndex].x) / gestureSize : 0f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SwipeRight
case Gestures.SwipeRight:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f &&
// (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
// gestureData.progress = 0.5f;
// }
if(jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[leftHandIndex].x >= gestureLeft && jointsPos[leftHandIndex].x < gestureRight)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.1f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
// bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
// (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.15f;
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[leftHandIndex].x > gestureRight;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else if(jointsPos[leftHandIndex].x >= gestureLeft)
{
float gestureSize = gestureRight - gestureLeft;
gestureData.progress = gestureSize > 0.01f ? (jointsPos[leftHandIndex].x - gestureLeft) / gestureSize : 0f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SwipeUp
case Gestures.SwipeUp:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < 0.0f &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < 0.0f &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SwipeDown
case Gestures.SwipeDown:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.15f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.15f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for RightHandCursor
case Gestures.RightHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f)
{
gestureData.joint = rightHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[rightHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.7f;
}
else
{
// cancel the gesture
//SetGestureCancelled(ref gestureData);
gestureData.progress = 0f;
}
break;
}
break;
// check for LeftHandCursor
case Gestures.LeftHandCursor:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1 (perpetual)
if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f)
{
gestureData.joint = leftHandIndex;
gestureData.timestamp = timestamp;
//gestureData.jointPos = jointsPos[leftHandIndex];
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
gestureData.progress = 0.7f;
}
else
{
// cancel the gesture
//SetGestureCancelled(ref gestureData);
gestureData.progress = 0f;
}
break;
}
break;
// check for ZoomOut
case Gestures.ZoomOut:
Vector3 vectorZoomOut = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distZoomOut = vectorZoomOut.magnitude;
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomOut < 0.3f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = 0f;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if((timestamp - gestureData.timestamp) < 1.5f)
{
float angleZoomOut = Vector3.Angle(gestureData.tagVector, vectorZoomOut) * Mathf.Sign(vectorZoomOut.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomOut < 1.5f && Mathf.Abs(angleZoomOut) < 20f;
if(isInPose)
{
SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ZoomIn
case Gestures.ZoomIn:
Vector3 vectorZoomIn = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distZoomIn = vectorZoomIn.magnitude;
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomIn >= 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = distZoomIn;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if((timestamp - gestureData.timestamp) < 1.5f)
{
float angleZoomIn = Vector3.Angle(gestureData.tagVector, vectorZoomIn) * Mathf.Sign(vectorZoomIn.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomIn >= 0.2f && Mathf.Abs(angleZoomIn) < 20f;
if(isInPose)
{
SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Wheel
case Gestures.Wheel:
Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distWheel = vectorWheel.magnitude;
// Debug.Log(string.Format("{0}. Dist: {1:F1}, Tag: {2:F1}, Diff: {3:F1}", gestureData.state,
// distWheel, gestureData.tagFloat, Mathf.Abs(distWheel - gestureData.tagFloat)));
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distWheel >= 0.3f && distWheel < 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = turning wheel
if((timestamp - gestureData.timestamp) < 1.5f)
{
float angle = Vector3.Angle(gestureData.tagVector, vectorWheel) * Mathf.Sign(vectorWheel.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distWheel >= 0.3f && distWheel < 0.7f &&
Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f;
if(isInPose)
{
//SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel);
gestureData.screenPos.z = angle; // wheel angle
gestureData.timestamp = timestamp;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Jump
case Gestures.Jump:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y > 0.9f) && (jointsPos[hipCenterIndex].y < 1.3f))
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y - gestureData.jointPos.y) > 0.15f &&
Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Squat
case Gestures.Squat:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y <= 0.9f))
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.15f &&
Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Push
case Gestures.Push:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f &&
(jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f &&
(jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.1f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Pull
case Gestures.Pull:
switch(gestureData.state)
{
case 0: // gesture detection - phase 1
if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f &&
(jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.3f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f &&
(jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.3f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.1f;
if(isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// here come more gesture-cases
}
}
private static void SetWheelRotation(long userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos)
{
float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y);
gestureData.screenPos.z = angle;
}
//设置在屏幕上的位置？？？？？？？
private static void SetScreenPos(long userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
Vector3 handPos = jointsPos[rightHandIndex];
// Vector3 elbowPos = jointsPos[rightElbowIndex];
// Vector3 shoulderPos = jointsPos[rightShoulderIndex];
bool calculateCoords = false;
if (gestureData.joint == rightHandIndex)
{
if (jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/)
{
calculateCoords = true;
}
}
else if (gestureData.joint == leftHandIndex)
{
if (jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/)
{
handPos = jointsPos[leftHandIndex];
// elbowPos = jointsPos[leftElbowIndex];
// shoulderPos = jointsPos[leftShoulderIndex];
calculateCoords = true;
}
}
if (calculateCoords)
{
// if(gestureData.tagFloat == 0f || gestureData.userId != userId)
// {
// // get length from shoulder to hand (screen range)
// Vector3 shoulderToElbow = elbowPos - shoulderPos;
// Vector3 elbowToHand = handPos - elbowPos;
// gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude);
// }
if (jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] &&
jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex])
{
Vector3 shoulderToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex];
Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex];
gestureData.tagVector2.x = rightToLeft.x; // * 1.2f;
gestureData.tagVector2.y = shoulderToHips.y; // * 1.2f;
if (gestureData.joint == rightHandIndex)
{
gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
else
{
gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2;
gestureData.tagVector.y = jointsPos[hipCenterIndex].y;
}
}
// Vector3 shoulderToHand = handPos - shoulderPos;
// gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat);
// gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat);
if (gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0)
{
Vector3 relHandPos = handPos - gestureData.tagVector;
gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x);
gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y);
}
//Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat));
}
}
// estimate the next state and completeness of the gesture
/// <summary>
/// estimate the state and progress of the given gesture.
/// </summary>
/// <param name="userId">User ID</param>
/// <param name="gestureData">Gesture-data structure</param>
/// <param name="timestamp">Current time</param>
/// <param name="jointsPos">Joints-position array</param>
/// <param name="jointsTracked">Joints-tracked array</param>
public static void CheckForGesture(long userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked)
{
if (gestureData.complete)
return;
float bandSize = (jointsPos[shoulderCenterIndex].y - jointsPos[hipCenterIndex].y);
float gestureTop = jointsPos[shoulderCenterIndex].y + bandSize * 1.2f / 3f;
float gestureBottom = jointsPos[shoulderCenterIndex].y - bandSize * 1.8f / 3f;
/*
float gestureRight = jointsPos[rightHipIndex].x;
float gestureLeft = jointsPos[leftHipIndex].x;
*/
switch (gestureData.gesture)
{
// check for RaiseRightHand
case Gestures.RaiseRightHand:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
break;
}
break;
// check for RaiseLeftHand
case Gestures.RaiseLeftHand:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
break;
}
break;
// check for Psi
case Gestures.Psi:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[shoulderCenterIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[shoulderCenterIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
break;
}
break;
// check for Tpose
case Gestures.Tpose:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.07f
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
break;
}
break;
// check for Stop
case Gestures.Stop:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.2f &&
(jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
else if (jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.2f &&
(jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
}
break;
case 1: // gesture complete
bool isInPose = (gestureData.joint == rightHandIndex) ?
(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.2f &&
(jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) :
(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.2f &&
(jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f);
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
break;
}
break;
// check for Wave
case Gestures.Wave:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
}
else if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture - phase 2
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f;
if (isInPose)
{
gestureData.timestamp = timestamp;
gestureData.state++;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
case 2: // gesture phase 3 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f &&
(jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f &&
(jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// Check for custom click
case Gestures.Click:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
else if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.3f;
// set screen position at the start, because this is the most accurate click position
SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
}
break;
case 1: // gesture - phase 2
{
// check for stay-in-place
Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
bool isInPose = distVector.magnitude < 0.05f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.ClickStayDuration);
// SetGestureCancelled(gestureData);
MGC.Instance.mouseCursor.GetComponent<Game.CursorReference>().cursorReference.cursorCircleRight.progress = gestureData.progress + 0.1f;
if(gestureData.progress >= 1)
{
MGC.Instance.mouseCursor.GetComponent<Game.CursorReference>().cursorReference.cursorCircleRight.progress = 0;
//MouseControl.MouseClick();
}
}
break;
}
break;
// // check for Click
// case Gestures.Click:
// switch(gestureData.state)
// {
// case 0: // gesture detection - phase 1
// if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f)
// {
// SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
// gestureData.progress = 0.3f;
//
// // set screen position at the start, because this is the most accurate click position
// SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
// }
// else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f)
// {
// SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
// gestureData.progress = 0.3f;
//
// // set screen position at the start, because this is the most accurate click position
// SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
// }
// break;
//
// case 1: // gesture - phase 2
//// if((timestamp - gestureData.timestamp) < 1.0f)
//// {
//// bool isInPose = gestureData.joint == rightHandIndex ?
//// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
//// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
//// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
//// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f :
//// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
//// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
//// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
//// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f;
////
//// if(isInPose)
//// {
//// gestureData.timestamp = timestamp;
//// gestureData.jointPos = jointsPos[gestureData.joint];
//// gestureData.state++;
//// gestureData.progress = 0.7f;
//// }
//// else
//// {
//// // check for stay-in-place
//// Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
//// isInPose = distVector.magnitude < 0.05f;
////
//// Vector3 jointPos = jointsPos[gestureData.joint];
//// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration);
//// }
//// }
//// else
// {
// // check for stay-in-place
// Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos;
// bool isInPose = distVector.magnitude < 0.05f;
//
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.ClickStayDuration);
//// SetGestureCancelled(gestureData);
// }
// break;
//
//// case 2: // gesture phase 3 = complete
//// if((timestamp - gestureData.timestamp) < 1.0f)
//// {
//// bool isInPose = gestureData.joint == rightHandIndex ?
//// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
//// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
//// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f &&
//// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f :
//// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
//// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
//// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f &&
//// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f;
////
//// if(isInPose)
//// {
//// Vector3 jointPos = jointsPos[gestureData.joint];
//// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
//// }
//// }
//// else
//// {
//// // cancel the gesture
//// SetGestureCancelled(ref gestureData);
//// }
//// break;
// }
// break;
// DEFAULT SWIPE LEFT
// check for SwipeLeft
/*case Gestures.SwipeLeft:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f &&
// (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
// gestureData.progress = 0.5f;
// }
if (jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].x <= gestureRight && jointsPos[rightHandIndex].x > gestureLeft)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.1f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
// bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
// Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f &&
// Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
// (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.15f;
//
// if(isInPose)
// {
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
// }
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].x <= gestureLeft;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else if (jointsPos[rightHandIndex].x <= gestureRight)
{
float gestureSize = gestureRight - gestureLeft;
gestureData.progress = gestureSize > 0.01f ? (gestureRight - jointsPos[rightHandIndex].x) / gestureSize : 0f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;*/
//EDITED SwipeLeft
// check for SwipeLeft
case Gestures.SwipeLeft:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.25f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.25f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
if (gestureData.complete)
{
Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_J);
}
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// DEFAULT SWIPE RIGHT
// check for SwipeRight
/*case Gestures.SwipeRight:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
// if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f &&
// (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f)
// {
// SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
// gestureData.progress = 0.5f;
// }
if (jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[leftHandIndex].x >= gestureLeft && jointsPos[leftHandIndex].x < gestureRight)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.1f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
// bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
// Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f &&
// Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
// (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.15f;
//
// if(isInPose)
// {
// Vector3 jointPos = jointsPos[gestureData.joint];
// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
// }
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[leftHandIndex].x >= gestureRight;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
else if (jointsPos[leftHandIndex].x >= gestureLeft)
{
float gestureSize = gestureRight - gestureLeft;
gestureData.progress = gestureSize > 0.01f ? (jointsPos[leftHandIndex].x - gestureLeft) / gestureSize : 0f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;*/
// EDITED SwipeRight
// check for SwipeRight
case Gestures.SwipeRight:
switch (gestureData.state)
{
case 0:
if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.25f :
jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.25f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
if (gestureData.complete)
{
Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_L);
}
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.SwipeLeftLHand:
switch (gestureData.state)
{
case 0:
if (jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex])
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
bool isInPose = gestureData.joint == leftHandIndex ?
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.1f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.25f :
jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.1f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.25f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
if (gestureData.complete)
{
Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_J);
}
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
case Gestures.SwipeRightRHand:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex])
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] &&
Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.1f &&
(jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.25f :
jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] &&
Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.1f &&
(jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.25f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
if (gestureData.complete)
{
Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_L);
}
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SwipeUp
case Gestures.SwipeUp:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.0f &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.0f &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for SwipeDown
case Gestures.SwipeDown:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) >= 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if (jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) >= 0.05f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.15f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.15f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// // check for RightHandCursor
// case Gestures.RightHandCursor:
// switch(gestureData.state)
// {
// case 0: // gesture detection - phase 1 (perpetual)
// if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] &&
// //(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f)
// (jointsPos[rightHandIndex].y - jointsPos[hipCenterIndex].y) >= 0f)
// {
// gestureData.joint = rightHandIndex;
// gestureData.timestamp = timestamp;
// gestureData.jointPos = jointsPos[rightHandIndex];
//
// SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
// gestureData.progress = 0.7f;
// }
// else
// {
// // cancel the gesture
// //SetGestureCancelled(ref gestureData);
// gestureData.progress = 0f;
// }
// break;
//
// }
// break;
//
// // check for LeftHandCursor
// case Gestures.LeftHandCursor:
// switch(gestureData.state)
// {
// case 0: // gesture detection - phase 1 (perpetual)
// if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] &&
// //(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f)
// (jointsPos[leftHandIndex].y - jointsPos[hipCenterIndex].y) >= 0f)
// {
// gestureData.joint = leftHandIndex;
// gestureData.timestamp = timestamp;
// gestureData.jointPos = jointsPos[leftHandIndex];
//
// SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked);
// gestureData.progress = 0.7f;
// }
// else
// {
// // cancel the gesture
// //SetGestureCancelled(ref gestureData);
// gestureData.progress = 0f;
// }
// break;
//
// }
// break;
// check for ZoomIn
case Gestures.ZoomIn:
Vector3 vectorZoomOut = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distZoomOut = vectorZoomOut.magnitude;
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomOut < 0.3f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = 0f;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if ((timestamp - gestureData.timestamp) < 1.0f)
{
float angleZoomOut = Vector3.Angle(gestureData.tagVector, vectorZoomOut) * Mathf.Sign(vectorZoomOut.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomOut < 1.5f && Mathf.Abs(angleZoomOut) < 20f;
if (isInPose)
{
SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
// else
// {
// // cancel the gesture
// SetGestureCancelled(ref gestureData);
// }
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ZoomOut
case Gestures.ZoomOut:
Vector3 vectorZoomIn = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distZoomIn = vectorZoomIn.magnitude;
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomIn >= 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = distZoomIn;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if ((timestamp - gestureData.timestamp) < 1.0f)
{
float angleZoomIn = Vector3.Angle(gestureData.tagVector, vectorZoomIn) * Mathf.Sign(vectorZoomIn.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distZoomIn >= 0.2f && Mathf.Abs(angleZoomIn) < 20f;
if (isInPose)
{
SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
// else
// {
// // cancel the gesture
// SetGestureCancelled(ref gestureData);
// }
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Wheel
case Gestures.Wheel:
Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex];
float distWheel = vectorWheel.magnitude;
// Debug.Log(string.Format("{0}. Dist: {1:F1}, Tag: {2:F1}, Diff: {3:F1}", gestureData.state,
// distWheel, gestureData.tagFloat, Mathf.Abs(distWheel - gestureData.tagFloat)));
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distWheel >= 0.3f && distWheel < 0.7f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.tagVector = Vector3.right;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = zooming
if ((timestamp - gestureData.timestamp) < 0.5f)
{
float angle = Vector3.Angle(gestureData.tagVector, vectorWheel) * Mathf.Sign(vectorWheel.y - gestureData.tagVector.y);
bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] &&
jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop &&
jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop &&
distWheel >= 0.3f && distWheel < 0.7f &&
Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f;
if (isInPose)
{
//SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel);
gestureData.screenPos.z = angle; // wheel angle
gestureData.timestamp = timestamp;
gestureData.tagFloat = distWheel;
gestureData.progress = 0.7f;
}
// else
// {
// // cancel the gesture
// SetGestureCancelled(ref gestureData);
// }
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Jump
case Gestures.Jump:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y > 0.6f) && (jointsPos[hipCenterIndex].y < 1.2f))
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y - gestureData.jointPos.y) > 0.15f &&
Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Squat
case Gestures.Squat:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y <= 0.7f))
{
SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[hipCenterIndex] &&
(jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.15f &&
Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Push
case Gestures.Push:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f &&
(jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f &&
(jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.2f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.2f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for Pull
case Gestures.Pull:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f &&
(jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.3f)
{
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
gestureData.progress = 0.5f;
}
else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f &&
(jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.3f)
{
SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = gestureData.joint == rightHandIndex ?
jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] &&
(jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.25f :
jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] &&
(jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f &&
Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f &&
(jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.25f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ShoulderLeftFron
case Gestures.ShoulderLeftFront:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[rightShoulderIndex].z - jointsPos[leftHipIndex].z) < 0f &&
(jointsPos[rightShoulderIndex].z - jointsPos[leftShoulderIndex].z) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, rightShoulderIndex, jointsPos[rightShoulderIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[rightShoulderIndex].z - jointsPos[leftShoulderIndex].z) < -0.2f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for ShoulderRightFront
case Gestures.ShoulderRightFront:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[leftShoulderIndex].z - jointsPos[rightHipIndex].z) < 0f &&
(jointsPos[leftShoulderIndex].z - jointsPos[rightShoulderIndex].z) > -0.15f)
{
SetGestureJoint(ref gestureData, timestamp, leftShoulderIndex, jointsPos[leftShoulderIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[leftShoulderIndex].z - jointsPos[rightShoulderIndex].z) < -0.2f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for LeanLeft
case Gestures.LeanLeft:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1 (right shoulder is left of the right hip, means leaning left)
if (jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightShoulderIndex].x - jointsPos[rightHipIndex].x) < 0f)
{
SetGestureJoint(ref gestureData, timestamp, rightShoulderIndex, jointsPos[rightShoulderIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
// check if right shoulder is still left of the right hip (leaning left)
bool isInPose = jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightShoulderIndex].x - jointsPos[rightHipIndex].x) < 0f;
if (isInPose)
{
// calculate lean angle
Vector3 vSpineLL = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex];
gestureData.screenPos.z = Vector3.Angle(Vector3.up, vSpineLL);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for LeanRight
case Gestures.LeanRight:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1 (left shoulder is right of the left hip, means leaning right)
if (jointsTracked[leftShoulderIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftShoulderIndex].x - jointsPos[leftHipIndex].x) > 0f)
{
SetGestureJoint(ref gestureData, timestamp, leftShoulderIndex, jointsPos[leftShoulderIndex]);
gestureData.progress = 0.3f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 0.5f)
{
// check if left shoulder is still right of the left hip (leaning right)
bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftShoulderIndex].x - jointsPos[leftHipIndex].x) > 0f;
if (isInPose)
{
// calculate lean angle
Vector3 vSpineLL = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex];
gestureData.screenPos.z = Vector3.Angle(Vector3.up, vSpineLL);
gestureData.timestamp = timestamp;
gestureData.progress = 0.7f;
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for KickLeft
case Gestures.KickLeft:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftAnkleIndex].z - jointsPos[leftHipIndex].z) < 0f &&
(jointsPos[leftAnkleIndex].z - jointsPos[rightAnkleIndex].z) > -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, leftAnkleIndex, jointsPos[leftAnkleIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[leftHipIndex] &&
(jointsPos[leftAnkleIndex].z - jointsPos[rightAnkleIndex].z) < -0.4f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for KickRight
case Gestures.KickRight:
switch (gestureData.state)
{
case 0: // gesture detection - phase 1
if (jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightAnkleIndex].z - jointsPos[rightHipIndex].z) < 0f &&
(jointsPos[rightAnkleIndex].z - jointsPos[leftAnkleIndex].z) > -0.2f)
{
SetGestureJoint(ref gestureData, timestamp, rightAnkleIndex, jointsPos[rightAnkleIndex]);
gestureData.progress = 0.5f;
}
break;
case 1: // gesture phase 2 = complete
if ((timestamp - gestureData.timestamp) < 1.5f)
{
bool isInPose = jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[rightHipIndex] &&
(jointsPos[rightAnkleIndex].z - jointsPos[leftAnkleIndex].z) < -0.4f;
if (isInPose)
{
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f);
}
}
else
{
// cancel the gesture
SetGestureCancelled(ref gestureData);
}
break;
}
break;
// check for HiddenGesture
case Gestures.HiddenGesture:
switch (gestureData.state)
{
case 0: // gesture detection
if (jointsTracked[rightHandIndex] && jointsTracked[hipsIndex] &&
(jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f &&
(jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f)
{
Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.NUMPAD7);
//Debug.LogError("Pressed 7? Why?");
SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]);
}
break;
case 1:
bool gestureDetected = (jointsTracked[rightHandIndex] && jointsTracked[hipsIndex] &&
(jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f &&
(jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f);
if (!gestureDetected)
{
Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.NUMPAD9);
SetGestureCancelled(ref gestureData);
}
if (timestamp - gestureData.timestamp <= 3)
{
gestureData.progress = (timestamp - gestureData.timestamp) / 3;
MGC.Instance.minigamesGUI.guiDetection.ShowDetection(gestureData.progress);
}
if (timestamp - gestureData.timestamp > 3)
gestureData.state++;
break;
case 2: // gesture complete
if ((timestamp - gestureData.timestamp) > 3)
{
bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[neck] &&
(jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f &&
(jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f;
Vector3 jointPos = jointsPos[gestureData.joint];
CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration);
//if (isInPose)
{
//Debug.LogError("Hidden gesture complete");
Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.KEY_I);
//Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.NUMPAD9);
#if UNITY_ANDROID
Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.ESC);
#endif
SetGestureCancelled(ref gestureData);
}
}
break;
}
break;
// here come more gesture-cases
}
}
internal static Boolean IsGestureFiredByHandINT(IntPtr instance, String gestureName, Int32 handID, out GestureData gestureData)
{
gestureData = new GestureData();
return PXCMHandData_IsGestureFiredByHand(instance, gestureName, handID, gestureData);
}
/// <summary>
/// Detects the movement and builds the frame list when movement is detected.
/// </summary>
/// <param name='frame'>
/// Leap frame.
/// </param>
public void DetectMovement(Frame frame)
{
GestureFrame gestureFrame = new GestureFrame(frame.Pointables[0].TipPosition, frame.Pointables[0].TipVelocity, frame.Timestamp);
this.detectionHeap.Add(gestureFrame);
Vector3 acceleration = Vector3.zero;
Vector3 delta = Vector3.zero;
Vector3 movementWork = Vector3.zero;
Vector3 kinecticEnergy = Vector3.zero;
float movementTotalWork;
float kinecticEnergyTotal;
if (this.detectionHeap.Count > this.maximumHeapSize) {
this.detectionHeap.RemoveAt(0);
GestureFrame firstFrame = (GestureFrame)this.detectionHeap[0];
GestureFrame lastFrame = (GestureFrame)this.detectionHeap[this.maximumHeapSize - 1];
float deltaTime = lastFrame.timestamp - firstFrame.timestamp;
acceleration.x = (lastFrame.velocity.x - firstFrame.velocity.x) / deltaTime;
acceleration.y = (lastFrame.velocity.y - firstFrame.velocity.y) / deltaTime;
acceleration.z = (lastFrame.velocity.z - firstFrame.velocity.z) / deltaTime;
this.movementDirection.x = acceleration.x > 0 ? 1 : -1;
this.movementDirection.y = acceleration.y > 0 ? 1 : -1;
this.movementDirection.z = acceleration.z > 0 ? 1 : -1;
delta.x = (lastFrame.position.x - firstFrame.position.x);
delta.y = (lastFrame.position.y - firstFrame.position.y);
delta.z = (lastFrame.position.z - firstFrame.position.z);
/*
* http://en.wikipedia.org/wiki/Work_(physics)
* Work calculation, using the acceleration previously calculated as the force to move the finger from one point to another
*/
movementWork.x = delta.x * acceleration.x;
movementWork.y = delta.y * acceleration.y;
movementWork.z = delta.z * acceleration.z;
/*
* http://en.wikipedia.org/wiki/Kinetic_energy
* Measures the pointable kinectic energy to detect when gesture have ended.
*/
kinecticEnergy.x = Mathf.Pow(lastFrame.velocity.x, 2) * 0.5f * 1.0e-6f;
kinecticEnergy.y = Mathf.Pow(lastFrame.velocity.y, 2) * 0.5f * 1.0e-6f;
kinecticEnergy.z = Mathf.Pow(lastFrame.velocity.z, 2) * 0.5f * 1.0e-6f;
kinecticEnergyTotal = kinecticEnergy.x + kinecticEnergy.y;
if (Mathf.Abs(movementWork.x) > this.minimumRelevantWorkThreshold) {
this.movementTotalWork.x += movementWork.x;
}
if (Mathf.Abs(movementWork.y) > this.minimumRelevantWorkThreshold) {
this.movementTotalWork.y += movementWork.y;
}
if (Mathf.Abs(movementWork.z) > this.minimumRelevantWorkThreshold) {
this.movementTotalWork.z += movementWork.z;
}
if (!this.performingMovement && lastFrame.timestamp - this.lastStoppedTime > 50000.0f) {
this.movementTotalWork.x = 0.0f;
this.movementTotalWork.y = 0.0f;
this.movementTotalWork.z = 0.0f;
this.lastStoppedTime = lastFrame.timestamp;
}
movementTotalWork = this.movementTotalWork.x + this.movementTotalWork.y;
if (movementTotalWork > this.minimumStartMovementTotalWork && !this.performingMovement) {
this.movementTotalWork.x = 0;
this.movementTotalWork.y = 0;
this.movementTotalWork.z = 0;
this.performingMovement = true;
if (!this.performingGesture) {
this.performingGesture = true;
this.gestureData = new GestureData();
if (this.callbackInstance != null)
{
this.callbackInstance.beginOfGestureCallback();
}
}
} else if (this.performingMovement && kinecticEnergyTotal < this.minimumEndMovementKinectEnergy) {
this.movementTotalWork.x = 0.0f;
this.movementTotalWork.y = 0.0f;
this.movementTotalWork.z = 0.0f;
this.performingMovement = false;
this.lastStoppedTime = this.lastEndMovementTime = lastFrame.timestamp;
}
if (this.performingGesture) {
this.gestureData.appendFrame(gestureFrame);
}
float timeSinceEndMovement = lastFrame.timestamp - this.lastEndMovementTime;
if (!this.performingMovement && this.performingGesture && timeSinceEndMovement > 500000) {
if (this.callbackInstance != null) {
this.callbackInstance.endOfGestureCallback(this.gestureData);
}
this.performingGesture = false;
}
}
}
