/// <summary> /// End of gesture movement callback. /// </summary> /// <param name='data'> /// Data of the gesture gathered. /// </param> public void endOfGestureCallback(GestureData data) { GestureUtils.EnumGestures gResult; Debug.Log("Gesture ended!"); List<Vector2> pointList = new List<Vector2>(); GestureData normalizedData = data.getNormalizedGestureData(CANVAS_SIZE); GestureFrame previousFrame = (GestureFrame)normalizedData.frames[0]; pointList.Add(new Vector2(previousFrame.position.x, previousFrame.position.y)); for (int i = 1; i < normalizedData.frames.Count; i++) { GestureFrame currentFrame = (GestureFrame)normalizedData.frames[i]; if ((Math.Abs(currentFrame.position.x - previousFrame.position.x) < NORMAL_FRAME_VARIATION) && (Math.Abs(currentFrame.position.y - previousFrame.position.y) < NORMAL_FRAME_VARIATION)) { pointList.Add(new Vector2(currentFrame.position.x, currentFrame.position.y)); previousFrame = currentFrame; } } //At gesture end, draw the normalized data. Vector2 previousPoint = pointList[0]; for (int i = 1; i < pointList.Count; i++) { Vector2 currentPoint = pointList[i]; DrawLine(canvas, previousPoint.x, previousPoint.y, currentPoint.x, currentPoint.y, Color.black); previousPoint = currentPoint; } canvas.Apply(); //The checker method only receives canvas if you want the points used drawn on the screen, //can just receive a points list otherwise. gResult = gu.GestureChecker(pointList, canvas, CANVAS_SIZE); switch(gResult) { case GestureUtils.EnumGestures.DownZigZag: Debug.Log("Downwards Zig Zag!"); status = "Downwards Zig Zag!"; break; case GestureUtils.EnumGestures.RightZigZag: Debug.Log("Rightwards Zig Zag!"); status = "Rightwards Zig Zag!"; break; case GestureUtils.EnumGestures.Square: Debug.Log("Square!"); status = "Square!"; break; default: Debug.Log("No Gesture found."); status = "Nothing."; break; } }
public int AddToQueue(GestureData data) { //Decrement the semaphore to make sure the spot is available this.requestGestureDataSem.WaitOne(); lock (_threadLock) { dataQueue.Enqueue(data); } //Increament the semaphore to indicate there is work to do int previousCount = handleRequests.Release(); return previousCount; }
private void OnClick(GestureData gestureData) { int unitUid = _order.units[_order.units.Count - 1]; var map = BattleProcedure.CurSession.Map; var mapGrid = map.GetMapGridByScreenPos(gestureData.touchPos); if (mapGrid != null) { var goal = mapGrid.Coord; _path = map.FindPath2Goal(unitUid, goal); if (_path.Count > _count + 1) { _path.RemoveRange(_count + 1, _path.Count - _count - 1); } map.ShowPath(_path, true); _order.paths.Add(_path); bFinish = true; } }
public void SendSpellToServer(RecognitionResult result, GestureData gestureData) { Debug.Log("SendSpellToServer"); if (gestureData != null) { SpellCastToJson values = new SpellCastToJson(); if (!testMode) { values.match_id = SessionData.Match.MatchID; } values.gestureData = gestureData; /*foreach (GestureLine line in gestureData.lines) * { * foreach (Vector2 point in line.points) * { * values.x.Add(point.x); * values.y.Add(point.y); * } * }*/ if (socket) { if (result.gesture == attackSpell) { JSONObject dataObj = new JSONObject(values.SaveToString()); //Debug.Log(dataObj); socket.Emit("player:cast:spell", dataObj); } else if (result.gesture == defendSpell) { Debug.Log("defend"); JSONObject dataObj = new JSONObject(values.SaveToString()); //Debug.Log(dataObj); socket.Emit("player:defend:spell", dataObj); } } } }
//检测姿势是否完成 private static void CheckPoseComplete(ref GestureData gestureData, float timestamp, Vector3 jointPos, bool isInPose, float durationToComplete) { if (isInPose) { float timeLeft = timestamp - gestureData.timestamp; gestureData.progress = durationToComplete > 0f ? Mathf.Clamp01(timeLeft / durationToComplete) : 1.0f; if (timeLeft >= durationToComplete) { gestureData.timestamp = timestamp; gestureData.jointPos = jointPos; gestureData.state++; gestureData.complete = true; } } else { SetGestureCancelled(ref gestureData); } }
void Start() { shootingGestureData = new GestureData { downPoint = new Vector3(0, 0, 0), dragPoint = new Vector3(0, 0, 0), index = -1 }; movementGestureData = new GestureData { downPoint = new Vector3(0, 0, 0), dragPoint = new Vector3(0, 0, 0), index = -1 }; TextAsset[] gesturesXml = Resources.LoadAll <TextAsset>("GestureSet/10-stylus-MEDIUM/"); foreach (TextAsset gestureXml in gesturesXml) { trainingSet.Add(GestureIO.ReadGestureFromXML(gestureXml.text)); } }
private void OnLongTap(GestureData data) { if (selectUnit != null) { Vector3 tapMapPos; MapCoordinates tapMapCoord; //检测 长按的地图位是否包含有效单位 if (GameOperation.Instance.MapPosSelector(data.pos, out tapMapPos)) { tapMapCoord = MapCoordinates.FromPosition(tapMapPos); if (selectPath.Count > 0) { MapCoordinates pathEnd = selectPath[selectPath.Count - 1]; var addPath = MapManager.Instance.FindPath(pathEnd, tapMapCoord); var astarPath = MapManager.Instance.FindPath(selectUnit.pos, tapMapCoord); if ((selectPath.Count + addPath.Count - 1) > astarPath.Count) { selectPath = astarPath; } else { for (int i = 1; i < addPath.Count; ++i) { selectPath.Add(addPath[i]); } } } else { var astarPath = MapManager.Instance.FindPath(selectUnit.pos, tapMapCoord); selectPath = astarPath; } } } }
public void Process(Vector2 position) { if (!_processor.InputZone.Contains(position)) { Finish(); } else if (_currentDirection == Directions.None) { float distance = Vector2.Distance(_initialPosition, position); if (distance > _minimumAmplitude) { GestureData data = GetGestureData(_initialPosition, position); _currentDirection = data.direction; // handle gesture via interface if (_processor != null) { _processor.ProcessGesture(data, distance); } } } }
//=========================== event ================================ private void OnLongTapDown(GestureData data) { Vector3 tapMapPos; MapCoordinates tapMapCoord; Unit unit; //检测 长按的地图位是否包含有效单位 if (GameOperation.Instance.MapPosSelector(data.pos, out tapMapPos)) { tapMapCoord = MapCoordinates.FromPosition(tapMapPos); if (MapManager.Instance.FindUnit(tapMapCoord, out unit)) { selectUnit = unit; SwitchOperationMode(BattleOperationMode.SelectPath); } else { return; } } ; }
void Update() { if (Input.GetKeyDown(KeyCode.Escape)) { OnEsc?.Invoke(KeyCode.Escape); } if (Input.touchCount == 0) { _isMoving = false; } else { foreach (var touch in Input.touches) { //register new input if (touch.phase == TouchPhase.Began) { //prevent UI taps if (EventSystem.current.IsPointerOverGameObject(touch.fingerId)) { continue; } //prevent taps while drag if (_isMoving) { continue; } GestureData gestureData = new GestureData(); gestureData.fingerId = touch.fingerId; gestureData.Type = GestureType.None; gestureData.startPosition = touch.position; gestureData.endPosition = touch.position; gestureData.deltaPosition = Vector2.zero; gestureData.time = 0; _gestures.Add(gestureData); ++count; } else //process existing input { int index = FindTouchIndexById(touch.fingerId); if (index == -1) { continue; } _gestures[index].endPosition = touch.position; _gestures[index].deltaPosition = touch.deltaPosition; _gestures[index].time += touch.deltaTime; float distance = (_gestures[index].endPosition - _gestures[index].startPosition).magnitude; if (touch.phase == TouchPhase.Moved) { if (distance > minDistance || _isMoving) { _isMoving = true; if (_gestures[index].time > maxTime) { _gestures[index].Type = GestureType.Drag; OnDrag.Invoke(_gestures[index]); } } } else if (touch.phase == TouchPhase.Ended) { if (distance < minDistance && _gestures[index].time < maxTime) { OnTap.Invoke(_gestures[index]); } else if (distance > minDistance && _gestures[index].time < maxTime) { _isMoving = false; OnSwipe.Invoke(_gestures[index]); } else if (distance > minDistance && _gestures[index].time > maxTime) { _isMoving = false; OnDrag.Invoke(_gestures[index]); } _gestures.RemoveAt(index); --count; } } //end proccess existing input } } }
// estimate the next state and completeness of the gesture public static void CheckForGesture(long userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked) { if (gestureData.complete) { return; } switch (gestureData.gesture) { case Gestures.MyRaiseUpRight: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[head]) //右手和头部能被追踪到 { Hip = jointsPos[head].y; SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[head] && jointsPos[rightHandIndex].y - jointsPos[head].y > 0.15f; //右手比头高一定的距离,这个距离几乎能适应所有年龄断的人 if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; } break; case Gestures.MyRaiseUpLeft: switch (gestureData.state) { case 0: if (jointsTracked[leftHandIndex] && jointsTracked[head]) //右手和头部能被追踪到 { Hip = jointsPos[head].y; SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[head] && jointsPos[leftHandIndex].y - jointsPos[head].y > 0.15f; //右手比头高一定的距离,这个距离几乎能适应所有年龄断的人 if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; } break; case Gestures.Myjump: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[AnkleLeftIndex] && jointsTracked[AnkleRightIndex]) //不用脚跟,采用脚踝 { ankleLeft_y = jointsPos[AnkleLeftIndex].y; SetGestureJoint(ref gestureData, timestamp, AnkleRightIndex, jointsPos[AnkleRightIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) //不用脚跟,采用脚踝 { bool isInPose = jointsTracked[AnkleLeftIndex] && jointsTracked[AnkleRightIndex] && (jointsPos[AnkleRightIndex].y - gestureData.jointPos.y) > 0.1f && (jointsPos[AnkleLeftIndex].y - ankleLeft_y) > 0.1f;//脚踝高度差10cm if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; case Gestures.MySquat: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[hipCenterIndex]) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.2f && (jointsPos[hipCenterIndex].y - jointsPos[footRightIndex].y) < 0.4f;//臀部下降0.2m,并且和臀部的距离差值降低. if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; case Gestures.Walk: switch (gestureData.state) { case 0: //走路的第一阶段,左脚右脚一高一低 if (jointsTracked[footRightIndex] && jointsTracked[footleftIndex] && Mathf.Abs(jointsPos[footRightIndex].y - jointsPos[footleftIndex].y) > 0.15f) { SetGestureJoint(ref gestureData, timestamp, footRightIndex, jointsPos[footRightIndex]); gestureData.progress = 0.5f; //过程时间为0.5s(不知道有什么用) } break; case 1: if ((timestamp - gestureData.timestamp) < 0.5f) //如果两个姿势的时间差不超过1秒 { bool isInPose = jointsTracked[footRightIndex] && jointsTracked[footleftIndex] && Mathf.Abs(jointsPos[footRightIndex].y - jointsPos[footleftIndex].y) < 0.03f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { SetGestureCancelled(ref gestureData); } break; } break; case Gestures.MyMoveLeft: switch (gestureData.state) { case 0: if (jointsTracked[hipCenterIndex]) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.1f; } break; case 1: if (timestamp - gestureData.timestamp < 1.0f) { bool isInPose = (jointsTracked[hipCenterIndex]) && (gestureData.jointPos.x - jointsPos[hipCenterIndex].x > 0.15f); ; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { SetGestureCancelled(ref gestureData); } break; } break; case Gestures.MyMoveRight: switch (gestureData.state) { case 0: if (jointsTracked[hipCenterIndex]) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.1f; } break; case 1: if (timestamp - gestureData.timestamp < 1.0f) { bool isInPose = (jointsTracked[hipCenterIndex]) && (gestureData.jointPos.x - jointsPos[hipCenterIndex].x < -0.15f); ; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { SetGestureCancelled(ref gestureData); } break; } break; } }
private static void CheckPoseComplete(ref GestureData gestureData, float timestamp, Vector3 jointPos, bool isInPose, float durationToComplete) { if(isInPose) { float timeLeft = timestamp - gestureData.timestamp; gestureData.progress = durationToComplete > 0f ? Mathf.Clamp01(timeLeft / durationToComplete) : 1.0f; if(timeLeft >= durationToComplete) { gestureData.timestamp = timestamp; gestureData.jointPos = jointPos; gestureData.state++; gestureData.complete = true; } } else { SetGestureCancelled(ref gestureData); } }
private static void SetGestureJoint(ref GestureData gestureData, float timestamp, int joint, Vector3 jointPos) { gestureData.joint = joint; gestureData.jointPos = jointPos; gestureData.timestamp = timestamp; gestureData.state++; }
/** @brief Get the details of the fired gesture with the given index. @param[in] index - the zero-based index of the requested fired gesture. @param[out] gestureData - the information for the fired gesture. @note The gesture index must be between 0 and [QueryFiredGesturesNumber() - 1] @return PXCM_STATUS_NO_ERROR - operation succeeded. @return PXCM_STATUS_PARAM_UNSUPPORTED - invalid input parameter. @see GestureData @see QueryFiredGesturesNumber */ public pxcmStatus QueryFiredGestureData(Int32 index, out GestureData gestureData) { return QueryFiredGestureDataINT(instance, index, out gestureData); }
internal static Boolean IsGestureFiredByHandINT(IntPtr instance, String gestureName, Int32 handID, out GestureData gestureData) { gestureData = new GestureData(); return(PXCMHandData_IsGestureFiredByHand(instance, gestureName, handID, gestureData)); }
private static extern Boolean PXCMHandData_IsGestureFiredByHand(IntPtr instance, String gestureName, Int32 handID, [Out] GestureData gestureData);
internal static pxcmStatus QueryFiredGestureDataINT(IntPtr instance, Int32 index, out GestureData gestureData) { gestureData = new GestureData(); return(PXCMHandData_QueryFiredGestureData(instance, index, gestureData)); }
private static extern pxcmStatus PXCMHandData_QueryFiredGestureData(IntPtr instance, Int32 index, [Out] GestureData gestureData);
// estimate the next state and completeness of the gesture public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked) { if(gestureData.complete) return; switch(gestureData.gesture) { // check for RaiseHand case Gestures.RaiseHand: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } else if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f : jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for Psi case Gestures.Psi: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for Wave case Gestures.Wave: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture - phase 2 if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f; if(isInPose) { gestureData.timestamp = timestamp; gestureData.state++; gestureData.progress = 0.7f; } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; case 2: // gesture phase 3 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; } break; // check for Click case Gestures.Click: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, as this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, as this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } break; case 1: // gesture - phase 2 if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f; if(isInPose) { gestureData.timestamp = timestamp; gestureData.jointPos = jointsPos[gestureData.joint]; gestureData.state++; gestureData.progress = 0.7f; } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; case 2: // gesture phase 3 = complete if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; } break; // check for SweepLeft case Gestures.SweepLeft: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } // else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && // (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0f) // { // SetGestureJoint(ref gestureData, timestamp, leftHandIndex); // gestureData.jointPos = jointsPos[leftHandIndex]; // gestureData.progress = 0.5f; // } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.2f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; } break; // check for SweepRight case Gestures.SweepRight: switch(gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && // (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < 0f) // { // SetGestureJoint(ref gestureData, timestamp, rightHandIndex); // gestureData.jointPos = jointsPos[rightHandIndex]; // gestureData.progress = 0.5f; // } // else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.2f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture gestureData.state = 0; gestureData.progress = 0f; } break; } break; // check for RightHandCursor case Gestures.RightHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[rightHandIndex]) { gestureData.joint = rightHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[rightHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.5f; } break; } break; // check for LeftHandCursor case Gestures.LeftHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[leftHandIndex]) { gestureData.joint = leftHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[leftHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.5f; } break; } break; } }
private static void SetWheelRotation(uint userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos) { float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y); gestureData.screenPos.z = angle; }
/// <summary> /// End of gesture movement callback. /// </summary> /// <param name='data'> /// Data of the gesture gathered. /// </param> public void endOfGestureCallback(GestureData data) { GestureUtils.EnumGestures gResult; Debug.Log("Gesture ended!"); List <Vector2> pointList = new List <Vector2>(); GestureData normalizedData = data.getNormalizedGestureData(CANVAS_SIZE); GestureFrame previousFrame = (GestureFrame)normalizedData.frames[0]; pointList.Add(new Vector2(previousFrame.position.x, previousFrame.position.y)); for (int i = 1; i < normalizedData.frames.Count; i++) { GestureFrame currentFrame = (GestureFrame)normalizedData.frames[i]; if ((Math.Abs(currentFrame.position.x - previousFrame.position.x) < NORMAL_FRAME_VARIATION) && (Math.Abs(currentFrame.position.y - previousFrame.position.y) < NORMAL_FRAME_VARIATION)) { pointList.Add(new Vector2(currentFrame.position.x, currentFrame.position.y)); previousFrame = currentFrame; } } //At gesture end, draw the normalized data. Vector2 previousPoint = pointList[0]; for (int i = 1; i < pointList.Count; i++) { Vector2 currentPoint = pointList[i]; DrawLine(canvas, previousPoint.x, previousPoint.y, currentPoint.x, currentPoint.y, Color.black); previousPoint = currentPoint; } canvas.Apply(); //The checker method only receives canvas if you want the points used drawn on the screen, //can just receive a points list otherwise. gResult = gu.GestureChecker(pointList, canvas, CANVAS_SIZE); switch (gResult) { case GestureUtils.EnumGestures.DownZigZag: Debug.Log("Downwards Zig Zag!"); status = "Downwards Zig Zag!"; break; case GestureUtils.EnumGestures.RightZigZag: Debug.Log("Rightwards Zig Zag!"); status = "Rightwards Zig Zag!"; break; case GestureUtils.EnumGestures.Square: Debug.Log("Square!"); status = "Square!"; break; default: Debug.Log("No Gesture found."); status = "Nothing."; break; } }
public int ProcessNewGestureData(GestureData newGestureData) { int result = (int)ResultCodes.Success; do { SkeletonData trackedSkeleton = (from skeletons in newGestureData.GetSkeletonFrame().Skeletons where skeletons.TrackingState == SkeletonTrackingState.Tracked select skeletons).FirstOrDefault(); if (trackedSkeleton == null) { result = (int)ResultCodes.NullTrackedSkeleton; break; } GestureBase.NuiElement nuiElement = new GestureBase.NuiElement(); nuiElement.SetSkeletonFrame(newGestureData.GetSkeletonFrame()); nuiElement.SetTimeStamp(DateTime.Now); if (legsGesture != null) legsGesture.AddToQueue(nuiElement); if (mouseGesture != null) mouseGesture.AddToQueue(nuiElement); if (frontbackGesture != null) frontbackGesture.AddToQueue(nuiElement); if (jumpGesture != null) jumpGesture.AddToQueue(nuiElement); if (shootGesture != null) shootGesture.AddToQueue(nuiElement); if (!this.stopVisualThreads) { TransformedSkeletonDrawing.SkeletalDrawingData newDrawingData = new TransformedSkeletonDrawing.SkeletalDrawingData(); newDrawingData.SetSkeletalFrame(newGestureData.GetSkeletonFrame()); this.drawSkeleton.AddToQueue(newDrawingData); } } while (false); return result; }
/** @brief Return whether the specified gesture is fired for a specific hand in the current frame, and if so retrieve its data. @param[in] gestureName - the name of the gesture to be checked. @param[in] handID - the ID of the hand whose alert should be retrieved. @param[out] gestureData - the information for the fired gesture. @return true if the gesture was fired, false otherwise. @see GestureData */ public Boolean IsGestureFiredByHand(String gestureName, Int32 handID, out GestureData gestureData) { return IsGestureFiredByHandINT(instance, gestureName, handID, out gestureData); }
public GestureInfos(GestureData data) { this.data = data; }
private static void SetZoomFactor(uint userId, ref GestureData gestureData, float initialZoom, ref Vector3[] jointsPos, ref bool[] jointsTracked) { Vector3 vectorZooming = jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; if(gestureData.tagFloat == 0f || gestureData.userId != userId) { gestureData.tagFloat = 0.5f; // this is 100% } float distZooming = vectorZooming.magnitude; gestureData.screenPos.z = initialZoom + (distZooming / gestureData.tagFloat); }
void LateUpdate() { #if UNITY_IOS || UNITY_ANDROID if (curState == OperationState.Normal) { var touchs = new List <Touch>(); //筛选有效touch for (int i = 0; i < Input.touches.Length; ++i) { if (Input.touches[i].phase == TouchPhase.Began && EventSystem.current.IsPointerOverGameObject(Input.touches[i].fingerId)) { ignoreTouchs.Add(Input.touches[i].fingerId); } if (IsTouchIgnored(Input.touches[i]) == -1) { touchs.Add(Input.touches[i]); } if (Input.touches[i].phase == TouchPhase.Ended) { int index = IsTouchIgnored(Input.touches[i]); if (index > 0) { ignoreTouchs.RemoveAt(index); } } } //double tap check if (doubleTapActive) { doubleTapElapsed += Time.deltaTime; if (doubleTapElapsed > doubleClickInterval) { doubleTapActive = false; doubleTapElapsed = 0; } } int lastTouchNum = curTouchNum; curTouchNum = touchs.Count; switch (curTouchNum) { case 0: holdElapesd = 0; break; case 1: if (lastTouchNum == 0 && touchs[0].phase == TouchPhase.Began) { //初次点击 tapPos_1 = touchs[0].position; holdElapesd = 0; } else if (lastTouchNum == 1) { if (touchs[0].phase == TouchPhase.Moved) { //Swipe var data = new GestureData(); data.type = GestureType.Swipe; data.offset = touchs[0].position - tapPos_1; if (onSwipe != null) { onSwipe.Invoke(data); } tapPos_1 = touchs[0].position; } else if (touchs[0].phase == TouchPhase.Stationary) { holdElapesd += Time.deltaTime; } else if (touchs[0].phase == TouchPhase.Ended) { if (holdElapesd <= this.clickInterval) { var data = new GestureData(); //TODO data if (onSingleTap != null) { onSingleTap.Invoke(data); } holdElapesd = 0; if (doubleTapActive) { if (doubleTapElapsed <= doubleClickInterval) { var doubleData = new GestureData(); //TODO data if (onDoubleTap != null) { onDoubleTap.Invoke(doubleData); } doubleTapActive = false; } } else { doubleTapActive = true; doubleTapElapsed = 0; } } } } break; case 2: default: if (lastTouchNum < 2) { //首次记录位置 tapPos_1 = touchs[0].position; tapPos_2 = touchs[1].position; } if (touchs[0].phase == TouchPhase.Moved || touchs[1].phase == TouchPhase.Moved) { //Pinch var data = new GestureData(); data.type = GestureType.Pinch; data.value = Vector2.Distance(touchs[0].position, touchs[1].position) - Vector2.Distance(tapPos_1, tapPos_2); if (onPinch != null) { onPinch.Invoke(data); } tapPos_1 = touchs[0].position; tapPos_2 = touchs[1].position; } break; } } else { if (Input.GetTouch(selectFingerId).phase == TouchPhase.Moved) { Vector3 pos = Input.GetTouch(selectFingerId).position; if (pos.y > cardExcuteBound) { switch (curState) { case OperationState.HoldCard: switch (selectCardType) { case CardExcuteType.SelectPos: SwitchOperationState(OperationState.SelectTarget); break; } break; } } else { SwitchOperationState(OperationState.HoldCard); } } if (Input.GetTouch(selectFingerId).phase == TouchPhase.Ended) { Vector3 pos = Input.GetTouch(selectFingerId).position; if (pos.y > cardExcuteBound) { var data = new GestureData(); switch (curState) { case OperationState.HoldCard: if (onReleaseCard != null) { onReleaseCard.Invoke(data); } break; case OperationState.SelectTarget: if (onReleaseCard != null) { onReleaseCard.Invoke(data); } break; } } SwitchOperationState(OperationState.Normal); } } #endif #if UNITY_STANDALONE_WIN //double tap check if (doubleTapActive) { doubleTapElapsed += Time.deltaTime; if (doubleTapElapsed > doubleClickInterval) { doubleTapActive = false; doubleTapElapsed = 0; } } //check valid if (Input.GetMouseButtonUp(0)) { ignoreMouse = false; } if (Input.GetMouseButtonDown(0)) { if (EventSystem.current.IsPointerOverGameObject()) { ignoreMouse = true; } else { //初次点击 tapPos_0 = Input.mousePosition; tapPos_1 = Input.mousePosition; holdElapesd = 0; var data = new GestureData(); data.pos = tapPos_0; if (onSingleDown != null) { onSingleDown.Invoke(data); } } } if (!ignoreMouse) { if (Input.GetMouseButton(0)) { Vector2 curPos = Input.mousePosition; //longTap var dist = Vector2.Distance(curPos, tapPos_0); if (holdElapesd > this.longClickInterval) { var longTapData = new GestureData(); longTapData.pos = tapPos_0; if (dist <= clickOffsetBound && longTapActive == false) { longTapActive = true; if (onLongTapDown != null) { onLongTapDown.Invoke(longTapData); } } longTapData.pos = curPos; if (onLongTap != null) { onLongTap.Invoke(longTapData); } } //Swipe var data = new GestureData(); data.type = GestureType.Swipe; data.offset = (curPos - tapPos_1) * MouseSwipeFactor; if (onSwipe != null) { onSwipe.Invoke(data); } tapPos_1 = Input.mousePosition; holdElapesd += Time.deltaTime; } if (Input.GetMouseButtonUp(0)) { if (holdElapesd <= this.clickInterval && Vector2.Distance(tapPos_0, Input.mousePosition) <= clickOffsetBound) { var data = new GestureData(); //TODO data if (onSingleTap != null) { onSingleTap.Invoke(data); } holdElapesd = 0; if (doubleTapActive) { if (doubleTapElapsed <= doubleClickInterval) { var doubleData = new GestureData(); //TODO data if (onDoubleTap != null) { onDoubleTap.Invoke(doubleData); } doubleTapActive = false; } } else { doubleTapActive = true; doubleTapElapsed = 0; } } if (longTapActive) { longTapActive = false; if (onLongTapUp != null) { GestureData data = new GestureData(); this.onLongTapUp.Invoke(data); } } } } if (Input.GetAxis("Mouse ScrollWheel") != 0) { //Pinch var data = new GestureData(); data.type = GestureType.Pinch; data.value = Input.GetAxis("Mouse ScrollWheel") * MouseSwipeFactor; if (onPinch != null) { if (onPinch != null) { onPinch.Invoke(data); } } } #endif }
private static void SetScreenPos(uint userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked) { Vector3 handPos = jointsPos[rightHandIndex]; Vector3 elbowPos = jointsPos[rightElbowIndex]; Vector3 shoulderPos = jointsPos[rightShoulderIndex]; bool calculateCoords = false; if(gestureData.joint == rightHandIndex) { if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]) { calculateCoords = true; } } else if(gestureData.joint == leftHandIndex) { if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]) { handPos = jointsPos[leftHandIndex]; elbowPos = jointsPos[leftElbowIndex]; shoulderPos = jointsPos[leftShoulderIndex]; calculateCoords = true; } } if(calculateCoords) { if(gestureData.tagFloat == 0f || gestureData.userId != userId) { // get length from shoulder to hand (screen range) Vector3 shoulderToElbow = elbowPos - shoulderPos; Vector3 elbowToHand = handPos - elbowPos; gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude); } Vector3 shoulderToHand = handPos - shoulderPos; gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat); gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat); //Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat)); } }
/** * @brief Get the details of the fired gesture with the given index. * * @param[in] index - the zero-based index of the requested fired gesture. * @param[out] gestureData - the information for the fired gesture. * * @note The gesture index must be between 0 and [QueryFiredGesturesNumber() - 1] * * @return PXCM_STATUS_NO_ERROR - operation succeeded. * @return PXCM_STATUS_PARAM_UNSUPPORTED - invalid input parameter. * * @see GestureData * @see QueryFiredGesturesNumber */ public pxcmStatus QueryFiredGestureData(Int32 index, out GestureData gestureData) { return(QueryFiredGestureDataINT(instance, index, out gestureData)); }
/// <summary> /// Detects the movement and builds the frame list when movement is detected. /// </summary> /// <param name='frame'> /// Leap frame. /// </param> public void DetectMovement(Frame frame) { GestureFrame gestureFrame = new GestureFrame(frame.Pointables[0].TipPosition, frame.Pointables[0].TipVelocity, frame.Timestamp); this.detectionHeap.Add(gestureFrame); Vector3 acceleration = Vector3.zero; Vector3 delta = Vector3.zero; Vector3 movementWork = Vector3.zero; Vector3 kinecticEnergy = Vector3.zero; float movementTotalWork; float kinecticEnergyTotal; if (this.detectionHeap.Count > this.maximumHeapSize) { this.detectionHeap.RemoveAt(0); GestureFrame firstFrame = (GestureFrame)this.detectionHeap[0]; GestureFrame lastFrame = (GestureFrame)this.detectionHeap[this.maximumHeapSize - 1]; float deltaTime = lastFrame.timestamp - firstFrame.timestamp; acceleration.x = (lastFrame.velocity.x - firstFrame.velocity.x) / deltaTime; acceleration.y = (lastFrame.velocity.y - firstFrame.velocity.y) / deltaTime; acceleration.z = (lastFrame.velocity.z - firstFrame.velocity.z) / deltaTime; this.movementDirection.x = acceleration.x > 0 ? 1 : -1; this.movementDirection.y = acceleration.y > 0 ? 1 : -1; this.movementDirection.z = acceleration.z > 0 ? 1 : -1; delta.x = (lastFrame.position.x - firstFrame.position.x); delta.y = (lastFrame.position.y - firstFrame.position.y); delta.z = (lastFrame.position.z - firstFrame.position.z); /* * http://en.wikipedia.org/wiki/Work_(physics) * Work calculation, using the acceleration previously calculated as the force to move the finger from one point to another */ movementWork.x = delta.x * acceleration.x; movementWork.y = delta.y * acceleration.y; movementWork.z = delta.z * acceleration.z; /* * http://en.wikipedia.org/wiki/Kinetic_energy * Measures the pointable kinectic energy to detect when gesture have ended. */ kinecticEnergy.x = Mathf.Pow(lastFrame.velocity.x, 2) * 0.5f * 1.0e-6f; kinecticEnergy.y = Mathf.Pow(lastFrame.velocity.y, 2) * 0.5f * 1.0e-6f; kinecticEnergy.z = Mathf.Pow(lastFrame.velocity.z, 2) * 0.5f * 1.0e-6f; kinecticEnergyTotal = kinecticEnergy.x + kinecticEnergy.y; if (Mathf.Abs(movementWork.x) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.x += movementWork.x; } if (Mathf.Abs(movementWork.y) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.y += movementWork.y; } if (Mathf.Abs(movementWork.z) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.z += movementWork.z; } if (!this.performingMovement && lastFrame.timestamp - this.lastStoppedTime > 50000.0f) { this.movementTotalWork.x = 0.0f; this.movementTotalWork.y = 0.0f; this.movementTotalWork.z = 0.0f; this.lastStoppedTime = lastFrame.timestamp; } movementTotalWork = this.movementTotalWork.x + this.movementTotalWork.y; if (movementTotalWork > this.minimumStartMovementTotalWork && !this.performingMovement) { this.movementTotalWork.x = 0; this.movementTotalWork.y = 0; this.movementTotalWork.z = 0; this.performingMovement = true; if (!this.performingGesture) { this.performingGesture = true; this.gestureData = new GestureData(); if (this.callbackInstance != null) { this.callbackInstance.beginOfGestureCallback(); } } } else if (this.performingMovement && kinecticEnergyTotal < this.minimumEndMovementKinectEnergy) { this.movementTotalWork.x = 0.0f; this.movementTotalWork.y = 0.0f; this.movementTotalWork.z = 0.0f; this.performingMovement = false; this.lastStoppedTime = this.lastEndMovementTime = lastFrame.timestamp; } if (this.performingGesture) { this.gestureData.appendFrame(gestureFrame); } float timeSinceEndMovement = lastFrame.timestamp - this.lastEndMovementTime; if (!this.performingMovement && this.performingGesture && timeSinceEndMovement > 500000) { if (this.callbackInstance != null) { this.callbackInstance.endOfGestureCallback(this.gestureData); } this.performingGesture = false; } } }
/** * @brief Check whether a gesture was fired and if so return its details. * * @param[in] gestureName - the name of the gesture to be checked. * @param[out] gestureData - the information for the fired gesture. * * @return true if the gesture was fired, false otherwise. * * @see GestureData */ public Boolean IsGestureFired(String gestureName, out GestureData gestureData) { return(IsGestureFiredINT(instance, gestureName, out gestureData)); }
/** * @brief Return whether the specified gesture is fired for a specific hand in the current frame, and if so retrieve its data. * @param[in] gestureName - the name of the gesture to be checked. * @param[in] handID - the ID of the hand whose alert should be retrieved. * @param[out] gestureData - the information for the fired gesture. * * @return true if the gesture was fired, false otherwise. * @see GestureData */ public Boolean IsGestureFiredByHand(String gestureName, Int32 handID, out GestureData gestureData) { return(IsGestureFiredByHandINT(instance, gestureName, handID, out gestureData)); }
/// <summary> /// Gets the normalized gesture data. /// </summary> /// <returns> /// The normalized gesture data. /// </returns> /// <param name='resolution'> /// Resolution to normalize to. /// </param> public GestureData getNormalizedGestureData(float resolution) { var dx = (this.maximumX - this.minimumX) * 1.0f; var dy = (this.maximumY - this.minimumY) * 1.0f; if (dx > dy) dy = dx; else dx = dy; GestureData normalizedData = new GestureData(); for (var i = 0; i < this.frames.Count; i++) { GestureFrame frame = (GestureFrame)this.frames[i]; float x = (frame.position.x * 1.0f - this.minimumX) / dx * resolution; float y = resolution - (frame.position.y * 1.0f - this.minimumY) / dy * resolution; GestureFrame normalizedFrame = new GestureFrame(new Vector(x, y, 0.0f), frame.velocity, frame.timestamp); normalizedData.appendFrame(normalizedFrame); } return normalizedData; }
private void OnLoneTapUp(GestureData data) { selectUnit = null; selectPath.Clear(); SwitchOperationMode(BattleOperationMode.Normal); }
public ClassificationAlgorithm(GestureData gestureData) { Train(gestureData); }
void Update() { if (handModelManager.leapProvider) { Frame frame = handModelManager.leapProvider.CurrentFrame; if (Input.GetKeyDown(KeyCode.Space) && experienceState == ExperienceState.Idle) { typeBox_white.SetActive(false); typeBox_blue.SetActive(true); experienceState = ExperienceState.Chatting; } else if (Input.GetKeyDown(KeyCode.Space) && experienceState == ExperienceState.Chatting) { typeBox_white.SetActive(true); typeBox_blue.SetActive(false); experienceState = ExperienceState.Idle; } else if (Input.GetKeyDown(KeyCode.Backspace) /* && experienceState == ExperienceState.Chatting*/) { RemoveWordsMark(Sign.Hello, Sign.My, Sign.Name, Sign.Sana, Sign.What, Sign.Your, Sign.Nice, Sign.Meet, Sign.You, Sign.How, Sign.Good); } else if (Input.GetKeyDown(KeyCode.R) && experienceState == ExperienceState.Idle) { StartCoroutine(CountingDown()); experienceState = ExperienceState.Recording; } if (frame.Hands.Count > 0) { var calibration = GetDeltaRotMaterix(); List <Hand> hands = frame.Hands; foreach (var hand in hands) { if (hand.IsLeft) { //Debug.Log(experienceState.ToString()); if (GetGesture(hand, Quote.l_Dirs, calibration) && experienceState == ExperienceState.Idle) { typeBox_white.SetActive(false); typeBox_blue.SetActive(true); experienceState = ExperienceState.Chatting; } if (Input.GetKeyDown(KeyCode.Alpha2) || isCapturing) { captured_L_Dirs = new Vector3[6]; Debug.Log("Capturing left hand..."); Capture(hand, out captured_L_Dirs); } } if (hand.IsRight) { if (GetGesture(hand, Quote.r_Dirs, calibration) && experienceState == ExperienceState.Chatting) { typeBox_white.SetActive(true); typeBox_blue.SetActive(false); experienceState = ExperienceState.Idle; } switch (experienceState) { case ExperienceState.Chatting: Translate(hand, calibration); break; case ExperienceState.Recording: break; case ExperienceState.Idle: DetectSentencesAndPlay(); break; default: break; } if (Input.GetKeyDown(KeyCode.Alpha1) || isCapturing) { captured_R_Dirs = new Vector3[6]; Debug.Log("Capturing left hand..."); Capture(hand, out captured_R_Dirs); } } } } } if (isCapturing) { gestureBase[highlightedSentence] = new GestureData(captured_R_Dirs, captured_L_Dirs, false); } var typeBoxText = ""; foreach (var gesture in gestureBase) { if (gesture.Value.isMarked) { typeBoxText += gesture.Key.ToString() + ", "; } } typeBox.text = typeBoxText; isCapturing = false; captured_L_Dirs = captured_R_Dirs = null; }
protected abstract void Train(GestureData data);
// estimate the next state and completeness of the gesture public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked) { if(gestureData.complete) return; switch(gestureData.gesture) { // check for RaiseRightHand case Gestures.RaiseRightHand: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for RaiseLeftHand case Gestures.RaiseLeftHand: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for Psi case Gestures.Psi: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for Stop case Gestures.Stop: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0f && jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0f && jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.PoseCompleteDuration); break; } break; // check for Wave case Gestures.Wave: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture - phase 2 if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f; if(isInPose) { gestureData.timestamp = timestamp; gestureData.state++; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; case 2: // gesture phase 3 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Click case Gestures.Click: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } break; case 1: // gesture - phase 2 // if((timestamp - gestureData.timestamp) < 1.0f) // { // bool isInPose = gestureData.joint == rightHandIndex ? // jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f : // jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f; // // if(isInPose) // { // gestureData.timestamp = timestamp; // gestureData.jointPos = jointsPos[gestureData.joint]; // gestureData.state++; // gestureData.progress = 0.7f; // } // else // { // // check for stay-in-place // Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; // isInPose = distVector.magnitude < 0.05f; // // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration); // } // } // else { // check for stay-in-place Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; bool isInPose = distVector.magnitude < 0.05f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration); // SetGestureCancelled(gestureData); } break; // case 2: // gesture phase 3 = complete // if((timestamp - gestureData.timestamp) < 1.0f) // { // bool isInPose = gestureData.joint == rightHandIndex ? // jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f : // jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f; // // if(isInPose) // { // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); // } // } // else // { // // cancel the gesture // SetGestureCancelled(ref gestureData); // } // break; } break; // check for SweepLeft case Gestures.SweepLeft: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } // else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && // (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0f) // { // SetGestureJoint(ref gestureData, timestamp, leftHandIndex); // gestureData.jointPos = jointsPos[leftHandIndex]; // gestureData.progress = 0.5f; // } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.2f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SweepRight case Gestures.SweepRight: switch(gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && // (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < 0f) // { // SetGestureJoint(ref gestureData, timestamp, rightHandIndex); // gestureData.jointPos = jointsPos[rightHandIndex]; // gestureData.progress = 0.5f; // } // else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.2f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SweepUp case Gestures.SweepUp: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) >= -0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) >= -0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].y - gestureData.jointPos.y) > 0.2f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].y - gestureData.jointPos.y) > 0.2f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SweepDown case Gestures.SweepDown: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) >= 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) >= 0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < -0.1f && (jointsPos[rightHandIndex].y - gestureData.jointPos.y) < -0.2f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < -0.1f && (jointsPos[leftHandIndex].y - gestureData.jointPos.y) < -0.2f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for RightHandCursor case Gestures.RightHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f) { gestureData.joint = rightHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[rightHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.7f; } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for LeftHandCursor case Gestures.LeftHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f) { gestureData.joint = leftHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[leftHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.7f; } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ZoomOut case Gestures.ZoomOut: switch(gestureData.state) { case 0: // gesture detection - phase 1 float distZoomOut = ((Vector3)(jointsPos[rightHandIndex] - jointsPos[leftHandIndex])).magnitude; if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && distZoomOut < 0.2f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && ((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f || (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f); if(isInPose) { SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ZoomIn case Gestures.ZoomIn: switch(gestureData.state) { case 0: // gesture detection - phase 1 float distZoomIn = ((Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]).magnitude; if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && distZoomIn >= 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagFloat = distZoomIn; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if((timestamp - gestureData.timestamp) < 1.0f) { bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && ((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f || (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f); if(isInPose) { SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Wheel case Gestures.Wheel: Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distWheel = vectorWheel.magnitude; switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && distWheel > 0.2f && distWheel < 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = vectorWheel; gestureData.tagFloat = distWheel; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && ((jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0f || (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0f && Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f); if(isInPose) { SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel); gestureData.timestamp = timestamp; gestureData.tagFloat = distWheel; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; case Gestures.Move: switch(gestureData.state) { case 0: if(jointsTracked[leftKneeIndex] && jointsTracked[rightKneeIndex] && Math.Abs(jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z) >0.1f) { SetGestureJoint(ref gestureData,timestamp,leftKneeIndex,jointsPos[leftKneeIndex]); } break; case 1: bool isInPose = jointsTracked[leftKneeIndex] && jointsTracked[rightKneeIndex] && Math.Abs(jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z) >0.1f; if(isInPose) { gestureData.timestamp = timestamp; gestureData.progress = 0.7f; gestureData.screenPos.z = jointsPos[leftKneeIndex].z - jointsPos[rightKneeIndex].z; } else { SetGestureCancelled(ref gestureData); } break; } break; } }
public abstract double ComputeError(GestureData data);
public static void Clear() { Data = new GestureData(); }
public MultivariateAlgorithm(GestureData input) : base(input) { }
/** @brief Check whether a gesture was fired and if so return its details. @param[in] gestureName - the name of the gesture to be checked. @param[out] gestureData - the information for the fired gesture. @return true if the gesture was fired, false otherwise. @see GestureData */ public Boolean IsGestureFired(String gestureName, out GestureData gestureData) { return IsGestureFiredINT(instance, gestureName, out gestureData); }
//设置姿势取消 private static void SetGestureCancelled(ref GestureData gestureData) { gestureData.state = 0; gestureData.progress = 0f; gestureData.cancelled = true; }
private static void SetGestureCancelled(ref GestureData gestureData) { gestureData.state = 0; gestureData.progress = 0f; gestureData.cancelled = true; }
internal static pxcmStatus QueryFiredGestureDataINT(IntPtr instance, Int32 index, out GestureData gestureData) { gestureData = new GestureData(); return PXCMHandData_QueryFiredGestureData(instance, index, gestureData); }
private static void SetScreenPos(uint userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked) { Vector3 handPos = jointsPos[rightHandIndex]; // Vector3 elbowPos = jointsPos[rightElbowIndex]; // Vector3 shoulderPos = jointsPos[rightShoulderIndex]; bool calculateCoords = false; if(gestureData.joint == rightHandIndex) { if(jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/) { calculateCoords = true; } } else if(gestureData.joint == leftHandIndex) { if(jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/) { handPos = jointsPos[leftHandIndex]; // elbowPos = jointsPos[leftElbowIndex]; // shoulderPos = jointsPos[leftShoulderIndex]; calculateCoords = true; } } if(calculateCoords) { // if(gestureData.tagFloat == 0f || gestureData.userId != userId) // { // // get length from shoulder to hand (screen range) // Vector3 shoulderToElbow = elbowPos - shoulderPos; // Vector3 elbowToHand = handPos - elbowPos; // gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude); // } if(jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex]) { Vector3 neckToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex]; Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex]; gestureData.tagVector2.x = rightToLeft.x; // * 1.2f; gestureData.tagVector2.y = neckToHips.y; // * 1.2f; if(gestureData.joint == rightHandIndex) { gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } else { gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } } // Vector3 shoulderToHand = handPos - shoulderPos; // gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat); // gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat); if(gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0) { Vector3 relHandPos = handPos - gestureData.tagVector; gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x); gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y); } //Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat)); } }
protected void SetScreenPos(long userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked) { Vector3 handPos = jointsPos[rightHandIndex]; bool calculateCoords = false; if(gestureData.joint == rightHandIndex) { if(jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/) { calculateCoords = true; } } else if(gestureData.joint == leftHandIndex) { if(jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/) { handPos = jointsPos[leftHandIndex]; calculateCoords = true; } } if(calculateCoords) { if(jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex]) { Vector3 shoulderToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex]; Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex]; gestureData.tagVector2.x = rightToLeft.x; // * 1.2f; gestureData.tagVector2.y = shoulderToHips.y; // * 1.2f; if(gestureData.joint == rightHandIndex) { gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } else { gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } } if(gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0) { Vector3 relHandPos = handPos - gestureData.tagVector; gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x); gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y); } } }
// private static void SetWheelRotation(uint userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos) // { // float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y); // gestureData.screenPos.z = angle; // } // estimate the next state and completeness of the gesture public static void CheckForGesture(uint userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked) { if(gestureData.complete) return; float bandSize = (jointsPos[shoulderCenterIndex].y - jointsPos[hipCenterIndex].y); float gestureTop = jointsPos[shoulderCenterIndex].y + bandSize / 2; float gestureBottom = jointsPos[shoulderCenterIndex].y - bandSize; float gestureRight = jointsPos[rightHipIndex].x; float gestureLeft = jointsPos[leftHipIndex].x; switch(gestureData.gesture) { // check for RaiseRightHand case Gestures.RaiseRightHand: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration); break; } break; // check for RaiseLeftHand case Gestures.RaiseLeftHand: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration); break; } break; // check for Psi case Gestures.Psi: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration); break; } break; // check for Tpose case Gestures.Tpose: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.07f Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration); break; } break; // check for Stop case Gestures.Stop: switch(gestureData.state) { case 0: // gesture detection if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.4f && (jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } else if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.4f && (jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = (gestureData.joint == rightHandIndex) ? (jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.4f && (jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) : (jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.4f && (jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f); Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.PoseCompleteDuration); break; } break; // check for Wave case Gestures.Wave: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture - phase 2 if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f; if(isInPose) { gestureData.timestamp = timestamp; gestureData.state++; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; case 2: // gesture phase 3 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Click case Gestures.Click: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } break; case 1: // gesture - phase 2 // if((timestamp - gestureData.timestamp) < 1.0f) // { // bool isInPose = gestureData.joint == rightHandIndex ? // jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f : // jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f; // // if(isInPose) // { // gestureData.timestamp = timestamp; // gestureData.jointPos = jointsPos[gestureData.joint]; // gestureData.state++; // gestureData.progress = 0.7f; // } // else // { // // check for stay-in-place // Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; // isInPose = distVector.magnitude < 0.05f; // // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration); // } // } // else { // check for stay-in-place Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; bool isInPose = distVector.magnitude < 0.05f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectWrapper.Constants.ClickStayDuration); // SetGestureCancelled(gestureData); } break; // case 2: // gesture phase 3 = complete // if((timestamp - gestureData.timestamp) < 1.0f) // { // bool isInPose = gestureData.joint == rightHandIndex ? // jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f : // jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && // Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && // (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f; // // if(isInPose) // { // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); // } // } // else // { // // cancel the gesture // SetGestureCancelled(ref gestureData); // } // break; } break; // check for SwipeLeft case Gestures.SwipeLeft: switch(gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f && // (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f) // { // SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); // gestureData.progress = 0.5f; // } if(jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && jointsPos[rightHandIndex].x <= gestureRight && jointsPos[rightHandIndex].x > gestureLeft) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.1f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { // bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && // Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && // (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.15f; bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && jointsPos[rightHandIndex].x < gestureLeft; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else if(jointsPos[rightHandIndex].x <= gestureRight) { float gestureSize = gestureRight - gestureLeft; gestureData.progress = gestureSize > 0.01f ? (gestureRight - jointsPos[rightHandIndex].x) / gestureSize : 0f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SwipeRight case Gestures.SwipeRight: switch(gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f && // (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f) // { // SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); // gestureData.progress = 0.5f; // } if(jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[leftHandIndex].x >= gestureLeft && jointsPos[leftHandIndex].x < gestureRight) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.1f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { // bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && // Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && // (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.15f; bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[leftHandIndex].x > gestureRight; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else if(jointsPos[leftHandIndex].x >= gestureLeft) { float gestureSize = gestureRight - gestureLeft; gestureData.progress = gestureSize > 0.01f ? (jointsPos[leftHandIndex].x - gestureLeft) / gestureSize : 0f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SwipeUp case Gestures.SwipeUp: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < 0.0f && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < 0.0f && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SwipeDown case Gestures.SwipeDown: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.15f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.15f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for RightHandCursor case Gestures.RightHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f) { gestureData.joint = rightHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[rightHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.7f; } else { // cancel the gesture //SetGestureCancelled(ref gestureData); gestureData.progress = 0f; } break; } break; // check for LeftHandCursor case Gestures.LeftHandCursor: switch(gestureData.state) { case 0: // gesture detection - phase 1 (perpetual) if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f) { gestureData.joint = leftHandIndex; gestureData.timestamp = timestamp; //gestureData.jointPos = jointsPos[leftHandIndex]; SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); gestureData.progress = 0.7f; } else { // cancel the gesture //SetGestureCancelled(ref gestureData); gestureData.progress = 0f; } break; } break; // check for ZoomOut case Gestures.ZoomOut: Vector3 vectorZoomOut = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distZoomOut = vectorZoomOut.magnitude; switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomOut < 0.3f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = 0f; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if((timestamp - gestureData.timestamp) < 1.5f) { float angleZoomOut = Vector3.Angle(gestureData.tagVector, vectorZoomOut) * Mathf.Sign(vectorZoomOut.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomOut < 1.5f && Mathf.Abs(angleZoomOut) < 20f; if(isInPose) { SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ZoomIn case Gestures.ZoomIn: Vector3 vectorZoomIn = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distZoomIn = vectorZoomIn.magnitude; switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomIn >= 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = distZoomIn; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if((timestamp - gestureData.timestamp) < 1.5f) { float angleZoomIn = Vector3.Angle(gestureData.tagVector, vectorZoomIn) * Mathf.Sign(vectorZoomIn.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomIn >= 0.2f && Mathf.Abs(angleZoomIn) < 20f; if(isInPose) { SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Wheel case Gestures.Wheel: Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distWheel = vectorWheel.magnitude; // Debug.Log(string.Format("{0}. Dist: {1:F1}, Tag: {2:F1}, Diff: {3:F1}", gestureData.state, // distWheel, gestureData.tagFloat, Mathf.Abs(distWheel - gestureData.tagFloat))); switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distWheel >= 0.3f && distWheel < 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = distWheel; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = turning wheel if((timestamp - gestureData.timestamp) < 1.5f) { float angle = Vector3.Angle(gestureData.tagVector, vectorWheel) * Mathf.Sign(vectorWheel.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distWheel >= 0.3f && distWheel < 0.7f && Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f; if(isInPose) { //SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel); gestureData.screenPos.z = angle; // wheel angle gestureData.timestamp = timestamp; gestureData.tagFloat = distWheel; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Jump case Gestures.Jump: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y > 0.9f) && (jointsPos[hipCenterIndex].y < 1.3f)) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y - gestureData.jointPos.y) > 0.15f && Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Squat case Gestures.Squat: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y <= 0.9f)) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.15f && Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Push case Gestures.Push: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f && (jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.2f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f && (jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.2f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.1f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Pull case Gestures.Pull: switch(gestureData.state) { case 0: // gesture detection - phase 1 if(jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f && (jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.3f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if(jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f && (jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.3f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.1f; if(isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // here come more gesture-cases } }
private static void SetWheelRotation(long userId, ref GestureData gestureData, Vector3 initialPos, Vector3 currentPos) { float angle = Vector3.Angle(initialPos, currentPos) * Mathf.Sign(currentPos.y - initialPos.y); gestureData.screenPos.z = angle; }
//设置在屏幕上的位置??????? private static void SetScreenPos(long userId, ref GestureData gestureData, ref Vector3[] jointsPos, ref bool[] jointsTracked) { Vector3 handPos = jointsPos[rightHandIndex]; // Vector3 elbowPos = jointsPos[rightElbowIndex]; // Vector3 shoulderPos = jointsPos[rightShoulderIndex]; bool calculateCoords = false; if (gestureData.joint == rightHandIndex) { if (jointsTracked[rightHandIndex] /**&& jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex]*/) { calculateCoords = true; } } else if (gestureData.joint == leftHandIndex) { if (jointsTracked[leftHandIndex] /**&& jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex]*/) { handPos = jointsPos[leftHandIndex]; // elbowPos = jointsPos[leftElbowIndex]; // shoulderPos = jointsPos[leftShoulderIndex]; calculateCoords = true; } } if (calculateCoords) { // if(gestureData.tagFloat == 0f || gestureData.userId != userId) // { // // get length from shoulder to hand (screen range) // Vector3 shoulderToElbow = elbowPos - shoulderPos; // Vector3 elbowToHand = handPos - elbowPos; // gestureData.tagFloat = (shoulderToElbow.magnitude + elbowToHand.magnitude); // } if (jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex]) { Vector3 shoulderToHips = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex]; Vector3 rightToLeft = jointsPos[rightShoulderIndex] - jointsPos[leftShoulderIndex]; gestureData.tagVector2.x = rightToLeft.x; // * 1.2f; gestureData.tagVector2.y = shoulderToHips.y; // * 1.2f; if (gestureData.joint == rightHandIndex) { gestureData.tagVector.x = jointsPos[rightShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } else { gestureData.tagVector.x = jointsPos[leftShoulderIndex].x - gestureData.tagVector2.x / 2; gestureData.tagVector.y = jointsPos[hipCenterIndex].y; } } // Vector3 shoulderToHand = handPos - shoulderPos; // gestureData.screenPos.x = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.x) / gestureData.tagFloat); // gestureData.screenPos.y = Mathf.Clamp01((gestureData.tagFloat / 2 + shoulderToHand.y) / gestureData.tagFloat); if (gestureData.tagVector2.x != 0 && gestureData.tagVector2.y != 0) { Vector3 relHandPos = handPos - gestureData.tagVector; gestureData.screenPos.x = Mathf.Clamp01(relHandPos.x / gestureData.tagVector2.x); gestureData.screenPos.y = Mathf.Clamp01(relHandPos.y / gestureData.tagVector2.y); } //Debug.Log(string.Format("{0} - S: {1}, H: {2}, SH: {3}, L : {4}", gestureData.gesture, shoulderPos, handPos, shoulderToHand, gestureData.tagFloat)); } }
// estimate the next state and completeness of the gesture /// <summary> /// estimate the state and progress of the given gesture. /// </summary> /// <param name="userId">User ID</param> /// <param name="gestureData">Gesture-data structure</param> /// <param name="timestamp">Current time</param> /// <param name="jointsPos">Joints-position array</param> /// <param name="jointsTracked">Joints-tracked array</param> public static void CheckForGesture(long userId, ref GestureData gestureData, float timestamp, ref Vector3[] jointsPos, ref bool[] jointsTracked) { if (gestureData.complete) return; float bandSize = (jointsPos[shoulderCenterIndex].y - jointsPos[hipCenterIndex].y); float gestureTop = jointsPos[shoulderCenterIndex].y + bandSize * 1.2f / 3f; float gestureBottom = jointsPos[shoulderCenterIndex].y - bandSize * 1.8f / 3f; /* float gestureRight = jointsPos[rightHipIndex].x; float gestureLeft = jointsPos[leftHipIndex].x; */ switch (gestureData.gesture) { // check for RaiseRightHand case Gestures.RaiseRightHand: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.1f && (jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); break; } break; // check for RaiseLeftHand case Gestures.RaiseLeftHand: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.1f && (jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); break; } break; // check for Psi case Gestures.Psi: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[shoulderCenterIndex] && (jointsPos[rightHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f && (jointsPos[leftHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[leftHandIndex] && jointsTracked[shoulderCenterIndex] && (jointsPos[rightHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f && (jointsPos[leftHandIndex].y - jointsPos[shoulderCenterIndex].y) > 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); break; } break; // check for Tpose case Gestures.Tpose: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.07f Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: // gesture complete bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightElbowIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightShoulderIndex].y) < 0.1f && // 0.7f jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftElbowIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f && Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftShoulderIndex].y) < 0.1f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); break; } break; // check for Stop case Gestures.Stop: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.2f && (jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } else if (jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.2f && (jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); } break; case 1: // gesture complete bool isInPose = (gestureData.joint == rightHandIndex) ? (jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) < 0.2f && (jointsPos[rightHandIndex].x - jointsPos[rightHipIndex].x) >= 0.4f) : (jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) < 0.2f && (jointsPos[leftHandIndex].x - jointsPos[leftHipIndex].x) <= -0.4f); Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); break; } break; // check for Wave case Gestures.Wave: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; } else if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture - phase 2 if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) < -0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) > 0.05f; if (isInPose) { gestureData.timestamp = timestamp; gestureData.state++; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; case 2: // gesture phase 3 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > 0.1f && (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0.05f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > 0.1f && (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < -0.05f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // Check for custom click case Gestures.Click: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } else if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.3f; // set screen position at the start, because this is the most accurate click position SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); } break; case 1: // gesture - phase 2 { // check for stay-in-place Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; bool isInPose = distVector.magnitude < 0.05f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.ClickStayDuration); // SetGestureCancelled(gestureData); MGC.Instance.mouseCursor.GetComponent<Game.CursorReference>().cursorReference.cursorCircleRight.progress = gestureData.progress + 0.1f; if(gestureData.progress >= 1) { MGC.Instance.mouseCursor.GetComponent<Game.CursorReference>().cursorReference.cursorCircleRight.progress = 0; //MouseControl.MouseClick(); } } break; } break; // // check for Click // case Gestures.Click: // switch(gestureData.state) // { // case 0: // gesture detection - phase 1 // if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f) // { // SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); // gestureData.progress = 0.3f; // // // set screen position at the start, because this is the most accurate click position // SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); // } // else if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f) // { // SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); // gestureData.progress = 0.3f; // // // set screen position at the start, because this is the most accurate click position // SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); // } // break; // // case 1: // gesture - phase 2 //// if((timestamp - gestureData.timestamp) < 1.0f) //// { //// bool isInPose = gestureData.joint == rightHandIndex ? //// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && //// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && //// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && //// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.05f : //// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && //// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && //// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && //// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.05f; //// //// if(isInPose) //// { //// gestureData.timestamp = timestamp; //// gestureData.jointPos = jointsPos[gestureData.joint]; //// gestureData.state++; //// gestureData.progress = 0.7f; //// } //// else //// { //// // check for stay-in-place //// Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; //// isInPose = distVector.magnitude < 0.05f; //// //// Vector3 jointPos = jointsPos[gestureData.joint]; //// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, Constants.ClickStayDuration); //// } //// } //// else // { // // check for stay-in-place // Vector3 distVector = jointsPos[gestureData.joint] - gestureData.jointPos; // bool isInPose = distVector.magnitude < 0.05f; // // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.ClickStayDuration); //// SetGestureCancelled(gestureData); // } // break; // //// case 2: // gesture phase 3 = complete //// if((timestamp - gestureData.timestamp) < 1.0f) //// { //// bool isInPose = gestureData.joint == rightHandIndex ? //// jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && //// //(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && //// Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.08f && //// (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.05f : //// jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && //// //(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && //// Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.08f && //// (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.05f; //// //// if(isInPose) //// { //// Vector3 jointPos = jointsPos[gestureData.joint]; //// CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); //// } //// } //// else //// { //// // cancel the gesture //// SetGestureCancelled(ref gestureData); //// } //// break; // } // break; // DEFAULT SWIPE LEFT // check for SwipeLeft /*case Gestures.SwipeLeft: switch (gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f && // (jointsPos[rightHandIndex].x - jointsPos[rightElbowIndex].x) > 0f) // { // SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); // gestureData.progress = 0.5f; // } if (jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && jointsPos[rightHandIndex].x <= gestureRight && jointsPos[rightHandIndex].x > gestureLeft) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.1f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { // bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && // Mathf.Abs(jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) < 0.1f && // Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && // (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.15f; // // if(isInPose) // { // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); // } bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && jointsPos[rightHandIndex].x <= gestureLeft; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else if (jointsPos[rightHandIndex].x <= gestureRight) { float gestureSize = gestureRight - gestureLeft; gestureData.progress = gestureSize > 0.01f ? (gestureRight - jointsPos[rightHandIndex].x) / gestureSize : 0f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break;*/ //EDITED SwipeLeft // check for SwipeLeft case Gestures.SwipeLeft: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[rightElbowIndex].y) > -0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.25f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.25f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); if (gestureData.complete) { Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_J); } } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // DEFAULT SWIPE RIGHT // check for SwipeRight /*case Gestures.SwipeRight: switch (gestureData.state) { case 0: // gesture detection - phase 1 // if(jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f && // (jointsPos[leftHandIndex].x - jointsPos[leftElbowIndex].x) < 0f) // { // SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); // gestureData.progress = 0.5f; // } if (jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[leftHandIndex].x >= gestureLeft && jointsPos[leftHandIndex].x < gestureRight) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.1f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { // bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && // Mathf.Abs(jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) < 0.1f && // Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && // (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.15f; // // if(isInPose) // { // Vector3 jointPos = jointsPos[gestureData.joint]; // CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); // } bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[leftHandIndex].x >= gestureRight; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } else if (jointsPos[leftHandIndex].x >= gestureLeft) { float gestureSize = gestureRight - gestureLeft; gestureData.progress = gestureSize > 0.01f ? (jointsPos[leftHandIndex].x - gestureLeft) / gestureSize : 0f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break;*/ // EDITED SwipeRight // check for SwipeRight case Gestures.SwipeRight: switch (gestureData.state) { case 0: if (jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[leftElbowIndex].y) > -0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightElbowIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.25f : jointsTracked[leftHandIndex] && jointsTracked[leftElbowIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.08f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.25f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); if (gestureData.complete) { Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_L); } } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; case Gestures.SwipeLeftLHand: switch (gestureData.state) { case 0: if (jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex]) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { bool isInPose = gestureData.joint == leftHandIndex ? jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.1f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) < -0.25f : jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.1f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) < -0.25f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); if (gestureData.complete) { Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_J); } } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; case Gestures.SwipeRightRHand: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex]) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[rightShoulderIndex] && Mathf.Abs(jointsPos[rightHandIndex].y - gestureData.jointPos.y) < 0.1f && (jointsPos[rightHandIndex].x - gestureData.jointPos.x) > 0.25f : jointsTracked[leftHandIndex] && jointsTracked[leftShoulderIndex] && Mathf.Abs(jointsPos[leftHandIndex].y - gestureData.jointPos.y) < 0.1f && (jointsPos[leftHandIndex].x - gestureData.jointPos.x) > 0.25f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); if (gestureData.complete) { Kinect.Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.KEY_L); } } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SwipeUp case Gestures.SwipeUp: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.0f && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.0f && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) > 0.05f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) > 0.05f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for SwipeDown case Gestures.SwipeDown: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftShoulderIndex].y) >= 0.05f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if (jointsTracked[leftHandIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightShoulderIndex].y) >= 0.05f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) < -0.15f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) <= 0.1f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) < -0.15f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) <= 0.1f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // // check for RightHandCursor // case Gestures.RightHandCursor: // switch(gestureData.state) // { // case 0: // gesture detection - phase 1 (perpetual) // if(jointsTracked[rightHandIndex] && jointsTracked[rightHipIndex] && // //(jointsPos[rightHandIndex].y - jointsPos[rightHipIndex].y) > -0.1f) // (jointsPos[rightHandIndex].y - jointsPos[hipCenterIndex].y) >= 0f) // { // gestureData.joint = rightHandIndex; // gestureData.timestamp = timestamp; // gestureData.jointPos = jointsPos[rightHandIndex]; // // SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); // gestureData.progress = 0.7f; // } // else // { // // cancel the gesture // //SetGestureCancelled(ref gestureData); // gestureData.progress = 0f; // } // break; // // } // break; // // // check for LeftHandCursor // case Gestures.LeftHandCursor: // switch(gestureData.state) // { // case 0: // gesture detection - phase 1 (perpetual) // if(jointsTracked[leftHandIndex] && jointsTracked[leftHipIndex] && // //(jointsPos[leftHandIndex].y - jointsPos[leftHipIndex].y) > -0.1f) // (jointsPos[leftHandIndex].y - jointsPos[hipCenterIndex].y) >= 0f) // { // gestureData.joint = leftHandIndex; // gestureData.timestamp = timestamp; // gestureData.jointPos = jointsPos[leftHandIndex]; // // SetScreenPos(userId, ref gestureData, ref jointsPos, ref jointsTracked); // gestureData.progress = 0.7f; // } // else // { // // cancel the gesture // //SetGestureCancelled(ref gestureData); // gestureData.progress = 0f; // } // break; // // } // break; // check for ZoomIn case Gestures.ZoomIn: Vector3 vectorZoomOut = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distZoomOut = vectorZoomOut.magnitude; switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomOut < 0.3f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = 0f; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if ((timestamp - gestureData.timestamp) < 1.0f) { float angleZoomOut = Vector3.Angle(gestureData.tagVector, vectorZoomOut) * Mathf.Sign(vectorZoomOut.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomOut < 1.5f && Mathf.Abs(angleZoomOut) < 20f; if (isInPose) { SetZoomFactor(userId, ref gestureData, 1.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } // else // { // // cancel the gesture // SetGestureCancelled(ref gestureData); // } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ZoomOut case Gestures.ZoomOut: Vector3 vectorZoomIn = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distZoomIn = vectorZoomIn.magnitude; switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomIn >= 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = distZoomIn; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if ((timestamp - gestureData.timestamp) < 1.0f) { float angleZoomIn = Vector3.Angle(gestureData.tagVector, vectorZoomIn) * Mathf.Sign(vectorZoomIn.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distZoomIn >= 0.2f && Mathf.Abs(angleZoomIn) < 20f; if (isInPose) { SetZoomFactor(userId, ref gestureData, 0.0f, ref jointsPos, ref jointsTracked); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } // else // { // // cancel the gesture // SetGestureCancelled(ref gestureData); // } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Wheel case Gestures.Wheel: Vector3 vectorWheel = (Vector3)jointsPos[rightHandIndex] - jointsPos[leftHandIndex]; float distWheel = vectorWheel.magnitude; // Debug.Log(string.Format("{0}. Dist: {1:F1}, Tag: {2:F1}, Diff: {3:F1}", gestureData.state, // distWheel, gestureData.tagFloat, Mathf.Abs(distWheel - gestureData.tagFloat))); switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distWheel >= 0.3f && distWheel < 0.7f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.tagVector = Vector3.right; gestureData.tagFloat = distWheel; gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = zooming if ((timestamp - gestureData.timestamp) < 0.5f) { float angle = Vector3.Angle(gestureData.tagVector, vectorWheel) * Mathf.Sign(vectorWheel.y - gestureData.tagVector.y); bool isInPose = jointsTracked[leftHandIndex] && jointsTracked[rightHandIndex] && jointsTracked[hipCenterIndex] && jointsTracked[shoulderCenterIndex] && jointsTracked[leftHipIndex] && jointsTracked[rightHipIndex] && jointsPos[leftHandIndex].y >= gestureBottom && jointsPos[leftHandIndex].y <= gestureTop && jointsPos[rightHandIndex].y >= gestureBottom && jointsPos[rightHandIndex].y <= gestureTop && distWheel >= 0.3f && distWheel < 0.7f && Mathf.Abs(distWheel - gestureData.tagFloat) < 0.1f; if (isInPose) { //SetWheelRotation(userId, ref gestureData, gestureData.tagVector, vectorWheel); gestureData.screenPos.z = angle; // wheel angle gestureData.timestamp = timestamp; gestureData.tagFloat = distWheel; gestureData.progress = 0.7f; } // else // { // // cancel the gesture // SetGestureCancelled(ref gestureData); // } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Jump case Gestures.Jump: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y > 0.6f) && (jointsPos[hipCenterIndex].y < 1.2f)) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y - gestureData.jointPos.y) > 0.15f && Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Squat case Gestures.Squat: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y <= 0.7f)) { SetGestureJoint(ref gestureData, timestamp, hipCenterIndex, jointsPos[hipCenterIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[hipCenterIndex] && (jointsPos[hipCenterIndex].y - gestureData.jointPos.y) < -0.15f && Mathf.Abs(jointsPos[hipCenterIndex].x - gestureData.jointPos.x) < 0.2f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Push case Gestures.Push: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f && (jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.2f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f && (jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.2f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) < -0.2f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) < -0.2f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for Pull case Gestures.Pull: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - jointsPos[rightShoulderIndex].x) < 0.2f && (jointsPos[rightHandIndex].z - jointsPos[leftElbowIndex].z) < -0.3f) { SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); gestureData.progress = 0.5f; } else if (jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - jointsPos[leftShoulderIndex].x) < 0.2f && (jointsPos[leftHandIndex].z - jointsPos[rightElbowIndex].z) < -0.3f) { SetGestureJoint(ref gestureData, timestamp, leftHandIndex, jointsPos[leftHandIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = gestureData.joint == rightHandIndex ? jointsTracked[rightHandIndex] && jointsTracked[leftElbowIndex] && jointsTracked[rightShoulderIndex] && (jointsPos[rightHandIndex].y - jointsPos[leftElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[rightHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[rightHandIndex].z - gestureData.jointPos.z) > 0.25f : jointsTracked[leftHandIndex] && jointsTracked[rightElbowIndex] && jointsTracked[leftShoulderIndex] && (jointsPos[leftHandIndex].y - jointsPos[rightElbowIndex].y) > -0.1f && Mathf.Abs(jointsPos[leftHandIndex].x - gestureData.jointPos.x) < 0.2f && (jointsPos[leftHandIndex].z - gestureData.jointPos.z) > 0.25f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ShoulderLeftFron case Gestures.ShoulderLeftFront: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[leftHipIndex] && (jointsPos[rightShoulderIndex].z - jointsPos[leftHipIndex].z) < 0f && (jointsPos[rightShoulderIndex].z - jointsPos[leftShoulderIndex].z) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, rightShoulderIndex, jointsPos[rightShoulderIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[leftHipIndex] && (jointsPos[rightShoulderIndex].z - jointsPos[leftShoulderIndex].z) < -0.2f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for ShoulderRightFront case Gestures.ShoulderRightFront: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] && (jointsPos[leftShoulderIndex].z - jointsPos[rightHipIndex].z) < 0f && (jointsPos[leftShoulderIndex].z - jointsPos[rightShoulderIndex].z) > -0.15f) { SetGestureJoint(ref gestureData, timestamp, leftShoulderIndex, jointsPos[leftShoulderIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] && (jointsPos[leftShoulderIndex].z - jointsPos[rightShoulderIndex].z) < -0.2f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for LeanLeft case Gestures.LeanLeft: switch (gestureData.state) { case 0: // gesture detection - phase 1 (right shoulder is left of the right hip, means leaning left) if (jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightShoulderIndex].x - jointsPos[rightHipIndex].x) < 0f) { SetGestureJoint(ref gestureData, timestamp, rightShoulderIndex, jointsPos[rightShoulderIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { // check if right shoulder is still left of the right hip (leaning left) bool isInPose = jointsTracked[rightShoulderIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightShoulderIndex].x - jointsPos[rightHipIndex].x) < 0f; if (isInPose) { // calculate lean angle Vector3 vSpineLL = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex]; gestureData.screenPos.z = Vector3.Angle(Vector3.up, vSpineLL); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for LeanRight case Gestures.LeanRight: switch (gestureData.state) { case 0: // gesture detection - phase 1 (left shoulder is right of the left hip, means leaning right) if (jointsTracked[leftShoulderIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftShoulderIndex].x - jointsPos[leftHipIndex].x) > 0f) { SetGestureJoint(ref gestureData, timestamp, leftShoulderIndex, jointsPos[leftShoulderIndex]); gestureData.progress = 0.3f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 0.5f) { // check if left shoulder is still right of the left hip (leaning right) bool isInPose = jointsTracked[leftShoulderIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftShoulderIndex].x - jointsPos[leftHipIndex].x) > 0f; if (isInPose) { // calculate lean angle Vector3 vSpineLL = jointsPos[shoulderCenterIndex] - jointsPos[hipCenterIndex]; gestureData.screenPos.z = Vector3.Angle(Vector3.up, vSpineLL); gestureData.timestamp = timestamp; gestureData.progress = 0.7f; } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for KickLeft case Gestures.KickLeft: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftAnkleIndex].z - jointsPos[leftHipIndex].z) < 0f && (jointsPos[leftAnkleIndex].z - jointsPos[rightAnkleIndex].z) > -0.2f) { SetGestureJoint(ref gestureData, timestamp, leftAnkleIndex, jointsPos[leftAnkleIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[leftHipIndex] && (jointsPos[leftAnkleIndex].z - jointsPos[rightAnkleIndex].z) < -0.4f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for KickRight case Gestures.KickRight: switch (gestureData.state) { case 0: // gesture detection - phase 1 if (jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightAnkleIndex].z - jointsPos[rightHipIndex].z) < 0f && (jointsPos[rightAnkleIndex].z - jointsPos[leftAnkleIndex].z) > -0.2f) { SetGestureJoint(ref gestureData, timestamp, rightAnkleIndex, jointsPos[rightAnkleIndex]); gestureData.progress = 0.5f; } break; case 1: // gesture phase 2 = complete if ((timestamp - gestureData.timestamp) < 1.5f) { bool isInPose = jointsTracked[leftAnkleIndex] && jointsTracked[rightAnkleIndex] && jointsTracked[rightHipIndex] && (jointsPos[rightAnkleIndex].z - jointsPos[leftAnkleIndex].z) < -0.4f; if (isInPose) { Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, 0f); } } else { // cancel the gesture SetGestureCancelled(ref gestureData); } break; } break; // check for HiddenGesture case Gestures.HiddenGesture: switch (gestureData.state) { case 0: // gesture detection if (jointsTracked[rightHandIndex] && jointsTracked[hipsIndex] && (jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f && (jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f) { Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.NUMPAD7); //Debug.LogError("Pressed 7? Why?"); SetGestureJoint(ref gestureData, timestamp, rightHandIndex, jointsPos[rightHandIndex]); } break; case 1: bool gestureDetected = (jointsTracked[rightHandIndex] && jointsTracked[hipsIndex] && (jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f && (jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f); if (!gestureDetected) { Win32.MouseKeySimulator.SendKeyPress(Kinect.Win32.KeyCode.NUMPAD9); SetGestureCancelled(ref gestureData); } if (timestamp - gestureData.timestamp <= 3) { gestureData.progress = (timestamp - gestureData.timestamp) / 3; MGC.Instance.minigamesGUI.guiDetection.ShowDetection(gestureData.progress); } if (timestamp - gestureData.timestamp > 3) gestureData.state++; break; case 2: // gesture complete if ((timestamp - gestureData.timestamp) > 3) { bool isInPose = jointsTracked[rightHandIndex] && jointsTracked[neck] && (jointsPos[hipsIndex].y - jointsPos[rightHandIndex].y) > 0f && (jointsPos[rightHandIndex].x - jointsPos[hipsIndex].x) > 0.5f; Vector3 jointPos = jointsPos[gestureData.joint]; CheckPoseComplete(ref gestureData, timestamp, jointPos, isInPose, KinectInterop.Constants.PoseCompleteDuration); //if (isInPose) { //Debug.LogError("Hidden gesture complete"); Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.KEY_I); //Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.NUMPAD9); #if UNITY_ANDROID Win32.MouseKeySimulator.SendKeyPress(Win32.KeyCode.ESC); #endif SetGestureCancelled(ref gestureData); } } break; } break; // here come more gesture-cases } }
internal static Boolean IsGestureFiredByHandINT(IntPtr instance, String gestureName, Int32 handID, out GestureData gestureData) { gestureData = new GestureData(); return PXCMHandData_IsGestureFiredByHand(instance, gestureName, handID, gestureData); }
/// <summary> /// Detects the movement and builds the frame list when movement is detected. /// </summary> /// <param name='frame'> /// Leap frame. /// </param> public void DetectMovement(Frame frame) { GestureFrame gestureFrame = new GestureFrame(frame.Pointables[0].TipPosition, frame.Pointables[0].TipVelocity, frame.Timestamp); this.detectionHeap.Add(gestureFrame); Vector3 acceleration = Vector3.zero; Vector3 delta = Vector3.zero; Vector3 movementWork = Vector3.zero; Vector3 kinecticEnergy = Vector3.zero; float movementTotalWork; float kinecticEnergyTotal; if (this.detectionHeap.Count > this.maximumHeapSize) { this.detectionHeap.RemoveAt(0); GestureFrame firstFrame = (GestureFrame)this.detectionHeap[0]; GestureFrame lastFrame = (GestureFrame)this.detectionHeap[this.maximumHeapSize - 1]; float deltaTime = lastFrame.timestamp - firstFrame.timestamp; acceleration.x = (lastFrame.velocity.x - firstFrame.velocity.x) / deltaTime; acceleration.y = (lastFrame.velocity.y - firstFrame.velocity.y) / deltaTime; acceleration.z = (lastFrame.velocity.z - firstFrame.velocity.z) / deltaTime; this.movementDirection.x = acceleration.x > 0 ? 1 : -1; this.movementDirection.y = acceleration.y > 0 ? 1 : -1; this.movementDirection.z = acceleration.z > 0 ? 1 : -1; delta.x = (lastFrame.position.x - firstFrame.position.x); delta.y = (lastFrame.position.y - firstFrame.position.y); delta.z = (lastFrame.position.z - firstFrame.position.z); /* * http://en.wikipedia.org/wiki/Work_(physics) * Work calculation, using the acceleration previously calculated as the force to move the finger from one point to another */ movementWork.x = delta.x * acceleration.x; movementWork.y = delta.y * acceleration.y; movementWork.z = delta.z * acceleration.z; /* * http://en.wikipedia.org/wiki/Kinetic_energy * Measures the pointable kinectic energy to detect when gesture have ended. */ kinecticEnergy.x = Mathf.Pow(lastFrame.velocity.x, 2) * 0.5f * 1.0e-6f; kinecticEnergy.y = Mathf.Pow(lastFrame.velocity.y, 2) * 0.5f * 1.0e-6f; kinecticEnergy.z = Mathf.Pow(lastFrame.velocity.z, 2) * 0.5f * 1.0e-6f; kinecticEnergyTotal = kinecticEnergy.x + kinecticEnergy.y; if (Mathf.Abs(movementWork.x) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.x += movementWork.x; } if (Mathf.Abs(movementWork.y) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.y += movementWork.y; } if (Mathf.Abs(movementWork.z) > this.minimumRelevantWorkThreshold) { this.movementTotalWork.z += movementWork.z; } if (!this.performingMovement && lastFrame.timestamp - this.lastStoppedTime > 50000.0f) { this.movementTotalWork.x = 0.0f; this.movementTotalWork.y = 0.0f; this.movementTotalWork.z = 0.0f; this.lastStoppedTime = lastFrame.timestamp; } movementTotalWork = this.movementTotalWork.x + this.movementTotalWork.y; if (movementTotalWork > this.minimumStartMovementTotalWork && !this.performingMovement) { this.movementTotalWork.x = 0; this.movementTotalWork.y = 0; this.movementTotalWork.z = 0; this.performingMovement = true; if (!this.performingGesture) { this.performingGesture = true; this.gestureData = new GestureData(); if (this.callbackInstance != null) { this.callbackInstance.beginOfGestureCallback(); } } } else if (this.performingMovement && kinecticEnergyTotal < this.minimumEndMovementKinectEnergy) { this.movementTotalWork.x = 0.0f; this.movementTotalWork.y = 0.0f; this.movementTotalWork.z = 0.0f; this.performingMovement = false; this.lastStoppedTime = this.lastEndMovementTime = lastFrame.timestamp; } if (this.performingGesture) { this.gestureData.appendFrame(gestureFrame); } float timeSinceEndMovement = lastFrame.timestamp - this.lastEndMovementTime; if (!this.performingMovement && this.performingGesture && timeSinceEndMovement > 500000) { if (this.callbackInstance != null) { this.callbackInstance.endOfGestureCallback(this.gestureData); } this.performingGesture = false; } } }