public void autoCalibrateMapping(bool leftHand) { uint id = Fubi.getClosestUserID(); if (id > 0) { FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW; FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER; FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND; if (leftHand) { elbow = FubiUtils.SkeletonJoint.LEFT_ELBOW; shoulder = FubiUtils.SkeletonJoint.LEFT_SHOULDER; hand = FubiUtils.SkeletonJoint.LEFT_HAND; } float confidence; double timeStamp; float elbowX, elbowY, elbowZ; Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp); if (confidence > 0.5f) { float shoulderX, shoulderY, shoulderZ; Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp); if (confidence > 0.5f) { double dist1 = Math.Sqrt(Math.Pow(elbowX - shoulderX, 2) + Math.Pow(elbowY - shoulderY, 2) + Math.Pow(elbowZ - shoulderZ, 2)); float handX, handY, handZ; Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp); if (confidence > 0.5f) { double dist2 = Math.Sqrt(Math.Pow(elbowX - handX, 2) + Math.Pow(elbowY - handY, 2) + Math.Pow(elbowZ - handZ, 2)); MapH = (float)(dist1 + dist2); // Calculate all others in depence of maph MapY = 250.0f / 550.0f * MapH; MapW = MapH / m_aspect; MapX = -100.0f / (550.0f / m_aspect) * MapW; } } } } }
protected Vector3D getCurrentJointValue(uint targetID, FubiUtils.SkeletonJoint jointID, FubiUtils.BodyMeasurement measureID = FubiUtils.BodyMeasurement.NUM_MEASUREMENTS) { float x, y, z, confidence; double timeStamp; if (Type == XMLGenerator.RecognizerType.JointRelation || Type == XMLGenerator.RecognizerType.LinearMovement) { if (UseHand) { Fubi.getCurrentHandJointPosition(targetID, (FubiUtils.SkeletonHandJoint)jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered); } else { Fubi.getCurrentSkeletonJointPosition(targetID, jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered); if (measureID != FubiUtils.BodyMeasurement.NUM_MEASUREMENTS && Type == XMLGenerator.RecognizerType.JointRelation) { float measureDist, measureConfidence; Fubi.getBodyMeasurementDistance(targetID, measureID, out measureDist, out measureConfidence); if (measureDist > 0) { x /= measureDist; y /= measureDist; z /= measureDist; } } } } else { var mat = new float[9]; if (UseHand) { Fubi.getCurrentHandJointOrientation(targetID, (FubiUtils.SkeletonHandJoint)jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered); } else { Fubi.getCurrentSkeletonJointOrientation(targetID, jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered); } FubiUtils.Math.rotMatToRotation(mat, out x, out y, out z); } return(new Vector3D(x, y, z)); }
// private string DisplayFubiSwipeMenu(Vector2 center, float radius, string[] options, GUIStyle optionStyle, GUIStyle centerStyle) // { // if (!(radius > 0 && options.Length > 0 && options.Length <= m_swipeRecognizers.Length)) // { // Debug.LogWarning("FubiSwipeMenu called with incorrect parameters!"); // Debug.DebugBreak(); // return ""; // } // // GUI.depth = -2; // string selection = ""; // m_swipeMenuDisplayed = true; // // ScaleMode smode = /*(options.Length >= 2) ? ScaleMode.StretchToFill :*/ ScaleMode.ScaleToFit; // // // Display center and explanation text // float centerRad = radius * 0.25f; // Rect centerRect = new Rect(center.x - centerRad, center.y - centerRad, 2.0f * centerRad, 2.0f * centerRad); // // // Check for right hand in front for general activation // if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_handToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED) // m_lastHandToFront = Fubi.getCurrentTime(); // else if (Fubi.getCurrentTime() - m_lastHandToFront > 1.2f) // m_handToFrontEnd = Fubi.getCurrentTime(); // // Check for left hand in front for general activation // if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_leftHandToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED) // m_lastLeftHandToFront = Fubi.getCurrentTime(); // else if (Fubi.getCurrentTime() - m_lastLeftHandToFront > 1.2f) // m_leftHandToFrontEnd = Fubi.getCurrentTime(); // // bool withinInteractionFrame = m_currentUser > 0 && (m_lastHandToFront - m_handToFrontEnd > 0.5 || m_lastLeftHandToFront - m_leftHandToFrontEnd > 0.5); // if (withinInteractionFrame) // { // if (!m_swipeMenuActive) // { // m_swipeMenuActive = true; // Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false); // } // GUI.DrawTexture(centerRect, m_swipeCenterActive, smode); // GUI.Label(centerRect, "Swipe for \nselection", centerStyle.name+"-hover"); // } // else // { // m_swipeMenuActive = false; // GUI.DrawTexture(centerRect, m_swipeCenterNormal, smode); // if (m_lastHandToFront > m_handToFrontEnd || m_lastLeftHandToFront > m_leftHandToFrontEnd) // { // GUI.Label(centerRect, "Hold arm", centerStyle); // } // else // GUI.Label(centerRect, "Stretch arm \nto front", centerStyle); // } // // // Display the options and check their recognizers // float rotAdd = 360.0f / options.Length; // float currRot = (options.Length > 2) ? 0 : 90.0f; // Rect buttonRect = new Rect(); // buttonRect.height = radius * 0.35f; // buttonRect.width = (options.Length > 2) ? (Mathf.PI * (radius / 2.0f) / options.Length) : buttonRect.height; // buttonRect.y = center.y - (0.65f* radius); // buttonRect.x = center.x - (0.5f * buttonRect.width); // for (uint i = 0; i < options.Length; ++i) // { // Rect textRect = new Rect(buttonRect); // textRect.y = center.y - radius; // textRect.height *= 0.5f; // string text = options[i]; // GUI.matrix = Matrix4x4.identity; // bool selected = false; // int recognizerGroup = options.Length - 1; // uint recognizerIndex = i; // // // if (Fubi.getCurrentTime() - m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5 // || Fubi.getCurrentTime() - m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5) // last recognition not longer than 0.5 seconds ago // { // GUIUtility.RotateAroundPivot(currRot, center); // GUI.DrawTexture(buttonRect, m_swipeButtonActive, smode); // Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y)); // textRect.x = newPos.x - textRect.width / 2.0f; // textRect.y = newPos.y - textRect.height / 2.0f; // GUI.matrix = Matrix4x4.identity; // selected = GUI.Button(textRect, text, optionStyle.name+"-hover"); // } // else // Display button also usable for mouse interaction // { // GUIUtility.RotateAroundPivot(currRot, center); // GUI.DrawTexture(buttonRect, m_swipeButtonNormal, smode); // Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y)); // textRect.x = newPos.x - textRect.width / 2.0f; // textRect.y = newPos.y - textRect.height / 2.0f; // GUI.matrix = Matrix4x4.identity; // selected = GUI.Button(textRect, text, optionStyle); // } // // // // Check for full swipe // // Of right hand // if (!selected && m_currentUser > 0 // && withinInteractionFrame // && Fubi.getCurrentTime() - m_lastSwipeRecognition > 1.0f) // at least one second between to swipes // { // selected = Fubi.getCombinationRecognitionProgressOn(m_swipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED; // if (selected) // { // m_swipeMenuActive = false; // Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false); // m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastSwipeRecognition = Fubi.getCurrentTime(); // } // else // Fubi.enableCombinationRecognition(m_swipeRecognizers[options.Length - 1][i], m_currentUser, true); // } // // // Or left hand // if (!selected && m_currentUser > 0 // && withinInteractionFrame // && Fubi.getCurrentTime() - m_lastLeftSwipeRecognition > 1.0f) // at least one second between to swipes // { // selected = Fubi.getCombinationRecognitionProgressOn(m_leftSwipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED; // if (selected) // { // m_swipeMenuActive = false; // Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false); // m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastLeftSwipeRecognition = Fubi.getCurrentTime(); // } // else // Fubi.enableCombinationRecognition(m_leftSwipeRecognizers[options.Length - 1][i], m_currentUser, true); // } // // // if (selected) // { // selection = text; // if (m_swipeSound && m_swipeSound.isReadyToPlay) // AudioSource.PlayClipAtPoint(m_swipeSound, new Vector3(0,0,0), 1.0f); // break; // } // currRot += rotAdd; // } // return selection; // } // bool calibrateCursorMapping(uint id) { m_aspect = (float)Screen.width / (float)Screen.height; if (id > 0) { FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW; FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER; FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND; float confidence; double timeStamp; float elbowX, elbowY, elbowZ; Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp); if (confidence > 0.5f) { float shoulderX, shoulderY, shoulderZ; Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp); if (confidence > 0.5f) { double dist1 = Mathf.Sqrt(Mathf.Pow(elbowX - shoulderX, 2) + Mathf.Pow(elbowY - shoulderY, 2) + Mathf.Pow(elbowZ - shoulderZ, 2)); float handX, handY, handZ; Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp); if (confidence > 0.5f) { double dist2 = Mathf.Sqrt(Mathf.Pow(elbowX - handX, 2) + Mathf.Pow(elbowY - handY, 2) + Mathf.Pow(elbowZ - handZ, 2)); m_mapping.height = (float)(dist1 + dist2); // Calculate all others in depence of maph m_mapping.y = 200.0f / 550.0f * m_mapping.height; m_mapping.width = m_mapping.height / m_aspect; m_mapping.x = -100.0f / (550.0f / m_aspect) * m_mapping.width; //Debug.Log ("m_mapping.x=" + m_mapping.x + "m_mapping.y=" + m_mapping.y + "m_mapping.height=" + m_mapping.height + "m_mapping.width=" + m_mapping.width); return(true); } } } } return(false); }
// Called for rendering the gui void OnGUI() { // AA: Position the depth image so the user can see the kinect output if (!m_disableTrackingImage && (!m_disableTrackingImageWithSwipeMenu || !m_swipeMenuDisplayedLastFrame)) { // Debug image GUI.depth = -4; GUI.DrawTexture(new Rect(25, Screen.height - m_yRes / m_factor - 25, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture); //GUI.DrawTexture(new Rect(Screen.width-m_xRes/m_factor, Screen.height-m_yRes/m_factor, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture); } //AA: add the GUI elements int shift = 42; GUI.Box(new Rect(5 + shift, 25, Screen.width / 3 - 130, Screen.height - 50), "FILTERS"); m_bUseSimpleAverage = GUI.Toggle(new Rect(45 + shift, 50, 200, 30), m_bUseSimpleAverage, " SIMPLE AVERAGE 10"); m_bUseMovingAverage = GUI.Toggle(new Rect(45 + shift, 90, 200, 30), m_bUseMovingAverage, " MOVING AVERAGE"); m_bUseSimpleAverage5 = GUI.Toggle(new Rect(45 + shift, 130, 200, 30), m_bUseSimpleAverage5, " SIMPLE AVERAGE 5"); m_bDblMovingAverage = GUI.Toggle(new Rect(45 + shift, 170, 200, 30), m_bDblMovingAverage, " DOUBLE MOV AVERAGE"); m_bUseExpSmoothing = GUI.Toggle(new Rect(45 + shift, 210, 200, 30), m_bUseExpSmoothing, " EXP SMOOTHING"); m_bUseDblExpSmoothing = GUI.Toggle(new Rect(45 + shift, 250, 200, 30), m_bUseDblExpSmoothing, " DOUBLE EXP SMOOTHING"); m_bUseAdaptive = GUI.Toggle(new Rect(45 + shift, 290, 200, 30), m_bUseAdaptive, " ADAPTIVE DBL EXP"); m_bUseMedian = GUI.Toggle(new Rect(45 + shift, 330, 200, 30), m_bUseMedian, " MEDIAN"); m_bUseCombination1 = GUI.Toggle(new Rect(45 + shift, 370, 200, 30), m_bUseCombination1, " SIMPLE AVG + Median"); m_bUseCombination2 = GUI.Toggle(new Rect(45 + shift, 410, 200, 30), m_bUseCombination2, " DBL MOV AVG + Median"); m_bUseNone = GUI.Toggle(new Rect(45 + shift, 450, 200, 30), m_bUseNone, " NONE"); if (GUI.Button(new Rect(50 + shift, Screen.height - 75, 150, 30), "Clear")) { WriteableAreaResize(); fv.DrawCircle(); } // If some button has been pressed OR this is the first exection if (GUI.changed) { LoadFilters(); if (fm.filters.Count == 1) { m_principalCursor = 0; } } int count = 0; if (m_bUseSimpleAverage) { GUI.Label(new Rect(15 + shift, 45, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseMovingAverage) { GUI.Label(new Rect(15 + shift, 85, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseSimpleAverage5) { GUI.Label(new Rect(15 + shift, 125, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bDblMovingAverage) { GUI.Label(new Rect(15 + shift, 165, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseExpSmoothing) { GUI.Label(new Rect(15 + shift, 205, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseDblExpSmoothing) { GUI.Label(new Rect(15 + shift, 245, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseAdaptive) { GUI.Label(new Rect(15 + shift, 285, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseMedian) { GUI.Label(new Rect(15 + shift, 325, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseCombination1) { GUI.Label(new Rect(15 + shift, 365, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseCombination2) { GUI.Label(new Rect(15 + shift, 405, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (m_bUseNone) { GUI.Label(new Rect(15 + shift, 445, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]); count++; } if (prevScreenWidth != Screen.width || prevScreenHeight != Screen.height) { // Resize writeable area, redraw the circle WriteableAreaResize(); prevScreenWidth = Screen.width; prevScreenHeight = Screen.height; fv.DrawCircle(); } //AA: Draw the writeable area fv.Apply(); for (int i = 0; i < fv.m_filterOutputTexture.Count; i++) { GUI.DrawTexture(new Rect(fv.filterOutputLocX, fv.filterOutputLocY, fv.filterOutputWidth, fv.filterOutputHeight), fv.m_filterOutputTexture[i]); } // Cursor m_gotNewFubiCoordinates = false; if (Fubi.isInitialized()) { // Take closest user uint userID = Fubi.getClosestUserID(); if (userID != m_currentUser) { m_currentUser = userID; m_lastCalibrationSucceded = false; } if (userID > 0) { if (!m_lastCalibrationSucceded) { m_lastCalibrationSucceded = calibrateCursorMapping(m_currentUser); } FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND; FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER; // Get hand and shoulder position and check their confidence double timeStamp; float handX, handY, handZ, confidence; Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp); if (confidence > 0.5f) { float relX, relY, relZ; Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp); if (confidence > 0.5f) { // AA: Filtering should happen here for the hand and relative joints separately // If true, use the smoothed joints for calculating screen coordinates fm.UpdateJointFilters(new Vector3(handX, handY, handZ), new Vector3(relX, relY, relZ)); for (int i = 0; i < fm.filters.Count; i++) { if (m_bUseJointFiltering) { //Debug.Log ("Prehand " + new Vector3(handX, handY, handZ) + " relJoint " + new Vector3(relX, relY, relZ)); Vector3 handPos = fm.joints[i]; // filter.Update(new Vector3(handX, handY, handZ), Filter.JOINT_TYPE.JOINT); Vector3 relJointPos = fm.relativeJoints[i]; //filter.Update(new Vector3(relX, relY, relZ), Filter.JOINT_TYPE.RELATIVEJOINT); //Debug.Log ("hand " + handPos + " relJoint " + relJointPos); handZ = handPos.z; handY = handPos.y; handX = handPos.x; relZ = relJointPos.z; relY = relJointPos.y; relX = relJointPos.x; m_relativeCursorPosition = fm.relativeCursorPosition[i]; } // AA: End // Take relative coordinates float zDiff = handZ - relZ; float yDiff = handY - relY; float xDiff = handX - relX; // Check if hand is enough in front of shoulder if ((yDiff > 0 && zDiff < -150.0f) || (Mathf.Abs(xDiff) > 150.0f && zDiff < -175.0f) || zDiff < -225.0f) { // Now get the possible cursor position // Convert to screen coordinates float newX, newY; float mapX = m_mapping.x; newX = (xDiff - mapX) / m_mapping.width; newY = (m_mapping.y - yDiff) / m_mapping.height; // Flip y for the screen coordinates // Filtering // New coordinate is weighted more if it represents a longer distance change // This should reduce the lagging of the cursor on higher distances, but still filter out small jittering float changeX = newX - m_relativeCursorPosition.x; float changeY = newY - m_relativeCursorPosition.y; if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp) { float changeLength = Mathf.Sqrt(changeX * changeX + changeY * changeY); float filterFactor = changeLength; //Mathf.Sqrt(changeLength); if (filterFactor > 1.0f) { filterFactor = 1.0f; } // Apply the tracking to the current position with the given filter factor // AA: Filtering should happen here for joint-to-relativejoint (VECTOR) filtering // AA: filtering code Vector2 tempNew = new Vector2(newX, newY); fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor); // If true, use the calculated factor for smoothing, else just use the new if (m_bUseVectorFiltering) { m_relativeCursorPosition = fm.vectors[i]; //filter.Update(m_relativeCursorPosition, tempNew, filterFactor); } else // Just give equal weight to both { m_relativeCursorPosition = filter.Update(m_relativeCursorPosition, tempNew, 0.5f); } // AA: Calculate all filters // fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor); m_timeStamp = timeStamp; // Send it, but only if it is more or less within the screen if (m_relativeCursorPosition.x > -0.1f && m_relativeCursorPosition.x < 1.1f && m_relativeCursorPosition.y > -0.1f && m_relativeCursorPosition.y < 1.1f) { MoveMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, i); // Each filter must store it's own value of relative position, absolute and previous absolute positions fm.relativeCursorPosition[i] = m_relativeCursorPosition; fm.absPixelPosition[i] = m_absPixelPosition; fm.prevAbsPixelPosition[i] = m_previousAbsPixelPosition; DrawFilterOutputs(i); m_gotNewFubiCoordinates = true; m_lastCursorChangeDoneByFubi = true; } } } } } } } } // AA: FUBI does not move mouse if the confidence value is too low if (!m_gotNewFubiCoordinates) // AA: this only executes when input is coming from mouse { // Got no mouse coordinates from fubi this frame Vector2 mousePos = Input.mousePosition; // Only move mouse if it wasn't changed by fubi the last time or or it really has changed if (!m_lastCursorChangeDoneByFubi || mousePos != m_lastMousePos) { //AA: Old code for cursor placement m_relativeCursorPosition.x = mousePos.x / (float)Screen.width; m_relativeCursorPosition.y = 1.0f - (mousePos.y / (float)Screen.height); // Get mouse X and Y position as a percentage of screen width and height MoveActualMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, true); m_lastMousePos = mousePos; m_lastCursorChangeDoneByFubi = false; } } }
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false) { x = float.NaN; y = float.NaN; float handX, handY, handZ, confidence; FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND; FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER; if (leftHand) { joint = FubiUtils.SkeletonJoint.LEFT_HAND; relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER; } double timeStamp; Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp); if (confidence > 0.5f) { float relX, relY, relZ; Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp); if (confidence > 0.5f) { // Take relative coordinates float rawX = handX - relX; float rawY = handY - relY; // Convert to screen coordinates float newX, newY; float mapX = m_mapX; if (leftHand) { // Mirror x area for left hand mapX = -m_mapX - m_mapW; } newX = (rawX - mapX) / m_mapW; newY = (m_mapY - rawY) / m_mapH; // Flip y for the screen coordinates // Filtering // New coordinate is weighted more if it represents a longer distance change // This should reduce the lagging of the cursor on higher distances, but still filter out small jittering float changeX = newX - m_x; float changeY = newY - m_y; if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp) { double changeLength = Math.Sqrt(changeX * changeX + changeY * changeY); double changeDur = timeStamp - m_timeStamp; float filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor; if (filterFactor > 1.0f) { filterFactor = 1.0f; } // Apply the tracking to the current position with the given filter factor m_x = (1.0f - filterFactor) * m_x + filterFactor * newX; m_y = (1.0f - filterFactor) * m_y + filterFactor * newY; m_timeStamp = timeStamp; // Send it MouseSimulator.sendMousePos(m_x, m_y); x = m_x; y = m_x; } // Check for mouse click if (Fubi.recognizeGestureOn("mouseClick", userID) == FubiUtils.RecognitionResult.RECOGNIZED || Fubi.recognizeGestureOn("mouseClick1", userID) == FubiUtils.RecognitionResult.RECOGNIZED) { if (m_timeStamp - m_lastMouseClick > 1) { MouseSimulator.sendMouseButton(true, true); MouseSimulator.sendMouseButton(true, false); m_lastMouseClick = m_timeStamp; } } } } }
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false) { x = float.NaN; y = float.NaN; float handX, handY, handZ, confidence; var joint = FubiUtils.SkeletonJoint.RIGHT_HAND; var relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER; if (leftHand) { joint = FubiUtils.SkeletonJoint.LEFT_HAND; relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER; } double timeStamp; Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp); if (confidence > 0.5f) { float relX, relY, relZ; Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp); if (confidence > 0.5f) { // Take relative coordinates var rawX = handX - relX; var rawY = handY - relY; // Convert to screen coordinates var mapX = m_mapX; if (leftHand) { // Mirror x area for left hand mapX = -m_mapX - m_mapW; } float newX = (rawX - mapX) / m_mapW; float newY = (m_mapY - rawY) / m_mapH; // Filtering // New coordinate is weighted more if it represents a longer distance change // This should reduce the lagging of the cursor on higher distances, but still filter out small jittering var changeX = newX - m_x; var changeY = newY - m_y; if (Math.Abs(changeX) > float.Epsilon || Math.Abs(changeY) > float.Epsilon && Math.Abs(timeStamp - m_timeStamp) > float.Epsilon) { var changeLength = Math.Sqrt(changeX * changeX + changeY * changeY); var filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor; if (filterFactor > 1.0f) { filterFactor = 1.0f; } // Apply the tracking to the current position with the given filter factor m_x = (1.0f - filterFactor) * m_x + filterFactor * newX; m_y = (1.0f - filterFactor) * m_y + filterFactor * newY; m_timeStamp = timeStamp; // Send it MouseKeyboardSimulator.sendMousePos(m_x, m_y); x = m_x; y = m_x; } } } }