public void autoCalibrateMapping(bool leftHand)
{
uint id = Fubi.getClosestUserID();
if (id > 0)
{
FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW;
FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND;
if (leftHand)
{
elbow = FubiUtils.SkeletonJoint.LEFT_ELBOW;
shoulder = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
hand = FubiUtils.SkeletonJoint.LEFT_HAND;
}
float confidence;
double timeStamp;
float elbowX, elbowY, elbowZ;
Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float shoulderX, shoulderY, shoulderZ;
Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist1 = Math.Sqrt(Math.Pow(elbowX - shoulderX, 2) + Math.Pow(elbowY - shoulderY, 2) + Math.Pow(elbowZ - shoulderZ, 2));
float handX, handY, handZ;
Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist2 = Math.Sqrt(Math.Pow(elbowX - handX, 2) + Math.Pow(elbowY - handY, 2) + Math.Pow(elbowZ - handZ, 2));
MapH = (float)(dist1 + dist2);
// Calculate all others in depence of maph
MapY = 250.0f / 550.0f * MapH;
MapW = MapH / m_aspect;
MapX = -100.0f / (550.0f / m_aspect) * MapW;
}
}
}
}
}
protected Vector3D getCurrentJointValue(uint targetID, FubiUtils.SkeletonJoint jointID, FubiUtils.BodyMeasurement measureID = FubiUtils.BodyMeasurement.NUM_MEASUREMENTS)
{
float x, y, z, confidence;
double timeStamp;
if (Type == XMLGenerator.RecognizerType.JointRelation || Type == XMLGenerator.RecognizerType.LinearMovement)
{
if (UseHand)
{
Fubi.getCurrentHandJointPosition(targetID, (FubiUtils.SkeletonHandJoint)jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
else
{
Fubi.getCurrentSkeletonJointPosition(targetID, jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered);
if (measureID != FubiUtils.BodyMeasurement.NUM_MEASUREMENTS && Type == XMLGenerator.RecognizerType.JointRelation)
{
float measureDist, measureConfidence;
Fubi.getBodyMeasurementDistance(targetID, measureID, out measureDist, out measureConfidence);
if (measureDist > 0)
{
x /= measureDist;
y /= measureDist;
z /= measureDist;
}
}
}
}
else
{
var mat = new float[9];
if (UseHand)
{
Fubi.getCurrentHandJointOrientation(targetID, (FubiUtils.SkeletonHandJoint)jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
else
{
Fubi.getCurrentSkeletonJointOrientation(targetID, jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
FubiUtils.Math.rotMatToRotation(mat, out x, out y, out z);
}
return(new Vector3D(x, y, z));
}
// private string DisplayFubiSwipeMenu(Vector2 center, float radius, string[] options, GUIStyle optionStyle, GUIStyle centerStyle)
// {
// if (!(radius > 0 && options.Length > 0 && options.Length <= m_swipeRecognizers.Length))
// {
// Debug.LogWarning("FubiSwipeMenu called with incorrect parameters!");
// Debug.DebugBreak();
// return "";
// }
//
// GUI.depth = -2;
// string selection = "";
// m_swipeMenuDisplayed = true;
//
// ScaleMode smode = /*(options.Length >= 2) ? ScaleMode.StretchToFill :*/ ScaleMode.ScaleToFit;
//
// // Display center and explanation text
// float centerRad = radius * 0.25f;
// Rect centerRect = new Rect(center.x - centerRad, center.y - centerRad, 2.0f * centerRad, 2.0f * centerRad);
//
// // Check for right hand in front for general activation
// if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_handToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED)
// m_lastHandToFront = Fubi.getCurrentTime();
// else if (Fubi.getCurrentTime() - m_lastHandToFront > 1.2f)
// m_handToFrontEnd = Fubi.getCurrentTime();
// // Check for left hand in front for general activation
// if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_leftHandToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED)
// m_lastLeftHandToFront = Fubi.getCurrentTime();
// else if (Fubi.getCurrentTime() - m_lastLeftHandToFront > 1.2f)
// m_leftHandToFrontEnd = Fubi.getCurrentTime();
//
// bool withinInteractionFrame = m_currentUser > 0 && (m_lastHandToFront - m_handToFrontEnd > 0.5 || m_lastLeftHandToFront - m_leftHandToFrontEnd > 0.5);
// if (withinInteractionFrame)
// {
// if (!m_swipeMenuActive)
// {
// m_swipeMenuActive = true;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// }
// GUI.DrawTexture(centerRect, m_swipeCenterActive, smode);
// GUI.Label(centerRect, "Swipe for \nselection", centerStyle.name+"-hover");
// }
// else
// {
// m_swipeMenuActive = false;
// GUI.DrawTexture(centerRect, m_swipeCenterNormal, smode);
// if (m_lastHandToFront > m_handToFrontEnd || m_lastLeftHandToFront > m_leftHandToFrontEnd)
// {
// GUI.Label(centerRect, "Hold arm", centerStyle);
// }
// else
// GUI.Label(centerRect, "Stretch arm \nto front", centerStyle);
// }
//
// // Display the options and check their recognizers
// float rotAdd = 360.0f / options.Length;
// float currRot = (options.Length > 2) ? 0 : 90.0f;
// Rect buttonRect = new Rect();
// buttonRect.height = radius * 0.35f;
// buttonRect.width = (options.Length > 2) ? (Mathf.PI * (radius / 2.0f) / options.Length) : buttonRect.height;
// buttonRect.y = center.y - (0.65f* radius);
// buttonRect.x = center.x - (0.5f * buttonRect.width);
// for (uint i = 0; i < options.Length; ++i)
// {
// Rect textRect = new Rect(buttonRect);
// textRect.y = center.y - radius;
// textRect.height *= 0.5f;
// string text = options[i];
// GUI.matrix = Matrix4x4.identity;
// bool selected = false;
// int recognizerGroup = options.Length - 1;
// uint recognizerIndex = i;
//
//
// if (Fubi.getCurrentTime() - m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5
// || Fubi.getCurrentTime() - m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5) // last recognition not longer than 0.5 seconds ago
// {
// GUIUtility.RotateAroundPivot(currRot, center);
// GUI.DrawTexture(buttonRect, m_swipeButtonActive, smode);
// Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y));
// textRect.x = newPos.x - textRect.width / 2.0f;
// textRect.y = newPos.y - textRect.height / 2.0f;
// GUI.matrix = Matrix4x4.identity;
// selected = GUI.Button(textRect, text, optionStyle.name+"-hover");
// }
// else // Display button also usable for mouse interaction
// {
// GUIUtility.RotateAroundPivot(currRot, center);
// GUI.DrawTexture(buttonRect, m_swipeButtonNormal, smode);
// Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y));
// textRect.x = newPos.x - textRect.width / 2.0f;
// textRect.y = newPos.y - textRect.height / 2.0f;
// GUI.matrix = Matrix4x4.identity;
// selected = GUI.Button(textRect, text, optionStyle);
// }
//
//
// // Check for full swipe
// // Of right hand
// if (!selected && m_currentUser > 0
// && withinInteractionFrame
// && Fubi.getCurrentTime() - m_lastSwipeRecognition > 1.0f) // at least one second between to swipes
// {
// selected = Fubi.getCombinationRecognitionProgressOn(m_swipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED;
// if (selected)
// {
// m_swipeMenuActive = false;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastSwipeRecognition = Fubi.getCurrentTime();
// }
// else
// Fubi.enableCombinationRecognition(m_swipeRecognizers[options.Length - 1][i], m_currentUser, true);
// }
//
// // Or left hand
// if (!selected && m_currentUser > 0
// && withinInteractionFrame
// && Fubi.getCurrentTime() - m_lastLeftSwipeRecognition > 1.0f) // at least one second between to swipes
// {
// selected = Fubi.getCombinationRecognitionProgressOn(m_leftSwipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED;
// if (selected)
// {
// m_swipeMenuActive = false;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastLeftSwipeRecognition = Fubi.getCurrentTime();
// }
// else
// Fubi.enableCombinationRecognition(m_leftSwipeRecognizers[options.Length - 1][i], m_currentUser, true);
// }
//
//
// if (selected)
// {
// selection = text;
// if (m_swipeSound && m_swipeSound.isReadyToPlay)
// AudioSource.PlayClipAtPoint(m_swipeSound, new Vector3(0,0,0), 1.0f);
// break;
// }
// currRot += rotAdd;
// }
// return selection;
// }
//
bool calibrateCursorMapping(uint id)
{
m_aspect = (float)Screen.width / (float)Screen.height;
if (id > 0)
{
FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW;
FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND;
float confidence;
double timeStamp;
float elbowX, elbowY, elbowZ;
Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float shoulderX, shoulderY, shoulderZ;
Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist1 = Mathf.Sqrt(Mathf.Pow(elbowX - shoulderX, 2) + Mathf.Pow(elbowY - shoulderY, 2) + Mathf.Pow(elbowZ - shoulderZ, 2));
float handX, handY, handZ;
Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist2 = Mathf.Sqrt(Mathf.Pow(elbowX - handX, 2) + Mathf.Pow(elbowY - handY, 2) + Mathf.Pow(elbowZ - handZ, 2));
m_mapping.height = (float)(dist1 + dist2);
// Calculate all others in depence of maph
m_mapping.y = 200.0f / 550.0f * m_mapping.height;
m_mapping.width = m_mapping.height / m_aspect;
m_mapping.x = -100.0f / (550.0f / m_aspect) * m_mapping.width;
//Debug.Log ("m_mapping.x=" + m_mapping.x + "m_mapping.y=" + m_mapping.y + "m_mapping.height=" + m_mapping.height + "m_mapping.width=" + m_mapping.width);
return(true);
}
}
}
}
return(false);
}
// Called for rendering the gui
void OnGUI()
{
// AA: Position the depth image so the user can see the kinect output
if (!m_disableTrackingImage && (!m_disableTrackingImageWithSwipeMenu || !m_swipeMenuDisplayedLastFrame))
{
// Debug image
GUI.depth = -4;
GUI.DrawTexture(new Rect(25, Screen.height - m_yRes / m_factor - 25, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture);
//GUI.DrawTexture(new Rect(Screen.width-m_xRes/m_factor, Screen.height-m_yRes/m_factor, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture);
}
//AA: add the GUI elements
int shift = 42;
GUI.Box(new Rect(5 + shift, 25, Screen.width / 3 - 130, Screen.height - 50), "FILTERS");
m_bUseSimpleAverage = GUI.Toggle(new Rect(45 + shift, 50, 200, 30), m_bUseSimpleAverage, " SIMPLE AVERAGE 10");
m_bUseMovingAverage = GUI.Toggle(new Rect(45 + shift, 90, 200, 30), m_bUseMovingAverage, " MOVING AVERAGE");
m_bUseSimpleAverage5 = GUI.Toggle(new Rect(45 + shift, 130, 200, 30), m_bUseSimpleAverage5, " SIMPLE AVERAGE 5");
m_bDblMovingAverage = GUI.Toggle(new Rect(45 + shift, 170, 200, 30), m_bDblMovingAverage, " DOUBLE MOV AVERAGE");
m_bUseExpSmoothing = GUI.Toggle(new Rect(45 + shift, 210, 200, 30), m_bUseExpSmoothing, " EXP SMOOTHING");
m_bUseDblExpSmoothing = GUI.Toggle(new Rect(45 + shift, 250, 200, 30), m_bUseDblExpSmoothing, " DOUBLE EXP SMOOTHING");
m_bUseAdaptive = GUI.Toggle(new Rect(45 + shift, 290, 200, 30), m_bUseAdaptive, " ADAPTIVE DBL EXP");
m_bUseMedian = GUI.Toggle(new Rect(45 + shift, 330, 200, 30), m_bUseMedian, " MEDIAN");
m_bUseCombination1 = GUI.Toggle(new Rect(45 + shift, 370, 200, 30), m_bUseCombination1, " SIMPLE AVG + Median");
m_bUseCombination2 = GUI.Toggle(new Rect(45 + shift, 410, 200, 30), m_bUseCombination2, " DBL MOV AVG + Median");
m_bUseNone = GUI.Toggle(new Rect(45 + shift, 450, 200, 30), m_bUseNone, " NONE");
if (GUI.Button(new Rect(50 + shift, Screen.height - 75, 150, 30), "Clear"))
{
WriteableAreaResize();
fv.DrawCircle();
}
// If some button has been pressed OR this is the first exection
if (GUI.changed)
{
LoadFilters();
if (fm.filters.Count == 1)
{
m_principalCursor = 0;
}
}
int count = 0;
if (m_bUseSimpleAverage)
{
GUI.Label(new Rect(15 + shift, 45, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseMovingAverage)
{
GUI.Label(new Rect(15 + shift, 85, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseSimpleAverage5)
{
GUI.Label(new Rect(15 + shift, 125, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bDblMovingAverage)
{
GUI.Label(new Rect(15 + shift, 165, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseExpSmoothing)
{
GUI.Label(new Rect(15 + shift, 205, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseDblExpSmoothing)
{
GUI.Label(new Rect(15 + shift, 245, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseAdaptive)
{
GUI.Label(new Rect(15 + shift, 285, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseMedian)
{
GUI.Label(new Rect(15 + shift, 325, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseCombination1)
{
GUI.Label(new Rect(15 + shift, 365, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseCombination2)
{
GUI.Label(new Rect(15 + shift, 405, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseNone)
{
GUI.Label(new Rect(15 + shift, 445, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (prevScreenWidth != Screen.width || prevScreenHeight != Screen.height)
{
// Resize writeable area, redraw the circle
WriteableAreaResize();
prevScreenWidth = Screen.width;
prevScreenHeight = Screen.height;
fv.DrawCircle();
}
//AA: Draw the writeable area
fv.Apply();
for (int i = 0; i < fv.m_filterOutputTexture.Count; i++)
{
GUI.DrawTexture(new Rect(fv.filterOutputLocX, fv.filterOutputLocY, fv.filterOutputWidth, fv.filterOutputHeight), fv.m_filterOutputTexture[i]);
}
// Cursor
m_gotNewFubiCoordinates = false;
if (Fubi.isInitialized())
{
// Take closest user
uint userID = Fubi.getClosestUserID();
if (userID != m_currentUser)
{
m_currentUser = userID;
m_lastCalibrationSucceded = false;
}
if (userID > 0)
{
if (!m_lastCalibrationSucceded)
{
m_lastCalibrationSucceded = calibrateCursorMapping(m_currentUser);
}
FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
// Get hand and shoulder position and check their confidence
double timeStamp;
float handX, handY, handZ, confidence;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// AA: Filtering should happen here for the hand and relative joints separately
// If true, use the smoothed joints for calculating screen coordinates
fm.UpdateJointFilters(new Vector3(handX, handY, handZ), new Vector3(relX, relY, relZ));
for (int i = 0; i < fm.filters.Count; i++)
{
if (m_bUseJointFiltering)
{
//Debug.Log ("Prehand " + new Vector3(handX, handY, handZ) + " relJoint " + new Vector3(relX, relY, relZ));
Vector3 handPos = fm.joints[i]; // filter.Update(new Vector3(handX, handY, handZ), Filter.JOINT_TYPE.JOINT);
Vector3 relJointPos = fm.relativeJoints[i]; //filter.Update(new Vector3(relX, relY, relZ), Filter.JOINT_TYPE.RELATIVEJOINT);
//Debug.Log ("hand " + handPos + " relJoint " + relJointPos);
handZ = handPos.z;
handY = handPos.y;
handX = handPos.x;
relZ = relJointPos.z;
relY = relJointPos.y;
relX = relJointPos.x;
m_relativeCursorPosition = fm.relativeCursorPosition[i];
}
// AA: End
// Take relative coordinates
float zDiff = handZ - relZ;
float yDiff = handY - relY;
float xDiff = handX - relX;
// Check if hand is enough in front of shoulder
if ((yDiff > 0 && zDiff < -150.0f) || (Mathf.Abs(xDiff) > 150.0f && zDiff < -175.0f) || zDiff < -225.0f)
{
// Now get the possible cursor position
// Convert to screen coordinates
float newX, newY;
float mapX = m_mapping.x;
newX = (xDiff - mapX) / m_mapping.width;
newY = (m_mapping.y - yDiff) / m_mapping.height; // Flip y for the screen coordinates
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
float changeX = newX - m_relativeCursorPosition.x;
float changeY = newY - m_relativeCursorPosition.y;
if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp)
{
float changeLength = Mathf.Sqrt(changeX * changeX + changeY * changeY);
float filterFactor = changeLength; //Mathf.Sqrt(changeLength);
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
// AA: Filtering should happen here for joint-to-relativejoint (VECTOR) filtering
// AA: filtering code
Vector2 tempNew = new Vector2(newX, newY);
fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor);
// If true, use the calculated factor for smoothing, else just use the new
if (m_bUseVectorFiltering)
{
m_relativeCursorPosition = fm.vectors[i]; //filter.Update(m_relativeCursorPosition, tempNew, filterFactor);
}
else // Just give equal weight to both
{
m_relativeCursorPosition = filter.Update(m_relativeCursorPosition, tempNew, 0.5f);
}
// AA: Calculate all filters
// fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor);
m_timeStamp = timeStamp;
// Send it, but only if it is more or less within the screen
if (m_relativeCursorPosition.x > -0.1f && m_relativeCursorPosition.x < 1.1f &&
m_relativeCursorPosition.y > -0.1f && m_relativeCursorPosition.y < 1.1f)
{
MoveMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, i);
// Each filter must store it's own value of relative position, absolute and previous absolute positions
fm.relativeCursorPosition[i] = m_relativeCursorPosition;
fm.absPixelPosition[i] = m_absPixelPosition;
fm.prevAbsPixelPosition[i] = m_previousAbsPixelPosition;
DrawFilterOutputs(i);
m_gotNewFubiCoordinates = true;
m_lastCursorChangeDoneByFubi = true;
}
}
}
}
}
}
}
}
// AA: FUBI does not move mouse if the confidence value is too low
if (!m_gotNewFubiCoordinates) // AA: this only executes when input is coming from mouse
{
// Got no mouse coordinates from fubi this frame
Vector2 mousePos = Input.mousePosition;
// Only move mouse if it wasn't changed by fubi the last time or or it really has changed
if (!m_lastCursorChangeDoneByFubi || mousePos != m_lastMousePos)
{
//AA: Old code for cursor placement
m_relativeCursorPosition.x = mousePos.x / (float)Screen.width;
m_relativeCursorPosition.y = 1.0f - (mousePos.y / (float)Screen.height);
// Get mouse X and Y position as a percentage of screen width and height
MoveActualMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, true);
m_lastMousePos = mousePos;
m_lastCursorChangeDoneByFubi = false;
}
}
}
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false)
{
x = float.NaN;
y = float.NaN;
float handX, handY, handZ, confidence;
FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
if (leftHand)
{
joint = FubiUtils.SkeletonJoint.LEFT_HAND;
relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
}
double timeStamp;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// Take relative coordinates
float rawX = handX - relX;
float rawY = handY - relY;
// Convert to screen coordinates
float newX, newY;
float mapX = m_mapX;
if (leftHand)
{
// Mirror x area for left hand
mapX = -m_mapX - m_mapW;
}
newX = (rawX - mapX) / m_mapW;
newY = (m_mapY - rawY) / m_mapH; // Flip y for the screen coordinates
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
float changeX = newX - m_x;
float changeY = newY - m_y;
if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp)
{
double changeLength = Math.Sqrt(changeX * changeX + changeY * changeY);
double changeDur = timeStamp - m_timeStamp;
float filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor;
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
m_x = (1.0f - filterFactor) * m_x + filterFactor * newX;
m_y = (1.0f - filterFactor) * m_y + filterFactor * newY;
m_timeStamp = timeStamp;
// Send it
MouseSimulator.sendMousePos(m_x, m_y);
x = m_x;
y = m_x;
}
// Check for mouse click
if (Fubi.recognizeGestureOn("mouseClick", userID) == FubiUtils.RecognitionResult.RECOGNIZED ||
Fubi.recognizeGestureOn("mouseClick1", userID) == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (m_timeStamp - m_lastMouseClick > 1)
{
MouseSimulator.sendMouseButton(true, true);
MouseSimulator.sendMouseButton(true, false);
m_lastMouseClick = m_timeStamp;
}
}
}
}
}
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false)
{
x = float.NaN;
y = float.NaN;
float handX, handY, handZ, confidence;
var joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
var relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
if (leftHand)
{
joint = FubiUtils.SkeletonJoint.LEFT_HAND;
relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
}
double timeStamp;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// Take relative coordinates
var rawX = handX - relX;
var rawY = handY - relY;
// Convert to screen coordinates
var mapX = m_mapX;
if (leftHand)
{
// Mirror x area for left hand
mapX = -m_mapX - m_mapW;
}
float newX = (rawX - mapX) / m_mapW;
float newY = (m_mapY - rawY) / m_mapH;
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
var changeX = newX - m_x;
var changeY = newY - m_y;
if (Math.Abs(changeX) > float.Epsilon || Math.Abs(changeY) > float.Epsilon && Math.Abs(timeStamp - m_timeStamp) > float.Epsilon)
{
var changeLength = Math.Sqrt(changeX * changeX + changeY * changeY);
var filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor;
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
m_x = (1.0f - filterFactor) * m_x + filterFactor * newX;
m_y = (1.0f - filterFactor) * m_y + filterFactor * newY;
m_timeStamp = timeStamp;
// Send it
MouseKeyboardSimulator.sendMousePos(m_x, m_y);
x = m_x;
y = m_x;
}
}
}
}