private int recordMedianFingerCount(Stopwatch timer, double duration, double overallDuration, uint targetId, CancellationToken ct)
{
var leftHand = Options.SelectedJoints[0].Main == FubiUtils.SkeletonJoint.LEFT_HAND;
var recordedFingerCounts = new List <int>();
do
{
if (ct.IsCancellationRequested)
{
setDescription("Cancelled!");
return(-1);
}
if (UseHand)
{
recordedFingerCounts.Add(Fubi.getHandFingerCount(targetId));
}
else
{
recordedFingerCounts.Add(Fubi.getFingerCount(targetId, leftHand, false));
}
decrementDuration(overallDuration, timer);
// With 10 ms sleep we should still catch all different values, but we don't get too many
Thread.Sleep(10);
} while (timer.Elapsed < TimeSpan.FromSeconds(duration));
recordedFingerCounts.Sort();
return(recordedFingerCounts[recordedFingerCounts.Count / 2]);
}
public void refreshGestureLists()
{
foreach (var dt in m_basicRecognizersTable)
{
// Remove rows if too many
while (dt.Rows.Count > Fubi.getNumUserDefinedRecognizers())
{
dt.Rows.RemoveAt(dt.Rows.Count - 1);
}
// Update/Add rows
for (uint p = 0; p < Fubi.getNumUserDefinedRecognizers(); p++)
{
string recName = Fubi.getUserDefinedRecognizerName(p);
if (p >= dt.Rows.Count)
{
// New entry
dt.Rows.Add(recName, false);
}
else if (dt.Rows[(int)p]["Name"] as string != recName)
{
// Replace entry with changed name
dt.Rows[(int)p]["Name"] = recName;
dt.Rows[(int)p]["IsChecked"] = false;
}
}
}
}
// Check what gestures the user is performing during the given time and take the one kept for the longest duration
List <string> recordLongestGestures(Stopwatch timer, TimeSpan duration, CancellationToken ct)
{
var longestGestures = new List <string>();
var recordedGestureDurations = new Dictionary <uint, double>();
var id = getTargetID();
double lastTime = 0;
do
{
if (ct.IsCancellationRequested)
{
setDescription("Cancelled!");
return(longestGestures);
}
var currentTime = timer.ElapsedMilliseconds / 1000.0;
var timeStep = currentTime - lastTime;
lastTime = currentTime;
setTimer(Math.Round(currentTime, 1));
for (uint p = 0; p < Fubi.getNumUserDefinedRecognizers(); p++)
{
FubiUtils.RecognitionResult res;
lock (MainWindow.LockFubiUpdate)
{
res = recognizeGesture(Fubi.getUserDefinedRecognizerName(p), id);
}
if (res == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (recordedGestureDurations.ContainsKey(p))
{
recordedGestureDurations[p] += timeStep;
}
else
{
recordedGestureDurations.Add(p, timeStep);
}
setGestureLabel(Fubi.getUserDefinedRecognizerName(p));
}
}
Thread.Sleep(10);
} while (timer.Elapsed < duration);
if (recordedGestureDurations.Count != 0)
{
// Find gestures that were recognized for at least half the duration
var halfDuration = duration.TotalSeconds / 2.0;
longestGestures.AddRange(from gesture in recordedGestureDurations where gesture.Value > halfDuration select Fubi.getUserDefinedRecognizerName(gesture.Key));
if (longestGestures.Count == 0)
{
// Add at least the gesture id with the longest duration in the dictionary
var gestureId = recordedGestureDurations.Aggregate((l, r) => l.Value > r.Value ? l : r).Key;
longestGestures.Add(Fubi.getUserDefinedRecognizerName(gestureId));
}
}
return(longestGestures);
}
public void applyBinding(string gesture, bool down)
{
var b = getBinding(gesture);
if (b != null && b.KeyMouseEvent != null && b.KeyMouseEvent != "")
{
var key = b.KeyMouseEvent;
var mod = "";
if (b.KeyMouseEvent.Contains("+"))
{
var tokens = key.Split('+');
if (tokens.Length > 1)
{
key = tokens[1];
mod = tokens[0];
}
else if (tokens.Length > 0)
{
key = tokens[0];
}
}
// First handle modifiers
if (mod.Length > 0)
{
MouseKeyboardSimulator.sendKeyEvent("Left" + mod, down);
}
// Handle mouse buttons
if (key == "LMB")
{
if (down || Fubi.getCurrentTime() - m_lastMouseClick > 1)
{
MouseKeyboardSimulator.sendMouseButton(MouseKeyboardSimulator.MouseButtonType.LEFT, down);
if (!down)
{
m_lastMouseClick = Fubi.getCurrentTime();
}
}
}
else if (key == "RMB")
{
MouseKeyboardSimulator.sendMouseButton(MouseKeyboardSimulator.MouseButtonType.RIGHT, down);
}
else if (key == "MMB")
{
MouseKeyboardSimulator.sendMouseButton(MouseKeyboardSimulator.MouseButtonType.MIDDLE, down);
}
else if (key == "X1MB" || key == "X2MB")
{
MouseKeyboardSimulator.sendMouseButton(MouseKeyboardSimulator.MouseButtonType.X, down);
}
// Handle keys
else if (key.Length > 0)
{
MouseKeyboardSimulator.sendKeyEvent(key, down);
}
}
}
private uint getPlaybackOrClosestHandID()
{
if (Fubi.isPlayingSkeletonData())
{
return(FubiUtils.PlaybackHandID);
}
return(Fubi.getClosestHandID());
}
// Called on deactivation
void OnDestroy()
{
if (this == instance)
{
Fubi.release();
Debug.Log("Fubi released!");
}
}
void UpdateDepthImageTexture()
{
depthImageUpdateCompleted = false;
// Get debug image
Fubi.getImage(m_rawImage, FubiUtils.ImageType.Depth, FubiUtils.ImageNumChannels.C4, FubiUtils.ImageDepth.D8);
UpdateDepthMapTexture();
depthImageUpdateCompleted = true;
}
// AA: Removing this function disables click functionality so have not commented it out
static public bool FubiGesture(Rect r, string name, GUIStyle style)
{
GUI.depth = -2;
instance.m_gesturesDisplayed = true;
if (instance.m_gestureSymbolDict.ContainsKey(name))
{
MultiTexture2D animation = instance.m_gestureSymbolDict[name];
int index = (int)(Time.realtimeSinceStartup * animation.animationFps) % animation.Length;
GUI.DrawTexture(r, animation[index], ScaleMode.ScaleToFit);
if (GUI.Button(r, "", style))
{
return(true);
}
if (instance.m_disableCursorWithGestures && instance.m_gesturesDisplayedLastFrame &&
rectContainsCursor(r) && clickRecognized())
{
return(true);
}
}
if (Fubi.getCurrentTime() - instance.m_lastGesture > 0.8f)
{
uint userID = instance.m_currentUser;
if (userID > 0)
{
FubiTrackingData[] userStates;
if (Fubi.getCombinationRecognitionProgressOn(name, userID, out userStates, false) == FubiUtils.RecognitionResult.RECOGNIZED && userStates.Length > 0)
{
double time = userStates[userStates.Length - 1].timeStamp;
// Check if gesture did not happen longer ago then 1 second
if (Fubi.getCurrentTime() - time < 1.0f)
{
instance.m_lastGesture = time;
// Reset all recognizers
Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, userID, false);
return(true);
}
}
// Unsuccesfull recognition so start the recognizer for the next recognition
Fubi.enableCombinationRecognition(name, userID, true);
if (Fubi.recognizeGestureOn(name, userID) == FubiUtils.RecognitionResult.RECOGNIZED)
{
instance.m_lastGesture = Fubi.getCurrentTime();
// Reset all recognizers
Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, userID, false);
return(true);
}
}
}
return(false);
}
//AA: We don't need the FubiSwipeMenu. Removed 'FubiSwipeMenu(Vector2 center, float radius, string[] options, GUIStyle optionStyle, GUIStyle centerStyle)'
// static public string FubiSwipeMenu(Vector2 center, float radius, string[] options, GUIStyle optionStyle, GUIStyle centerStyle)
// {
// if (instance)
// return instance.DisplayFubiSwipeMenu(center, radius, options, optionStyle, centerStyle);
// return "";
// }
// static public bool FubiCroppedUserImage(int x, int y, bool forceReload = false)
// {
// if (instance)
// return instance.DisplayFubiCroppedUserImage(x, y, forceReload);
// return false;
// }
// AA: The 'DisplayFubiCroppedUserImage' function could be used to explain the problems with filtering
private bool DisplayFubiCroppedUserImage(int x, int y, bool forceReload)
{
if (m_userImageTexture == null || forceReload == true)
{
// First get user image
Fubi.getImage(m_rawImage, FubiUtils.ImageType.Color, FubiUtils.ImageNumChannels.C4, FubiUtils.ImageDepth.D8, (uint)FubiUtils.RenderOptions.None, (uint)FubiUtils.JointsToRender.ALL_JOINTS, FubiUtils.DepthImageModification.Raw, m_currentUser, FubiUtils.SkeletonJoint.HEAD, true);
// Now look for the image borders
int xMax = m_xRes; int yMax = m_yRes;
int index = 0;
for (int x1 = 0; x1 < m_xRes; ++x1, index += 4)
{
if (m_rawImage[index + 3] == 0)
{
xMax = x1;
break;
}
}
index = 0;
for (int y1 = 0; y1 < m_yRes; ++y1, index += (m_xRes + 1) * 4)
{
if (m_rawImage[index + 3] == 0)
{
yMax = y1;
break;
}
}
// Create the texture
m_userImageTexture = new Texture2D(xMax, yMax, TextureFormat.RGBA32, false);
Color[] pixels = new Color[xMax * yMax];
// And copy the pixels
int i = xMax * yMax - 1;
index = 0;
for (int yy = 0; yy < yMax; ++yy)
{
index += (xMax - 1) * 4; // Move to line end
for (int xx = 0; xx < xMax; ++xx, --i, index -= 4)
{
pixels[i] = new Color(m_rawImage[index] / 255.0f, m_rawImage[index + 1] / 255.0f, m_rawImage[index + 2] / 255.0f, m_rawImage[index + 3] / 255.0f);
}
index += (m_xRes + 1) * 4; // Move to next line
}
m_userImageTexture.SetPixels(pixels);
m_userImageTexture.Apply();
}
GUI.depth = -4;
GUI.DrawTexture(new Rect(x, y, m_userImageTexture.width, m_userImageTexture.height), m_userImageTexture);
return(false);
}
public override void trainValues(double start, double duration, CancellationToken ct)
{
waitForStart(start, ct);
var userId = getTargetID();
var fifthDuration = duration / 5;
var stopwatch = Stopwatch.StartNew();
// Record avg pos for first fifth duration
var startPos = recordAvgValue(stopwatch, fifthDuration, duration, userId, ct);
while (stopwatch.Elapsed < TimeSpan.FromSeconds(duration - fifthDuration))
{
if (ct.IsCancellationRequested)
{
setDescription("Cancelled!");
return;
}
decrementDuration(duration, stopwatch);
}
// Record avg pos for last fifth duration
var endPos = recordAvgValue(stopwatch, duration, duration, userId, ct);
// Record measurement distance
if (Options.MeasuringUnit != FubiUtils.BodyMeasurement.NUM_MEASUREMENTS)
{
float measureConfidence;
Fubi.getBodyMeasurementDistance(userId, Options.MeasuringUnit, out m_measureDist, out measureConfidence);
}
// Calc change between first and last fifth avg position
AvgValue = endPos - startPos;
// Apply active axes
if ((Options.IgnoreAxes & (uint)FubiUtils.CoordinateAxis.X) != 0)
{
AvgValue.X = 0;
}
if ((Options.IgnoreAxes & (uint)FubiUtils.CoordinateAxis.Y) != 0)
{
AvgValue.Y = 0;
}
if ((Options.IgnoreAxes & (uint)FubiUtils.CoordinateAxis.Z) != 0)
{
AvgValue.Z = 0;
}
// Calculate speed
m_speed = (endPos - startPos).Length / (duration - fifthDuration);
}
static private bool clickRecognized()
{
bool click = false;
if (Fubi.getCurrentTime() - instance.m_lastMouseClick > 0.5f)
{
uint userID = instance.m_currentUser;
if (userID > 0)
{
// Check for mouse click as defined in xml
FubiTrackingData[] userStates;
if (Fubi.getCombinationRecognitionProgressOn("mouseClick", userID, out userStates, false) == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (userStates != null && userStates.Length > 0)
{
double clickTime = userStates[userStates.Length - 1].timeStamp;
// Check that click occured no longer ago than 1 second
if (Fubi.getCurrentTime() - clickTime < 1.0f)
{
click = true;
instance.m_lastMouseClick = clickTime;
// Reset all recognizers
Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, userID, false);
}
}
}
if (!click)
{
Fubi.enableCombinationRecognition("mouseClick", userID, true);
}
if (Fubi.recognizeGestureOn("mouseClick", userID) == FubiUtils.RecognitionResult.RECOGNIZED)
{
Debug.Log("Mouse click recognized.");
click = true;
instance.m_lastMouseClick = Fubi.getCurrentTime();
// Reset all recognizers
Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, userID, false);
}
}
}
return(click);
}
private void initFubi()
{
List <string> availableSensors = new List <string>();
availableSensors.Add(Enum.GetName(typeof(FubiUtils.SensorType), FubiUtils.SensorType.NONE));
FubiUtils.SensorType type = FubiUtils.SensorType.NONE;
int avSensors = Fubi.getAvailableSensors();
if ((avSensors & (int)FubiUtils.SensorType.OPENNI2) != 0)
{
type = FubiUtils.SensorType.OPENNI2;
availableSensors.Add(Enum.GetName(typeof(FubiUtils.SensorType), type));
}
if ((avSensors & (int)FubiUtils.SensorType.KINECTSDK) != 0)
{
if (type == FubiUtils.SensorType.NONE)
{
type = FubiUtils.SensorType.KINECTSDK;
}
availableSensors.Add(Enum.GetName(typeof(FubiUtils.SensorType), FubiUtils.SensorType.KINECTSDK));
}
if ((avSensors & (int)FubiUtils.SensorType.OPENNI1) != 0)
{
if (type == FubiUtils.SensorType.NONE)
{
type = FubiUtils.SensorType.OPENNI1;
}
availableSensors.Add(Enum.GetName(typeof(FubiUtils.SensorType), FubiUtils.SensorType.OPENNI1));
}
string selectedName = Enum.GetName(typeof(FubiUtils.SensorType), type);
foreach (string sType in availableSensors)
{
int index = sensorSelectionComboBox.Items.Add(sType);
if (index > -1 && sType == selectedName)
{
sensorSelectionComboBox.SelectedIndex = index;
}
}
switchSensorOnNextUpdate = false;
Fubi.init(new FubiUtils.SensorOptions(new FubiUtils.StreamOptions(640, 480, 30), new FubiUtils.StreamOptions(640, 480), new FubiUtils.StreamOptions(-1, -1, -1), type));
Fubi.loadRecognizersFromXML("MouseControlRecognizers.xml");
}
public void autoCalibrateMapping(bool leftHand)
{
uint id = Fubi.getClosestUserID();
if (id > 0)
{
FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW;
FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND;
if (leftHand)
{
elbow = FubiUtils.SkeletonJoint.LEFT_ELBOW;
shoulder = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
hand = FubiUtils.SkeletonJoint.LEFT_HAND;
}
float confidence;
double timeStamp;
float elbowX, elbowY, elbowZ;
Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float shoulderX, shoulderY, shoulderZ;
Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist1 = Math.Sqrt(Math.Pow(elbowX - shoulderX, 2) + Math.Pow(elbowY - shoulderY, 2) + Math.Pow(elbowZ - shoulderZ, 2));
float handX, handY, handZ;
Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist2 = Math.Sqrt(Math.Pow(elbowX - handX, 2) + Math.Pow(elbowY - handY, 2) + Math.Pow(elbowZ - handZ, 2));
MapH = (float)(dist1 + dist2);
// Calculate all others in depence of maph
MapY = 250.0f / 550.0f * MapH;
MapW = MapH / m_aspect;
MapX = -100.0f / (550.0f / m_aspect) * MapW;
}
}
}
}
}
protected Vector3D getCurrentJointValue(uint targetID, FubiUtils.SkeletonJoint jointID, FubiUtils.BodyMeasurement measureID = FubiUtils.BodyMeasurement.NUM_MEASUREMENTS)
{
float x, y, z, confidence;
double timeStamp;
if (Type == XMLGenerator.RecognizerType.JointRelation || Type == XMLGenerator.RecognizerType.LinearMovement)
{
if (UseHand)
{
Fubi.getCurrentHandJointPosition(targetID, (FubiUtils.SkeletonHandJoint)jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
else
{
Fubi.getCurrentSkeletonJointPosition(targetID, jointID, out x, out y, out z, out confidence, out timeStamp, Options.Local, Options.Filtered);
if (measureID != FubiUtils.BodyMeasurement.NUM_MEASUREMENTS && Type == XMLGenerator.RecognizerType.JointRelation)
{
float measureDist, measureConfidence;
Fubi.getBodyMeasurementDistance(targetID, measureID, out measureDist, out measureConfidence);
if (measureDist > 0)
{
x /= measureDist;
y /= measureDist;
z /= measureDist;
}
}
}
}
else
{
var mat = new float[9];
if (UseHand)
{
Fubi.getCurrentHandJointOrientation(targetID, (FubiUtils.SkeletonHandJoint)jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
else
{
Fubi.getCurrentSkeletonJointOrientation(targetID, jointID, mat, out confidence, out timeStamp, Options.Local, Options.Filtered);
}
FubiUtils.Math.rotMatToRotation(mat, out x, out y, out z);
}
return(new Vector3D(x, y, z));
}
private void recognitionStart(string gestureName, uint targetID, bool isHand, FubiUtils.RecognizerType recognizerType)
{
switch (recognizerType)
{
// case FubiUtils.RecognizerType.PREDEFINED_GESTURE:
// {
// Console.WriteLine("-->" + (isHand ? "Hand " : "User ") + targetID + ": END OF " + gestureName + "\n");
// }
// break;
// case FubiUtils.RecognizerType.USERDEFINED_GESTURE:
// {
// Console.WriteLine("-->" + (isHand ? "Hand " : "User ") + targetID + ": END OF " + gestureName + "\n");
// }
// break;
case FubiUtils.RecognizerType.USERDEFINED_COMBINATION:
// case FubiUtils.RecognizerType.PREDEFINED_COMBINATION:
// {
// // Nothing to do here..
// }
Console.WriteLine(recognizerType.ToString() + "-->" + "User " + targetID + ": START OF " + gestureName + "\n");
uint numStates;
bool isInterrupted, isInTransition;
int gestureState = Fubi.getCurrentCombinationRecognitionState(gestureName, targetID, out numStates, out isInterrupted, out isInTransition) + 1;
Console.WriteLine("State:" + gestureState + " NumStates:" + numStates + " IsInterupted:" + isInterrupted + " IsInTransition:" + isInTransition);
activeGestures.Add(gestureName);
//if (true) // (gestureState == numStates)
//{
// if (gestureName == gameStageGesture[gameStage])
// {
// if (gameStage != 5 || isHipWobble)
// setProgress(gameStage + 1);
// }
// else if (gestureName == "hipWobble")
// isHipWobble = true;
//}
if (gameStage < gameStageGesture.Length)
{
gestureStep = testExpression(gameStageGesture[gameStage], activeGestures); // lets gui update know to advance to next stage
}
break;
}
}
private void button1_Click(object sender, RoutedEventArgs e)
{
Microsoft.Win32.OpenFileDialog dlg = new Microsoft.Win32.OpenFileDialog();
dlg.FileName = "SampleRecognizers"; // Default file name
dlg.DefaultExt = ".xml"; // Default file extension
dlg.Filter = "MXL documents (.xml)|*.xml"; // Filter files by extension
// Show open file dialog box
Nullable <bool> result = dlg.ShowDialog();
// Process open file dialog box results
if (result == true && dlg.FileName != null)
{
// Open document
Fubi.loadRecognizersFromXML(dlg.FileName);
// Enable clear button if we have loaded some recognizers
button3.IsEnabled = (Fubi.getNumUserDefinedCombinationRecognizers() > 0 || Fubi.getNumUserDefinedRecognizers() > 0);
}
}
private GameObject cam; //, pointerObject;
void Start()
{
FubiNET.Fubi.init(new FubiUtils.SensorOptions(new FubiUtils.StreamOptions(640, 480, 30), new FubiUtils.StreamOptions(-1, -1, -1), new FubiUtils.StreamOptions(-1, -1, -1), FubiUtils.SensorType.KINECTSDK));
if (!FubiNET.Fubi.isInitialized())
{
if (!FubiNET.Fubi.init(configXmlPath))
{
Debug.Log("Fubi can't initialiaze");
BroadcastMessage("LabelMsgVisualize", "Failed to initialize kinect sensor....", SendMessageOptions.DontRequireReceiver);
BroadcastMessage("KinectFailed", SendMessageOptions.DontRequireReceiver); //for session manager
}
}
if (Fubi.isInitialized())
{
FubiNET.Fubi.setAutoStartCombinationRecognition(true); //Fubi 0.5.1 wrapper
FubiNET.Fubi.loadRecognizersFromXML(recognnizerXmlPath);
BroadcastMessage("KinectStarted", SendMessageOptions.DontRequireReceiver); //for session manager
}
cam = GameObject.Find("Main Camera");
//pointerObject = GameObject.Find("PointerObjects");
}
// private string DisplayFubiSwipeMenu(Vector2 center, float radius, string[] options, GUIStyle optionStyle, GUIStyle centerStyle)
// {
// if (!(radius > 0 && options.Length > 0 && options.Length <= m_swipeRecognizers.Length))
// {
// Debug.LogWarning("FubiSwipeMenu called with incorrect parameters!");
// Debug.DebugBreak();
// return "";
// }
//
// GUI.depth = -2;
// string selection = "";
// m_swipeMenuDisplayed = true;
//
// ScaleMode smode = /*(options.Length >= 2) ? ScaleMode.StretchToFill :*/ ScaleMode.ScaleToFit;
//
// // Display center and explanation text
// float centerRad = radius * 0.25f;
// Rect centerRect = new Rect(center.x - centerRad, center.y - centerRad, 2.0f * centerRad, 2.0f * centerRad);
//
// // Check for right hand in front for general activation
// if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_handToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED)
// m_lastHandToFront = Fubi.getCurrentTime();
// else if (Fubi.getCurrentTime() - m_lastHandToFront > 1.2f)
// m_handToFrontEnd = Fubi.getCurrentTime();
// // Check for left hand in front for general activation
// if (m_currentUser > 0 && Fubi.recognizeGestureOn(m_leftHandToFrontRecognizer, m_currentUser) == FubiUtils.RecognitionResult.RECOGNIZED)
// m_lastLeftHandToFront = Fubi.getCurrentTime();
// else if (Fubi.getCurrentTime() - m_lastLeftHandToFront > 1.2f)
// m_leftHandToFrontEnd = Fubi.getCurrentTime();
//
// bool withinInteractionFrame = m_currentUser > 0 && (m_lastHandToFront - m_handToFrontEnd > 0.5 || m_lastLeftHandToFront - m_leftHandToFrontEnd > 0.5);
// if (withinInteractionFrame)
// {
// if (!m_swipeMenuActive)
// {
// m_swipeMenuActive = true;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// }
// GUI.DrawTexture(centerRect, m_swipeCenterActive, smode);
// GUI.Label(centerRect, "Swipe for \nselection", centerStyle.name+"-hover");
// }
// else
// {
// m_swipeMenuActive = false;
// GUI.DrawTexture(centerRect, m_swipeCenterNormal, smode);
// if (m_lastHandToFront > m_handToFrontEnd || m_lastLeftHandToFront > m_leftHandToFrontEnd)
// {
// GUI.Label(centerRect, "Hold arm", centerStyle);
// }
// else
// GUI.Label(centerRect, "Stretch arm \nto front", centerStyle);
// }
//
// // Display the options and check their recognizers
// float rotAdd = 360.0f / options.Length;
// float currRot = (options.Length > 2) ? 0 : 90.0f;
// Rect buttonRect = new Rect();
// buttonRect.height = radius * 0.35f;
// buttonRect.width = (options.Length > 2) ? (Mathf.PI * (radius / 2.0f) / options.Length) : buttonRect.height;
// buttonRect.y = center.y - (0.65f* radius);
// buttonRect.x = center.x - (0.5f * buttonRect.width);
// for (uint i = 0; i < options.Length; ++i)
// {
// Rect textRect = new Rect(buttonRect);
// textRect.y = center.y - radius;
// textRect.height *= 0.5f;
// string text = options[i];
// GUI.matrix = Matrix4x4.identity;
// bool selected = false;
// int recognizerGroup = options.Length - 1;
// uint recognizerIndex = i;
//
//
// if (Fubi.getCurrentTime() - m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5
// || Fubi.getCurrentTime() - m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] < 0.5) // last recognition not longer than 0.5 seconds ago
// {
// GUIUtility.RotateAroundPivot(currRot, center);
// GUI.DrawTexture(buttonRect, m_swipeButtonActive, smode);
// Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y));
// textRect.x = newPos.x - textRect.width / 2.0f;
// textRect.y = newPos.y - textRect.height / 2.0f;
// GUI.matrix = Matrix4x4.identity;
// selected = GUI.Button(textRect, text, optionStyle.name+"-hover");
// }
// else // Display button also usable for mouse interaction
// {
// GUIUtility.RotateAroundPivot(currRot, center);
// GUI.DrawTexture(buttonRect, m_swipeButtonNormal, smode);
// Vector3 newPos = GUI.matrix.MultiplyPoint(new Vector3(textRect.center.x, textRect.center.y));
// textRect.x = newPos.x - textRect.width / 2.0f;
// textRect.y = newPos.y - textRect.height / 2.0f;
// GUI.matrix = Matrix4x4.identity;
// selected = GUI.Button(textRect, text, optionStyle);
// }
//
//
// // Check for full swipe
// // Of right hand
// if (!selected && m_currentUser > 0
// && withinInteractionFrame
// && Fubi.getCurrentTime() - m_lastSwipeRecognition > 1.0f) // at least one second between to swipes
// {
// selected = Fubi.getCombinationRecognitionProgressOn(m_swipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED;
// if (selected)
// {
// m_swipeMenuActive = false;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// m_lastSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastSwipeRecognition = Fubi.getCurrentTime();
// }
// else
// Fubi.enableCombinationRecognition(m_swipeRecognizers[options.Length - 1][i], m_currentUser, true);
// }
//
// // Or left hand
// if (!selected && m_currentUser > 0
// && withinInteractionFrame
// && Fubi.getCurrentTime() - m_lastLeftSwipeRecognition > 1.0f) // at least one second between to swipes
// {
// selected = Fubi.getCombinationRecognitionProgressOn(m_leftSwipeRecognizers[recognizerGroup][recognizerIndex], m_currentUser, false) == FubiUtils.RecognitionResult.RECOGNIZED;
// if (selected)
// {
// m_swipeMenuActive = false;
// Fubi.enableCombinationRecognition(FubiPredefinedGestures.Combinations.NUM_COMBINATIONS, m_currentUser, false);
// m_lastLeftSwipeRecognitions[recognizerGroup][recognizerIndex] = m_lastLeftSwipeRecognition = Fubi.getCurrentTime();
// }
// else
// Fubi.enableCombinationRecognition(m_leftSwipeRecognizers[options.Length - 1][i], m_currentUser, true);
// }
//
//
// if (selected)
// {
// selection = text;
// if (m_swipeSound && m_swipeSound.isReadyToPlay)
// AudioSource.PlayClipAtPoint(m_swipeSound, new Vector3(0,0,0), 1.0f);
// break;
// }
// currRot += rotAdd;
// }
// return selection;
// }
//
bool calibrateCursorMapping(uint id)
{
m_aspect = (float)Screen.width / (float)Screen.height;
if (id > 0)
{
FubiUtils.SkeletonJoint elbow = FubiUtils.SkeletonJoint.RIGHT_ELBOW;
FubiUtils.SkeletonJoint shoulder = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
FubiUtils.SkeletonJoint hand = FubiUtils.SkeletonJoint.RIGHT_HAND;
float confidence;
double timeStamp;
float elbowX, elbowY, elbowZ;
Fubi.getCurrentSkeletonJointPosition(id, elbow, out elbowX, out elbowY, out elbowZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float shoulderX, shoulderY, shoulderZ;
Fubi.getCurrentSkeletonJointPosition(id, shoulder, out shoulderX, out shoulderY, out shoulderZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist1 = Mathf.Sqrt(Mathf.Pow(elbowX - shoulderX, 2) + Mathf.Pow(elbowY - shoulderY, 2) + Mathf.Pow(elbowZ - shoulderZ, 2));
float handX, handY, handZ;
Fubi.getCurrentSkeletonJointPosition(id, hand, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
double dist2 = Mathf.Sqrt(Mathf.Pow(elbowX - handX, 2) + Mathf.Pow(elbowY - handY, 2) + Mathf.Pow(elbowZ - handZ, 2));
m_mapping.height = (float)(dist1 + dist2);
// Calculate all others in depence of maph
m_mapping.y = 200.0f / 550.0f * m_mapping.height;
m_mapping.width = m_mapping.height / m_aspect;
m_mapping.x = -100.0f / (550.0f / m_aspect) * m_mapping.width;
//Debug.Log ("m_mapping.x=" + m_mapping.x + "m_mapping.y=" + m_mapping.y + "m_mapping.height=" + m_mapping.height + "m_mapping.width=" + m_mapping.width);
return(true);
}
}
}
}
return(false);
}
// Update is called once per frame
void Update()
{
// update FUBI
Fubi.updateSensor();
// // AA: Collision detection for our game object
// // Convert from GUI coordinates (Top-left: 0,0 to Bottom-right ) to Screen coordinates ((Bottom-left: 0,0 to Top-right )):
// // Screen-coordinate.y = Screen.height - GUI-coordinate.y;
// Ray ray = Camera.mainCamera.ScreenPointToRay(new Vector2(m_absPixelPosition.x, Screen.height - m_absPixelPosition.y));
//
// // check for underlying objects
// RaycastHit hit;
// if(Physics.Raycast(ray, out hit))
// {
// foreach(GameObject obj in MyGameObjects)
// {
// if(hit.collider.gameObject == obj)
// {
//
// if (clickRecognized()) {
// // an object was hit by the ray. Color it
// if(obj.renderer.material.color == Color.red) {
// obj.renderer.material.color = Color.blue;
// break;
// }
// else {
// obj.renderer.material.color = Color.red;
// break;
// }
// }
// }
// }
// }
//
// update flags if gestures or the swipe menu have been displayed
if (m_gesturesDisplayed)
{
m_gesturesDisplayedLastFrame = true;
m_gesturesDisplayed = false;
}
else
{
m_gesturesDisplayedLastFrame = false;
}
if (m_swipeMenuDisplayed)
{
m_swipeMenuDisplayedLastFrame = true;
m_swipeMenuDisplayed = false;
}
else
{
m_swipeMenuDisplayedLastFrame = false;
}
// Render tracking image
// AA: This
if (!m_disableTrackingImage && (!m_disableTrackingImageWithSwipeMenu || !m_swipeMenuDisplayedLastFrame))
{
// uint renderOptions = (uint)(FubiUtils.RenderOptions.Default | FubiUtils.RenderOptions.DetailedFaceShapes | FubiUtils.RenderOptions.FingerShapes);
// Fubi.getImage(m_rawImage, FubiUtils.ImageType.Depth, FubiUtils.ImageNumChannels.C4, FubiUtils.ImageDepth.D8, renderOptions);
// Updatem_depthMapTexture();
}
}
private void updateStats()
{
//Check all users
var numUsers = Fubi.getNumUsers();
var numHands = Fubi.getNumHands();
if (numUsers == 0 && numHands == 0)
{
warnLabel.Visibility = Visibility.Visible;
}
else
{
warnLabel.Visibility = Visibility.Hidden;
}
int currentIndex = 0;
for (uint i = 0; i < numUsers + 1; i++)
{
var id = (i == numUsers && Fubi.isPlayingSkeletonData()) ? FubiUtils.PlaybackUserID : Fubi.getUserID(i);
if (id > 0)
{
// Not existent yet
if (currentIndex >= statsTree.Items.Count)
{
statsTree.Items.Add(new TvUser());
((TvUser)statsTree.Items[currentIndex]).IsExpanded = true;
}
// Wrong type, i.e. TvHand instead of TvUser
if (statsTree.Items[currentIndex].GetType() != typeof(TvUser))
{
statsTree.Items[currentIndex] = new TvUser();
((TvUser)statsTree.Items[currentIndex]).IsExpanded = true;
}
var tUser = (TvUser)statsTree.Items[currentIndex];
tUser.id = id;
// Update user defined combinations
var numRecs = Fubi.getNumUserDefinedCombinationRecognizers();
uint actualRecs = 0;
for (uint pc = 0; pc < numRecs; ++pc)
{
var name = Fubi.getUserDefinedCombinationRecognizerName(pc);
while (actualRecs >= tUser.Recs.Count)
{
tUser.Recs.Add(new TvRec());
}
var rec = tUser.Recs[(int)actualRecs];
rec.id = pc;
rec.name = name;
uint numStates;
bool isInterrupted, isInTransition;
rec.currState = Fubi.getCurrentCombinationRecognitionState(name, id, out numStates, out isInterrupted, out isInTransition) + 1;
rec.numStates = numStates;
rec.isInterrupted = isInterrupted;
rec.isInTransition = isInTransition;
if (Recognitions.ContainsKey(id) && Recognitions[id].ContainsKey(name) && Fubi.getCurrentTime() - Recognitions[id][name] < 2.0)
{
rec.bgColor = "LightGreen";
}
else
{
rec.bgColor = Color.Transparent.Name;
}
if (Hints.ContainsKey(id) && Hints[id].ContainsKey(name))
{
rec.hint = Hints[id][name];
}
actualRecs++;
}
while (tUser.Recs.Count > actualRecs)
{
tUser.Recs.RemoveAt(tUser.Recs.Count - 1);
}
++currentIndex;
}
}
for (uint i = 0; i < numHands + 1; i++)
{
var id = (i == numHands && Fubi.isPlayingSkeletonData()) ? FubiUtils.PlaybackHandID : Fubi.getHandID(i);
if (id > 0)
{
if (currentIndex >= statsTree.Items.Count)
{
statsTree.Items.Add(new TvHand());
((TvHand)statsTree.Items[currentIndex]).IsExpanded = true;
}
// Wrong type, i.e. TvUser instead of TvUHand
if (statsTree.Items[currentIndex].GetType() != typeof(TvHand))
{
statsTree.Items[currentIndex] = new TvHand();
((TvHand)statsTree.Items[currentIndex]).IsExpanded = true;
}
var tHand = (TvHand)statsTree.Items[currentIndex];
tHand.id = id;
// Update combinations
var numRecs = Fubi.getNumUserDefinedCombinationRecognizers();
uint actualRecs = 0;
for (uint pc = 0; pc < numRecs; ++pc)
{
var name = Fubi.getUserDefinedCombinationRecognizerName(pc);
while (actualRecs >= tHand.Recs.Count)
{
tHand.Recs.Add(new TvRec());
}
var rec = tHand.Recs[(int)actualRecs];
rec.id = pc;
rec.name = name;
uint numStates;
bool isInterrupted, isInTransition;
rec.currState = Fubi.getCurrentCombinationRecognitionStateForHand(name, id, out numStates, out isInterrupted, out isInTransition) + 1;
rec.numStates = numStates;
rec.isInterrupted = isInterrupted;
rec.isInTransition = isInTransition;
if (HandRecognitions.ContainsKey(id) && HandRecognitions[id].ContainsKey(name) && Fubi.getCurrentTime() - HandRecognitions[id][name] < 2.0)
{
rec.bgColor = "LightGreen";
}
else
{
rec.bgColor = Color.Transparent.Name;
}
if (HandHints.ContainsKey(id) && HandHints[id].ContainsKey(name))
{
rec.hint = HandHints[id][name];
}
actualRecs++;
}
while (tHand.Recs.Count > actualRecs)
{
tHand.Recs.RemoveAt(tHand.Recs.Count - 1);
}
++currentIndex;
}
}
while (statsTree.Items.Count > currentIndex)
{
statsTree.Items.RemoveAt(statsTree.Items.Count - 1);
}
}
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false)
{
x = float.NaN;
y = float.NaN;
float handX, handY, handZ, confidence;
FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
if (leftHand)
{
joint = FubiUtils.SkeletonJoint.LEFT_HAND;
relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
}
double timeStamp;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// Take relative coordinates
float rawX = handX - relX;
float rawY = handY - relY;
// Convert to screen coordinates
float newX, newY;
float mapX = m_mapX;
if (leftHand)
{
// Mirror x area for left hand
mapX = -m_mapX - m_mapW;
}
newX = (rawX - mapX) / m_mapW;
newY = (m_mapY - rawY) / m_mapH; // Flip y for the screen coordinates
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
float changeX = newX - m_x;
float changeY = newY - m_y;
if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp)
{
double changeLength = Math.Sqrt(changeX * changeX + changeY * changeY);
double changeDur = timeStamp - m_timeStamp;
float filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor;
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
m_x = (1.0f - filterFactor) * m_x + filterFactor * newX;
m_y = (1.0f - filterFactor) * m_y + filterFactor * newY;
m_timeStamp = timeStamp;
// Send it
MouseSimulator.sendMousePos(m_x, m_y);
x = m_x;
y = m_x;
}
// Check for mouse click
if (Fubi.recognizeGestureOn("mouseClick", userID) == FubiUtils.RecognitionResult.RECOGNIZED ||
Fubi.recognizeGestureOn("mouseClick1", userID) == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (m_timeStamp - m_lastMouseClick > 1)
{
MouseSimulator.sendMouseButton(true, true);
MouseSimulator.sendMouseButton(true, false);
m_lastMouseClick = m_timeStamp;
}
}
}
}
}
private void releaseFubi()
{
Fubi.release();
}
private void fubiMain(object filterOptions)
{
m_renderOptions = 0;
m_renderOptions |= (int)FubiUtils.RenderOptions.Shapes;
//m_renderOptions |= (int)FubiUtils.RenderOptions.Skeletons;
//m_renderOptions |= (int)FubiUtils.RenderOptions.UserCaptions;
//m_renderOptions |= (int)FubiUtils.RenderOptions.Background;
//m_renderOptions |= (int)FubiUtils.RenderOptions.SwapRAndB;
// m_renderOptions |= (int)FubiUtils.RenderOptions.DetailedFaceShapes;
//m_renderOptions |= (int)FubiUtils.RenderOptions.BodyMeasurements;
// m_renderOptions |= (int)FubiUtils.RenderOptions.GlobalOrientCaptions; // or
//m_renderOptions |= (int)FubiUtils.RenderOptions.LocalOrientCaptions;
// m_renderOptions |= (int)FubiUtils.RenderOptions.GlobalPosCaptions; // or
//m_renderOptions |= (int)FubiUtils.RenderOptions.LocalPosCaptions;
m_renderOptions |= (int)FubiUtils.RenderOptions.UseFilteredValues;
var rgbOptions = new FubiUtils.StreamOptions();
var irOptions = new FubiUtils.StreamOptions();
var depthOptions = new FubiUtils.StreamOptions();
var invalidOptions = new FubiUtils.StreamOptions(-1);
lock (LockFubiUpdate)
{
if (!Fubi.init(new FubiUtils.SensorOptions(depthOptions, rgbOptions, irOptions, m_selectedSensor),
(FubiUtils.FilterOptions)filterOptions))
{ // the following for when the init fails
m_selectedSensor = FubiUtils.SensorType.NONE;
Fubi.init(new FubiUtils.SensorOptions(depthOptions, invalidOptions, invalidOptions, m_selectedSensor),
(FubiUtils.FilterOptions)filterOptions);
Dispatcher.BeginInvoke(new TwoStringDelegate(showWarnMsg),
new object[]
{
"Error starting sensor! \nDid you connect the sensor and install the correct driver? \nTry selecting a different sensor.",
"Error starting sensor"
});
return;
}
Fubi.getDepthResolution(out m_width, out m_height);
m_numChannels = FubiUtils.ImageNumChannels.C4;
// All known combination recognizers will be started automatically for new users
Fubi.setAutoStartCombinationRecognition(true);
// Load XML with sample mouse control gestures
if (Fubi.loadRecognizersFromXML("TutorialRecognizers.xml"))
{
// // This requires to update the gesture list used for selecting key/button bindings and for xml generation
// Dispatcher.BeginInvoke(new NoArgDelegate(refreshGestureList), null);
// // Now we can load the default bindings using the above recognizers
// Dispatcher.BeginInvoke(new OneStringDelegate(m_fubiMouseKeyboard.Bindings.loadFromXML), "KeyMouseBindings.xml");
}
}
Fubi.RecognitionStart += new Fubi.RecognitionHandler(recognitionStart);
Fubi.RecognitionEnd += new Fubi.RecognitionHandler(recognitionEnd);
DispatcherOperation currentOp = null;
videoFile.fileName = "trainingData/tempRecord2.vid";
videoFile.startPlayback();
while (m_running)
{
lock (LockFubiUpdate)
{
// Now update the sensors
Fubi.updateSensor();
Fubi.getImage(s_buffer2, FubiUtils.ImageType.Depth, m_numChannels, FubiUtils.ImageDepth.D8, m_renderOptions,
(int)FubiUtils.JointsToRender.ALL_JOINTS, m_selectedDepthMod);
}
// And trigger a GUI update event
currentOp = Dispatcher.BeginInvoke(new NoArgDelegate(updateGUI), null);
// Wait for the GUI update to finish
while (currentOp.Status != DispatcherOperationStatus.Completed &&
currentOp.Status != DispatcherOperationStatus.Aborted)
{
Thread.Sleep(5);
}
}
// Wait for the last GUI update to really have finished
while (currentOp != null && currentOp.Status != DispatcherOperationStatus.Completed &&
currentOp.Status != DispatcherOperationStatus.Aborted)
{
Thread.Sleep(2);
}
// Now we can release Fubi safely
Fubi.release();
}
private void updateFubi()
{
if (clearRecognizersOnNextUpdate)
{
Fubi.clearUserDefinedRecognizers();
if (Fubi.getNumUserDefinedCombinationRecognizers() == 0 && Fubi.getNumUserDefinedRecognizers() == 0)
{
button3.IsEnabled = false;
}
clearRecognizersOnNextUpdate = false;
}
if (switchSensorOnNextUpdate)
{
Fubi.switchSensor(new FubiUtils.SensorOptions(new FubiUtils.StreamOptions(), new FubiUtils.StreamOptions(), new FubiUtils.StreamOptions(-1, -1, -1),
(FubiUtils.SensorType)Enum.Parse(typeof(FubiUtils.SensorType), sensorSelectionComboBox.SelectedItem.ToString())));
switchSensorOnNextUpdate = false;
}
label1.Content = "User Count : " + Fubi.getNumUsers().ToString();
// Update Fubi and get the debug image
int width = 0, height = 0;
FubiUtils.ImageNumChannels channels = FubiUtils.ImageNumChannels.C4;
FubiUtils.ImageType type = FubiUtils.ImageType.Depth;
uint renderOptions = 0;
if (shapeCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.Shapes;
}
if (skeletonCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.Skeletons;
}
if (userCaptionscheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.UserCaptions;
}
if (localOrientCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.LocalOrientCaptions;
}
if (globalOrientCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.GlobalOrientCaptions;
}
if (localPosCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.LocalPosCaptions;
}
if (globalPosCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.GlobalPosCaptions;
}
if (backgroundCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.Background;
}
if (swapRAndBcheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.SwapRAndB;
}
if (fingerShapecheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.FingerShapes;
}
if (detailedFaceCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.DetailedFaceShapes;
}
if (bodyMeasuresCheckBox.IsChecked == true)
{
renderOptions |= (uint)FubiUtils.RenderOptions.BodyMeasurements;
}
FubiUtils.DepthImageModification depthMods = (FubiUtils.DepthImageModification)Enum.Parse(typeof(FubiUtils.DepthImageModification), comboBox1.SelectedItem.ToString(), true);
if (checkBox3.IsChecked == true)
{
Fubi.getRgbResolution(out width, out height);
channels = FubiUtils.ImageNumChannels.C3;
type = FubiUtils.ImageType.Color;
}
else
{
Fubi.getDepthResolution(out width, out height);
channels = FubiUtils.ImageNumChannels.C4;
type = FubiUtils.ImageType.Depth;
}
// Display the debug image
if (width > 0 && height > 0)
{
WriteableBitmap wb = image1.Source as WriteableBitmap;
if (wb != null && (wb.Width != width || wb.Height != height || wb.Format.BitsPerPixel != (int)channels * 8))
{
wb = null;
image1.Width = width;
image1.Height = height;
buffer = new byte[(int)channels * width * height];
}
if (wb == null)
{
PixelFormat format = PixelFormats.Bgra32;
if (channels == FubiUtils.ImageNumChannels.C3)
{
format = PixelFormats.Rgb24;
}
else if (channels == FubiUtils.ImageNumChannels.C1)
{
format = PixelFormats.Gray8;
}
wb = new WriteableBitmap(width, height, 0, 0, format, BitmapPalettes.Gray256);
image1.Source = wb;
}
Fubi.updateSensor();
Fubi.getImage(buffer, type, channels, FubiUtils.ImageDepth.D8, renderOptions, depthMods);
int stride = wb.PixelWidth * (wb.Format.BitsPerPixel / 8);
wb.WritePixels(new Int32Rect(0, 0, wb.PixelWidth, wb.PixelHeight), buffer, stride, 0);
}
//Check postures for all users
for (uint i = 0; i < Fubi.getNumUsers(); i++)
{
uint id = Fubi.getUserID(i);
if (id > 0)
{
bool changedSomething = false;
// Print postures
if (checkBox1.IsChecked == true)
{
// Only user defined postures
for (uint p = 0; p < Fubi.getNumUserDefinedRecognizers(); ++p)
{
if (Fubi.recognizeGestureOn(p, id) == FubiUtils.RecognitionResult.RECOGNIZED)
{
// Posture recognized
if (!currentPostures.ContainsKey(p) || !currentPostures[p])
{
// Posture start
textBox1.Text += "User" + id + ": START OF " + Fubi.getUserDefinedRecognizerName(p) + " -->\n";
currentPostures[p] = true;
changedSomething = true;
}
}
else if (currentPostures.ContainsKey(p) && currentPostures[p])
{
// Posture end
textBox1.Text += "User" + id + ": --> END OF " + Fubi.getUserDefinedRecognizerName(p) + "\n";
currentPostures[p] = false;
changedSomething = true;
}
}
if (PredefinedCheckBox.IsChecked == true)
{
for (int p = 0; p < (int)FubiPredefinedGestures.Postures.NUM_POSTURES; ++p)
{
if (Fubi.recognizeGestureOn((FubiPredefinedGestures.Postures)p, id) == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (!currentPostures1[p])
{
// Posture recognized
textBox1.Text += "User" + id + ": START OF" + FubiPredefinedGestures.getPostureName((FubiPredefinedGestures.Postures)p) + "\n";
currentPostures1[p] = true;
changedSomething = true;
}
}
else if (currentPostures1[p])
{
textBox1.Text += "User" + id + ": --> END OF " + FubiPredefinedGestures.getPostureName((FubiPredefinedGestures.Postures)p) + "\n";
currentPostures1[p] = false;
changedSomething = true;
}
}
}
if (changedSomething)
{
textBox1.ScrollToEnd();
}
}
// Print combinations
for (uint pc = 0; pc < Fubi.getNumUserDefinedCombinationRecognizers(); ++pc)
{
// Only user defined postures
if (checkBox2.IsChecked == true)
{
if (Fubi.getCombinationRecognitionProgressOn(Fubi.getUserDefinedCombinationRecognizerName(pc), id) == FubiUtils.RecognitionResult.RECOGNIZED)
{
// Posture recognized
textBox2.Text += "User" + id + ": " + Fubi.getUserDefinedCombinationRecognizerName(pc) + "\n";
}
else
{
Fubi.enableCombinationRecognition(Fubi.getUserDefinedCombinationRecognizerName(pc), id, true);
}
}
//else
// Fubi.enableCombinationRecognition(Fubi.getUserDefinedCombinationRecognizerName(pc), id, false);
}
for (uint pc = 0; pc < (uint)FubiPredefinedGestures.Combinations.NUM_COMBINATIONS; ++pc)
{
if (checkBox2.IsChecked == true && PredefinedCheckBox.IsChecked == true)
{
if (Fubi.getCombinationRecognitionProgressOn((FubiPredefinedGestures.Combinations)pc, id) == FubiUtils.RecognitionResult.RECOGNIZED)
{
// Posture recognized
textBox2.Text += "User" + id + ": " + FubiPredefinedGestures.getCombinationName((FubiPredefinedGestures.Combinations)pc) + "\n";
}
else
{
Fubi.enableCombinationRecognition((FubiPredefinedGestures.Combinations)pc, id, true);
}
}
//else
// Fubi.enableCombinationRecognition((FubiPredefinedGestures.Combinations)pc, id, false);
}
if (checkBox2.IsChecked == true)
{
textBox2.ScrollToEnd();
}
}
}
uint closestId = Fubi.getClosestUserID();
if (closestId > 0)
{
// For printing the user orientation
//float[] mat = new float[9];
//float confidence;
//double timeStamp;
//Fubi.getCurrentSkeletonJointOrientation(closestId, FubiUtils.SkeletonJoint.Torso, mat, out confidence, out timeStamp);
//float rx, ry, rz;
//FubiUtils.Math.rotMatToRotation(mat, out rx, out ry, out rz);
//label1.Content = "UserOrient:" + String.Format("{0:0.#}", rx) + "/" + String.Format("{0:0.#}", ry) + "/" + String.Format("{0:0.#}", rz);
if (controlMouse)
{
float x, y;
FubiMouse.applyHandPositionToMouse(closestId, out x, out y, leftHandRadioButton.IsChecked == true);
label2.Content = "X:" + x + " Y:" + y;
}
if (checkBox4.IsChecked == true)
{
FubiPredefinedGestures.Combinations activationGesture = FubiPredefinedGestures.Combinations.WAVE_RIGHT_HAND_OVER_SHOULDER;
// TODO use left hand waving
if (Fubi.getCombinationRecognitionProgressOn(activationGesture, closestId, false) == FubiUtils.RecognitionResult.RECOGNIZED)
{
if (controlMouse)
{
stopMouseEmulation();
}
else
{
startMouseEmulation();
}
}
else
{
Fubi.enableCombinationRecognition(activationGesture, closestId, true);
}
}
}
}
private FubiUtils.RecognitionResult recognizeGesture(string recognizerName, uint targetID)
{
return(UseHand
? Fubi.recognizeGestureOnHand(recognizerName, targetID)
: Fubi.recognizeGestureOn(recognizerName, targetID));
}
private void button4_Click(object sender, RoutedEventArgs e)
{
Fubi.resetTracking();
}
public void generateXML(MainWindow w, CancellationToken ct)
{
// Important for corrrectly converting numbers to strings
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
RecognizerXMLGenerator recognizerGen = null;
switch (Type)
{
case RecognizerType.JointRelation:
case RecognizerType.JointOrientation:
recognizerGen = new StaticPostureXmlGenerator(RecognizerName, w, RecOptions, Type);
break;
case RecognizerType.LinearMovement:
recognizerGen = new LinearMovementXmlGenerator(RecognizerName, w, RecOptions, Type);
break;
case RecognizerType.AngularMovement:
recognizerGen = new AngularMovementXmlGenerator(RecognizerName, w, RecOptions, Type);
break;
case RecognizerType.FingerCount:
recognizerGen = new FingerCountXmlGenerator(RecognizerName, w, RecOptions, Type);
break;
case RecognizerType.TemplateRecording:
recognizerGen = new TemplateRecordingXMLGenerator(RecognizerName, w, RecOptions);
break;
case RecognizerType.Combination:
recognizerGen = new CombinationXMLGenerator(RecognizerName, w, CombOptions);
break;
}
if (recognizerGen != null)
{
recognizerGen.UseHand = UseHand;
recognizerGen.trainValues(Start, Duration, ct);
if (!ct.IsCancellationRequested)
{
var trainedFromPlaybackUser = Fubi.isPlayingSkeletonData() || Type == RecognizerType.TemplateRecording || StartedWithPlayback;
if (recognizerGen.generateAndSaveXML() && trainedFromPlaybackUser && !String.IsNullOrEmpty(RecOptions.PlaybackFile))
{
// Save start and end markers of the file in case of success
var file = new XmlDocument();
file.Load(RecOptions.PlaybackFile);
var currentFubiRecNode = file.GetElementsByTagName("FubiRecording")[0] as XmlElement;
if (currentFubiRecNode != null)
{
currentFubiRecNode.SetAttribute("startMarker", RecOptions.PlaybackStart.ToString());
currentFubiRecNode.SetAttribute("endMarker", RecOptions.PlaybackEnd.ToString());
file.Save(RecOptions.PlaybackFile);
}
}
}
}
w.Dispatcher.BeginInvoke((Action)(() =>
{
w.menuTabCtrl.IsEnabled = true;
}));
}
// Initialization
void Start()
{
//AA: Filter visalization related initializations
filter = new Filter();
fv = new FilterVisualization();
fm = new FilterManager();
fv.filterOutputLocX = Screen.width / 3 - 50;
fv.Initialise();
fv.DrawCircle();
m_colorTextureDictionary = new Dictionary <string, Texture2D>();
foreach (Texture2D tex in m_colorTextures)
{
m_colorTextureDictionary.Add(tex.name, tex);
}
LoadFilters();
// First set instance so Fubi.release will not be called while destroying old objects
instance = this;
// Remain this instance active until new one is created
DontDestroyOnLoad(this);
// Destroy old instance of Fubi
object[] objects = GameObject.FindObjectsOfType(typeof(FubiUnity));
if (objects.Length > 1)
{
Destroy(((FubiUnity)objects[0]));
}
m_lastMouseClick = 0;
m_lastGesture = 0;
// Init FUBI
if (!m_disableFubi)
{
// Only init if not already done
if (!Fubi.isInitialized())
{
Fubi.init(new FubiUtils.SensorOptions(new FubiUtils.StreamOptions(640, 480, 30), new FubiUtils.StreamOptions(640, 480, 30),
new FubiUtils.StreamOptions(-1, -1, -1), FubiUtils.SensorType.OPENNI2), new FubiUtils.FilterOptions());
if (!Fubi.isInitialized())
{
Debug.Log("Fubi: FAILED to initialize Fubi!");
}
else
{
Debug.Log("Fubi: initialized!");
}
}
}
else
{
m_disableTrackingImage = true;
}
// Initialize debug image
// m_depthMapTexture = new Texture2D((int)(m_xRes / m_factor), (int)(m_yRes / m_factor), TextureFormat.RGBA32, false);
// m_depthMapPixels = new Color[(int)((m_xRes / m_factor) * (m_yRes / m_factor))];
// m_rawImage = new byte[(int)(m_xRes * m_yRes * 4)];
m_userImageTexture = null;
// Disable system cursor
if (m_defaultCursor != null && m_disableFubi == false)
{
Screen.showCursor = false;
}
else
{
Screen.showCursor = true;
}
// Default mapping values
m_mapping.x = -100.0f;
m_mapping.y = 200.0f;
m_mapping.height = 550.0f;
// Get screen aspect
m_aspect = (float)Screen.width / (float)Screen.height;
// Calculated Map width with aspect
m_mapping.width = m_mapping.height / m_aspect;
if (Fubi.isInitialized())
{
// Clear old gesture recognizers
Fubi.clearUserDefinedRecognizers();
// And (re)load them
if (Fubi.loadRecognizersFromXML("UnitySampleRecognizers.xml"))
{
Debug.Log("Fubi: gesture recognizers 'BarRecognizers.xml' loaded!");
}
else
{
Debug.Log("Fubi: loading XML recognizers failed!");
}
// load mouse control recognizers
if (Fubi.loadRecognizersFromXML("MouseControlRecognizers.xml"))
{
Debug.Log("Fubi: mouse control recognizers loaded!");
}
else
{
Debug.Log("Fubi: loading mouse control recognizers failed!");
}
}
}
public void applyHandPositionToMouse(uint userID, out float x, out float y, bool leftHand = false)
{
x = float.NaN;
y = float.NaN;
float handX, handY, handZ, confidence;
var joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
var relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
if (leftHand)
{
joint = FubiUtils.SkeletonJoint.LEFT_HAND;
relJoint = FubiUtils.SkeletonJoint.LEFT_SHOULDER;
}
double timeStamp;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// Take relative coordinates
var rawX = handX - relX;
var rawY = handY - relY;
// Convert to screen coordinates
var mapX = m_mapX;
if (leftHand)
{
// Mirror x area for left hand
mapX = -m_mapX - m_mapW;
}
float newX = (rawX - mapX) / m_mapW;
float newY = (m_mapY - rawY) / m_mapH;
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
var changeX = newX - m_x;
var changeY = newY - m_y;
if (Math.Abs(changeX) > float.Epsilon || Math.Abs(changeY) > float.Epsilon && Math.Abs(timeStamp - m_timeStamp) > float.Epsilon)
{
var changeLength = Math.Sqrt(changeX * changeX + changeY * changeY);
var filterFactor = (float)Math.Sqrt(changeLength) * m_smoothingFactor;
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
m_x = (1.0f - filterFactor) * m_x + filterFactor * newX;
m_y = (1.0f - filterFactor) * m_y + filterFactor * newY;
m_timeStamp = timeStamp;
// Send it
MouseKeyboardSimulator.sendMousePos(m_x, m_y);
x = m_x;
y = m_x;
}
}
}
}
// Called for rendering the gui
void OnGUI()
{
// AA: Position the depth image so the user can see the kinect output
if (!m_disableTrackingImage && (!m_disableTrackingImageWithSwipeMenu || !m_swipeMenuDisplayedLastFrame))
{
// Debug image
GUI.depth = -4;
GUI.DrawTexture(new Rect(25, Screen.height - m_yRes / m_factor - 25, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture);
//GUI.DrawTexture(new Rect(Screen.width-m_xRes/m_factor, Screen.height-m_yRes/m_factor, m_xRes / m_factor, m_yRes / m_factor), m_depthMapTexture);
}
//AA: add the GUI elements
int shift = 42;
GUI.Box(new Rect(5 + shift, 25, Screen.width / 3 - 130, Screen.height - 50), "FILTERS");
m_bUseSimpleAverage = GUI.Toggle(new Rect(45 + shift, 50, 200, 30), m_bUseSimpleAverage, " SIMPLE AVERAGE 10");
m_bUseMovingAverage = GUI.Toggle(new Rect(45 + shift, 90, 200, 30), m_bUseMovingAverage, " MOVING AVERAGE");
m_bUseSimpleAverage5 = GUI.Toggle(new Rect(45 + shift, 130, 200, 30), m_bUseSimpleAverage5, " SIMPLE AVERAGE 5");
m_bDblMovingAverage = GUI.Toggle(new Rect(45 + shift, 170, 200, 30), m_bDblMovingAverage, " DOUBLE MOV AVERAGE");
m_bUseExpSmoothing = GUI.Toggle(new Rect(45 + shift, 210, 200, 30), m_bUseExpSmoothing, " EXP SMOOTHING");
m_bUseDblExpSmoothing = GUI.Toggle(new Rect(45 + shift, 250, 200, 30), m_bUseDblExpSmoothing, " DOUBLE EXP SMOOTHING");
m_bUseAdaptive = GUI.Toggle(new Rect(45 + shift, 290, 200, 30), m_bUseAdaptive, " ADAPTIVE DBL EXP");
m_bUseMedian = GUI.Toggle(new Rect(45 + shift, 330, 200, 30), m_bUseMedian, " MEDIAN");
m_bUseCombination1 = GUI.Toggle(new Rect(45 + shift, 370, 200, 30), m_bUseCombination1, " SIMPLE AVG + Median");
m_bUseCombination2 = GUI.Toggle(new Rect(45 + shift, 410, 200, 30), m_bUseCombination2, " DBL MOV AVG + Median");
m_bUseNone = GUI.Toggle(new Rect(45 + shift, 450, 200, 30), m_bUseNone, " NONE");
if (GUI.Button(new Rect(50 + shift, Screen.height - 75, 150, 30), "Clear"))
{
WriteableAreaResize();
fv.DrawCircle();
}
// If some button has been pressed OR this is the first exection
if (GUI.changed)
{
LoadFilters();
if (fm.filters.Count == 1)
{
m_principalCursor = 0;
}
}
int count = 0;
if (m_bUseSimpleAverage)
{
GUI.Label(new Rect(15 + shift, 45, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseMovingAverage)
{
GUI.Label(new Rect(15 + shift, 85, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseSimpleAverage5)
{
GUI.Label(new Rect(15 + shift, 125, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bDblMovingAverage)
{
GUI.Label(new Rect(15 + shift, 165, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseExpSmoothing)
{
GUI.Label(new Rect(15 + shift, 205, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseDblExpSmoothing)
{
GUI.Label(new Rect(15 + shift, 245, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseAdaptive)
{
GUI.Label(new Rect(15 + shift, 285, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseMedian)
{
GUI.Label(new Rect(15 + shift, 325, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseCombination1)
{
GUI.Label(new Rect(15 + shift, 365, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseCombination2)
{
GUI.Label(new Rect(15 + shift, 405, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (m_bUseNone)
{
GUI.Label(new Rect(15 + shift, 445, 30, 30), m_colorTextureDictionary[fm.colors[count].ToString()]);
count++;
}
if (prevScreenWidth != Screen.width || prevScreenHeight != Screen.height)
{
// Resize writeable area, redraw the circle
WriteableAreaResize();
prevScreenWidth = Screen.width;
prevScreenHeight = Screen.height;
fv.DrawCircle();
}
//AA: Draw the writeable area
fv.Apply();
for (int i = 0; i < fv.m_filterOutputTexture.Count; i++)
{
GUI.DrawTexture(new Rect(fv.filterOutputLocX, fv.filterOutputLocY, fv.filterOutputWidth, fv.filterOutputHeight), fv.m_filterOutputTexture[i]);
}
// Cursor
m_gotNewFubiCoordinates = false;
if (Fubi.isInitialized())
{
// Take closest user
uint userID = Fubi.getClosestUserID();
if (userID != m_currentUser)
{
m_currentUser = userID;
m_lastCalibrationSucceded = false;
}
if (userID > 0)
{
if (!m_lastCalibrationSucceded)
{
m_lastCalibrationSucceded = calibrateCursorMapping(m_currentUser);
}
FubiUtils.SkeletonJoint joint = FubiUtils.SkeletonJoint.RIGHT_HAND;
FubiUtils.SkeletonJoint relJoint = FubiUtils.SkeletonJoint.RIGHT_SHOULDER;
// Get hand and shoulder position and check their confidence
double timeStamp;
float handX, handY, handZ, confidence;
Fubi.getCurrentSkeletonJointPosition(userID, joint, out handX, out handY, out handZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
float relX, relY, relZ;
Fubi.getCurrentSkeletonJointPosition(userID, relJoint, out relX, out relY, out relZ, out confidence, out timeStamp);
if (confidence > 0.5f)
{
// AA: Filtering should happen here for the hand and relative joints separately
// If true, use the smoothed joints for calculating screen coordinates
fm.UpdateJointFilters(new Vector3(handX, handY, handZ), new Vector3(relX, relY, relZ));
for (int i = 0; i < fm.filters.Count; i++)
{
if (m_bUseJointFiltering)
{
//Debug.Log ("Prehand " + new Vector3(handX, handY, handZ) + " relJoint " + new Vector3(relX, relY, relZ));
Vector3 handPos = fm.joints[i]; // filter.Update(new Vector3(handX, handY, handZ), Filter.JOINT_TYPE.JOINT);
Vector3 relJointPos = fm.relativeJoints[i]; //filter.Update(new Vector3(relX, relY, relZ), Filter.JOINT_TYPE.RELATIVEJOINT);
//Debug.Log ("hand " + handPos + " relJoint " + relJointPos);
handZ = handPos.z;
handY = handPos.y;
handX = handPos.x;
relZ = relJointPos.z;
relY = relJointPos.y;
relX = relJointPos.x;
m_relativeCursorPosition = fm.relativeCursorPosition[i];
}
// AA: End
// Take relative coordinates
float zDiff = handZ - relZ;
float yDiff = handY - relY;
float xDiff = handX - relX;
// Check if hand is enough in front of shoulder
if ((yDiff > 0 && zDiff < -150.0f) || (Mathf.Abs(xDiff) > 150.0f && zDiff < -175.0f) || zDiff < -225.0f)
{
// Now get the possible cursor position
// Convert to screen coordinates
float newX, newY;
float mapX = m_mapping.x;
newX = (xDiff - mapX) / m_mapping.width;
newY = (m_mapping.y - yDiff) / m_mapping.height; // Flip y for the screen coordinates
// Filtering
// New coordinate is weighted more if it represents a longer distance change
// This should reduce the lagging of the cursor on higher distances, but still filter out small jittering
float changeX = newX - m_relativeCursorPosition.x;
float changeY = newY - m_relativeCursorPosition.y;
if (changeX != 0 || changeY != 0 && timeStamp != m_timeStamp)
{
float changeLength = Mathf.Sqrt(changeX * changeX + changeY * changeY);
float filterFactor = changeLength; //Mathf.Sqrt(changeLength);
if (filterFactor > 1.0f)
{
filterFactor = 1.0f;
}
// Apply the tracking to the current position with the given filter factor
// AA: Filtering should happen here for joint-to-relativejoint (VECTOR) filtering
// AA: filtering code
Vector2 tempNew = new Vector2(newX, newY);
fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor);
// If true, use the calculated factor for smoothing, else just use the new
if (m_bUseVectorFiltering)
{
m_relativeCursorPosition = fm.vectors[i]; //filter.Update(m_relativeCursorPosition, tempNew, filterFactor);
}
else // Just give equal weight to both
{
m_relativeCursorPosition = filter.Update(m_relativeCursorPosition, tempNew, 0.5f);
}
// AA: Calculate all filters
// fm.UpdateVectorFilters(m_relativeCursorPosition, tempNew, filterFactor);
m_timeStamp = timeStamp;
// Send it, but only if it is more or less within the screen
if (m_relativeCursorPosition.x > -0.1f && m_relativeCursorPosition.x < 1.1f &&
m_relativeCursorPosition.y > -0.1f && m_relativeCursorPosition.y < 1.1f)
{
MoveMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, i);
// Each filter must store it's own value of relative position, absolute and previous absolute positions
fm.relativeCursorPosition[i] = m_relativeCursorPosition;
fm.absPixelPosition[i] = m_absPixelPosition;
fm.prevAbsPixelPosition[i] = m_previousAbsPixelPosition;
DrawFilterOutputs(i);
m_gotNewFubiCoordinates = true;
m_lastCursorChangeDoneByFubi = true;
}
}
}
}
}
}
}
}
// AA: FUBI does not move mouse if the confidence value is too low
if (!m_gotNewFubiCoordinates) // AA: this only executes when input is coming from mouse
{
// Got no mouse coordinates from fubi this frame
Vector2 mousePos = Input.mousePosition;
// Only move mouse if it wasn't changed by fubi the last time or or it really has changed
if (!m_lastCursorChangeDoneByFubi || mousePos != m_lastMousePos)
{
//AA: Old code for cursor placement
m_relativeCursorPosition.x = mousePos.x / (float)Screen.width;
m_relativeCursorPosition.y = 1.0f - (mousePos.y / (float)Screen.height);
// Get mouse X and Y position as a percentage of screen width and height
MoveActualMouse(m_relativeCursorPosition.x, m_relativeCursorPosition.y, true);
m_lastMousePos = mousePos;
m_lastCursorChangeDoneByFubi = false;
}
}
}