void Update()
{
if (isVisualGestureInitialized)
{
KinectManager kinectManager = KinectManager.Instance;
for (int i = 0; i < kinectManager.GetUsersCount(); i++)
{
if (kinectManager && kinectManager.IsInitialized())
{
primaryUserID = kinectManager.GetUserIdByIndex(i);
}
// update visual gesture tracking
if (UpdateVisualGestures(primaryUserID))
{
// process the gestures
//foreach (string gestureName in gestureNames)
//{
if (gestureData.ContainsKey(gestureNames[0]))
{
VisualGestureData data = gestureData[gestureNames[0]];
Debug.Log(gestureNames[0] + "==" + data.userId + "==" + data.isProcessed + "==" + data.isComplete + "==" + data.confidence);
if (data.userId != 0 && !data.isProcessed && data.isComplete && data.confidence >= minConfidence)
{
//Debug.Log(gestureName + "-gesture detected.");
int userIndex = kinectManager ? kinectManager.GetUserIndexById(data.userId) : -1;
foreach (VisualGestureListenerInterface listener in visualGestureListeners)
{
if (listener.GestureCompleted(data.userId, userIndex, data.gestureName, data.confidence))
{
//data.isResetting = true;
data.isProcessed = true;
gestureData[gestureNames[0]] = data;
}
}
}
else if (data.userId != 0 && !data.isProcessed && data.progress >= minConfidence)
{
//Debug.Log(gestureName + "-gesture progres: " + data.progress);
int userIndex = kinectManager ? kinectManager.GetUserIndexById(data.userId) : -1;
foreach (VisualGestureListenerInterface listener in visualGestureListeners)
{
listener.GestureInProgress(data.userId, userIndex, data.gestureName, data.progress);
//data.isResetting = true;
data.isProcessed = true;
gestureData[gestureNames[0]] = data;
}
}
}
//}
}
}
}
}
/// <summary>
/// Gets the progress of the specified continuous gesture, in range [0, 1].
/// </summary>
/// <returns>The gesture progress.</returns>
/// <param name="gestureName">Gesture name</param>
public float GetGestureProgress(string gestureName)
{
if (gestureNames.Contains(gestureName))
{
VisualGestureData data = gestureData[gestureName];
if (data.userId != 0 && data.isContinuous)
{
return(data.progress);
}
}
return(0f);
}
/// <summary>
/// Gets the confidence of the specified discrete gesture, in range [0, 1].
/// </summary>
/// <returns>The gesture confidence.</returns>
/// <param name="gestureName">Gesture name</param>
public float GetGestureConfidence(string gestureName)
{
if (gestureNames.Contains(gestureName))
{
VisualGestureData data = gestureData[gestureName];
if (data.userId != 0 && data.isDiscrete)
{
return(data.confidence);
}
}
return(0f);
}
/// <summary>
/// Resets the gesture state.
/// </summary>
/// <returns><c>true</c>, if gesture state was reset, <c>false</c> if gesture was not found.</returns>
/// <param name="gestureName">Gesture name.</param>
public bool ResetGesture(string gestureName)
{
if (gestureNames.Contains(gestureName))
{
VisualGestureData data = gestureData[gestureName];
//data.isResetting = true;
data.isProcessed = true;
gestureData[gestureName] = data;
return(true);
}
return(false);
}
// gets all gesture dara as csv line
public string GetGestureDataAsCsv(char delimiter)
{
if (!isVisualGestureInitialized)
{
return(string.Empty);
}
// create the output string
StringBuilder sbBuf = new StringBuilder();
//const char delimiter = ',';
sbBuf.Append("vg").Append(delimiter);
sbBuf.Append(gestureNames.Count).Append(delimiter);
foreach (string gestureName in gestureNames)
{
VisualGestureData data = gestureData[gestureName];
if (data.userId != 0 && data.lastTimestamp != data.timestamp)
{
sbBuf.Append(data.userId).Append(delimiter);
sbBuf.AppendFormat("{0:F3}", data.timestamp).Append(delimiter);
sbBuf.Append(data.gestureName).Append(delimiter);
sbBuf.Append(data.isDiscrete ? 1 : 0).Append(delimiter);
sbBuf.Append(data.isContinuous ? 1 : 0).Append(delimiter);
sbBuf.Append(data.isComplete ? 1 : 0).Append(delimiter);
sbBuf.AppendFormat("{0:F3}", data.confidence).Append(delimiter);
sbBuf.AppendFormat("{0:F3}", data.progress).Append(delimiter);
data.lastTimestamp = data.timestamp;
gestureData[gestureName] = data;
}
else
{
sbBuf.Append(0).Append(delimiter);
}
}
// remove the last delimiter
if (sbBuf.Length > 0 && sbBuf[sbBuf.Length - 1] == delimiter)
{
sbBuf.Remove(sbBuf.Length - 1, 1);
}
return(sbBuf.ToString());
}
void Update()
{
if (isVisualGestureInitialized)
{
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
primaryUserID = kinectManager.GetUserIdByIndex(playerIndex);
}
// update visual gesture tracking
if (UpdateVisualGestures(primaryUserID))
{
// process the gestures
foreach (string gestureName in gestureNames)
{
if (gestureData.ContainsKey(gestureName))
{
VisualGestureData data = gestureData[gestureName];
if (data.isComplete && !data.isResetting && data.confidence >= minConfidence)
{
Debug.Log(gestureName + " detected.");
int userIndex = kinectManager ? kinectManager.GetUserIndexById(data.userId) : 0;
foreach (VisualGestureListenerInterface listener in visualGestureListeners)
{
if (listener.GestureCompleted(data.userId, userIndex, data.gestureName, data.confidence))
{
data.isResetting = true;
gestureData[gestureName] = data;
}
}
}
else if (data.progress >= 0.1f)
{
int userIndex = kinectManager ? kinectManager.GetUserIndexById(data.userId) : 0;
foreach (VisualGestureListenerInterface listener in visualGestureListeners)
{
listener.GestureInProgress(data.userId, userIndex, data.gestureName, data.progress);
}
}
}
}
}
}
}
/// <summary>
/// Determines whether the specified discrete gesture is completed.
/// </summary>
/// <returns><c>true</c> if the specified discrete gesture is completed; otherwise, <c>false</c>.</returns>
/// <param name="gestureName">Gesture name</param>
/// <param name="bResetOnComplete">If set to <c>true</c>, resets the gesture state.</param>
public bool IsGestureCompleted(string gestureName, bool bResetOnComplete)
{
if (gestureNames.Contains(gestureName))
{
VisualGestureData data = gestureData[gestureName];
if (data.isDiscrete && data.isComplete && !data.isResetting && data.confidence >= minConfidence)
{
if (bResetOnComplete)
{
data.isResetting = true;
gestureData[gestureName] = data;
}
return(true);
}
}
return(false);
}
/// <summary>
/// Determines whether the specified discrete gesture is completed.
/// </summary>
/// <returns><c>true</c> if the specified discrete gesture is completed; otherwise, <c>false</c>.</returns>
/// <param name="gestureName">Gesture name</param>
/// <param name="bResetOnComplete">If set to <c>true</c>, resets the gesture state.</param>
public bool IsGestureCompleted(string gestureName, bool bResetOnComplete)
{
if (gestureNames.Contains(gestureName))
{
VisualGestureData data = gestureData[gestureName];
if (data.userId != 0 && data.isDiscrete && data.isComplete && /**!data.isProcessed &&*/ data.confidence >= minConfidence)
{
if (bResetOnComplete)
{
//data.isResetting = true;
data.isProcessed = true;
gestureData[gestureName] = data;
}
return(true);
}
}
return(false);
}
private bool UpdateVisualGestures(long userId)
{
if (vgbFrameSource == null || vgbFrameReader == null)
{
return(false);
}
bool wasPaused = vgbFrameReader.IsPaused;
vgbFrameSource.TrackingId = (ulong)userId;
vgbFrameReader.IsPaused = (userId == 0);
if (vgbFrameReader.IsPaused)
{
if (!wasPaused)
{
// clear the gesture states
foreach (Gesture gesture in vgbFrameSource.Gestures)
{
if (gestureData.ContainsKey(gesture.Name))
{
VisualGestureData data = gestureData[gesture.Name];
data.userId = 0;
data.isComplete = false;
//data.isResetting = false;
data.isProcessed = false;
data.confidence = 0f;
data.progress = 0f;
data.timestamp = Time.realtimeSinceStartup;
gestureData[gesture.Name] = data;
}
}
}
return(false);
}
VisualGestureBuilderFrame frame = vgbFrameReader.CalculateAndAcquireLatestFrame();
if (frame != null)
{
Dictionary <Gesture, DiscreteGestureResult> discreteResults = frame.DiscreteGestureResults;
Dictionary <Gesture, ContinuousGestureResult> continuousResults = frame.ContinuousGestureResults;
if (discreteResults != null)
{
foreach (Gesture gesture in discreteResults.Keys)
{
if (gesture.GestureType == GestureType.Discrete && gestureData.ContainsKey(gesture.Name))
{
DiscreteGestureResult result = discreteResults[gesture];
VisualGestureData data = gestureData[gesture.Name];
data.userId = vgbFrameSource.IsTrackingIdValid ? (long)vgbFrameSource.TrackingId : 0;
data.isComplete = result.Detected;
data.confidence = result.Confidence;
data.timestamp = Time.realtimeSinceStartup;
//Debug.Log(string.Format ("{0} - {1}, confidence: {2:F0}%", data.gestureName, data.isComplete ? "Yes" : "No", data.confidence * 100f));
if (data.isProcessed && !data.isComplete)
{
//data.isResetting = false;
data.isProcessed = false;
}
gestureData[gesture.Name] = data;
}
}
}
if (continuousResults != null)
{
foreach (Gesture gesture in continuousResults.Keys)
{
if (gesture.GestureType == GestureType.Continuous && gestureData.ContainsKey(gesture.Name))
{
ContinuousGestureResult result = continuousResults[gesture];
VisualGestureData data = gestureData[gesture.Name];
data.userId = vgbFrameSource.IsTrackingIdValid ? (long)vgbFrameSource.TrackingId : 0;
float prevProgress = data.progress;
data.progress = result.Progress;
data.timestamp = Time.realtimeSinceStartup;
if (data.isProcessed && data.progress >= minConfidence && data.progress != prevProgress)
{
//data.isResetting = false;
data.isProcessed = false;
}
gestureData[gesture.Name] = data;
}
}
}
frame.Dispose();
frame = null;
}
return(true);
}
private bool InitVisualGestures()
{
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null;
Kinect2Interface kinectInterface = sensorData.sensorInterface as Kinect2Interface;
KinectSensor kinectSensor = kinectInterface != null ? kinectInterface.kinectSensor : null;
if (kinectSensor == null)
{
return(false);
}
if (gestureDatabase == string.Empty)
{
Debug.LogError("Please specify gesture database file!");
return(false);
}
// copy the gesture database file from Resources, if available
if (!File.Exists(gestureDatabase))
{
TextAsset textRes = Resources.Load(gestureDatabase, typeof(TextAsset)) as TextAsset;
if (textRes != null && textRes.bytes.Length != 0)
{
File.WriteAllBytes(gestureDatabase, textRes.bytes);
}
}
// create the vgb source
vgbFrameSource = VisualGestureBuilderFrameSource.Create(kinectSensor, 0);
// open the reader
vgbFrameReader = vgbFrameSource != null?vgbFrameSource.OpenReader() : null;
if (vgbFrameReader != null)
{
vgbFrameReader.IsPaused = true;
}
using (VisualGestureBuilderDatabase database = VisualGestureBuilderDatabase.Create(gestureDatabase))
{
if (database == null)
{
Debug.LogError("Gesture database not found: " + gestureDatabase);
return(false);
}
// check if we need to load all gestures
bool bAllGestures = (gestureNames.Count == 0);
foreach (Gesture gesture in database.AvailableGestures)
{
bool bAddGesture = bAllGestures || gestureNames.Contains(gesture.Name);
if (bAddGesture)
{
string sGestureName = gesture.Name;
vgbFrameSource.AddGesture(gesture);
if (!gestureNames.Contains(sGestureName))
{
gestureNames.Add(sGestureName);
}
if (!gestureData.ContainsKey(sGestureName))
{
VisualGestureData data = new VisualGestureData();
data.gestureName = sGestureName;
data.timestamp = Time.realtimeSinceStartup;
data.isDiscrete = (gesture.GestureType == GestureType.Discrete);
data.isContinuous = (gesture.GestureType == GestureType.Continuous);
gestureData.Add(sGestureName, data);
}
}
}
}
return(true);
}
// sets gesture data arrays from a csv line
public bool SetGestureDataFromCsv(string sCsvLine, char[] delimiters)
{
if (sCsvLine.Length == 0)
{
return(false);
}
// split the csv line in parts
//char[] delimiters = { ',' };
string[] alCsvParts = sCsvLine.Split(delimiters);
if (alCsvParts.Length < 1 || alCsvParts[0] != "vg")
{
return(false);
}
int iIndex = 1;
int iLength = alCsvParts.Length;
if (iLength < (iIndex + 1))
{
return(false);
}
// number of gestures
int gestureCount = 0;
int.TryParse(alCsvParts[iIndex], out gestureCount);
iIndex++;
if (gestureCount > 0)
{
for (int i = 0; i < gestureCount && iLength >= (iIndex + 1); i++)
{
long userId = 0;
long.TryParse(alCsvParts[iIndex], out userId);
iIndex++;
if (userId != 0 && iLength >= (iIndex + 7))
{
int discrete = 0, continuous = 0, complete = 0;
float timestamp = 0f, confidence = 0f, progress = 0f;
float.TryParse(alCsvParts[iIndex], out timestamp);
string gestureName = alCsvParts[iIndex + 1];
int.TryParse(alCsvParts[iIndex + 2], out discrete);
int.TryParse(alCsvParts[iIndex + 3], out continuous);
int.TryParse(alCsvParts[iIndex + 4], out complete);
float.TryParse(alCsvParts[iIndex + 5], out confidence);
float.TryParse(alCsvParts[iIndex + 6], out progress);
iIndex += 7;
if (!gestureNames.Contains(gestureName))
{
gestureNames.Add(gestureName);
VisualGestureData newData = new VisualGestureData();
newData.gestureName = gestureName;
gestureData[gestureName] = newData;
}
VisualGestureData data = gestureData[gestureName];
data.userId = userId;
data.timestamp = timestamp;
data.isDiscrete = (discrete != 0);
data.isContinuous = (continuous != 0);
data.isComplete = (complete != 0);
data.confidence = confidence;
float prevProgress = data.progress;
data.progress = progress;
if (data.isDiscrete)
{
if (data.isProcessed && !data.isComplete)
{
//data.isResetting = false;
data.isProcessed = false;
}
}
else if (data.isContinuous)
{
if (data.isProcessed && data.progress >= minConfidence && data.progress != prevProgress)
{
//data.isResetting = false;
data.isProcessed = false;
}
}
gestureData[gestureName] = data;
}
}
}
return(true);
}
private bool InitVisualGestures()
{
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null ? kinectManager.GetSensorData() : null;
Kinect2Interface kinectInterface = sensorData.sensorInterface as Kinect2Interface;
KinectSensor kinectSensor = kinectInterface != null ? kinectInterface.kinectSensor : null;
if(kinectSensor == null)
return false;
if(gestureDatabase == string.Empty)
{
Debug.LogError("Please specify gesture database file!");
return false;
}
// copy the gesture database file from Resources, if available
if(!File.Exists(gestureDatabase))
{
TextAsset textRes = Resources.Load(gestureDatabase, typeof(TextAsset)) as TextAsset;
if(textRes != null && textRes.bytes.Length != 0)
{
File.WriteAllBytes(gestureDatabase, textRes.bytes);
}
}
// create the vgb source
vgbFrameSource = VisualGestureBuilderFrameSource.Create(kinectSensor, 0);
// open the reader
vgbFrameReader = vgbFrameSource != null ? vgbFrameSource.OpenReader() : null;
if(vgbFrameReader != null)
{
vgbFrameReader.IsPaused = true;
}
using (VisualGestureBuilderDatabase database = VisualGestureBuilderDatabase.Create(gestureDatabase))
{
if(database == null)
{
Debug.LogError("Gesture database not found: " + gestureDatabase);
return false;
}
// check if we need to load all gestures
bool bAllGestures = (gestureNames.Count == 0);
foreach (Gesture gesture in database.AvailableGestures)
{
bool bAddGesture = bAllGestures || gestureNames.Contains(gesture.Name);
if(bAddGesture)
{
string sGestureName = gesture.Name;
vgbFrameSource.AddGesture(gesture);
if(!gestureNames.Contains(sGestureName))
{
gestureNames.Add(sGestureName);
}
if(!gestureData.ContainsKey(sGestureName))
{
VisualGestureData data = new VisualGestureData();
data.gestureName = sGestureName;
data.isDiscrete = (gesture.GestureType == GestureType.Discrete);
data.isContinuous = (gesture.GestureType == GestureType.Continuous);
data.timestamp = Time.realtimeSinceStartup;
gestureData.Add(sGestureName, data);
}
}
}
}
return true;
}