public float DistanceTo(DGInstance otherInstance, bool useDTW)
{
float dist = useDTW ? GetDTWDistance(otherInstance) : GetLerpedDistance(otherInstance);
// Factor in path length (with weight 1/5):
int newFeatureCount = 0;
float additionalDist = 0;
if (IncludesLeftHand)
{
additionalDist += Math.Abs(otherInstance.LeftHandPathLength - MeanLeftHandPathLength) / StdDevLeftHandPathLength;
newFeatureCount++;
}
if (IncludesRightHand)
{
additionalDist += Math.Abs(otherInstance.RightHandPathLength - MeanRightHandPathLength) / StdDevRightHandPathLength;
newFeatureCount++;
}
additionalDist /= (float)newFeatureCount;
float additionalDistWeight = 0; // 0.1f;
dist = ((1 - additionalDistWeight) * dist) + (additionalDistWeight * additionalDist);
return(dist);
}
public DGInstanceWrapper(DGInstance instance)
{
Instance = instance;
Id = -1;
ClassId = -1;
InstanceName = "new instance";
}
public void SaveNewDynamicGestureInstance(int classId, DGInstance instance) // Whenever this is called it will be a new instance.
{
string serializedInstance = JsonConvert.SerializeObject(instance);
string sql = String.Format("INSERT INTO DynamicGestureInstances (class_id, json) VALUES ('{0}', '{1}')", classId, serializedInstance);
executeNonQuery(sql);
}
public DGInstance GetDtwMappedInstance(DGInstance target)
{
DGInstance mappedInstance;
GetDtwDistance(target, out mappedInstance);
return(mappedInstance);
}
public Dictionary <DGClassWrapper, float> GetDistancesFromAllClasses(DGInstance gestureInstance)
{
lock (_dynamicGestureClasses)
{
var gestureDistances = new Dictionary <DGClassWrapper, float>();
foreach (var gestureClass in _dynamicGestureClasses)
{
gestureDistances.Add(gestureClass, gestureClass.Gesture.DistanceTo(gestureInstance, UseDTW));
}
return(gestureDistances);
}
}
// Lerps instance to make its sample collection same size as this class.
public float GetLerpedDistance(DGInstance otherInstance)
{
float distance = 0;
var instanceSamples = otherInstance.GetResizedSampleList(LerpedSamples.Count);
for (int i = 0; i < LerpedSamples.Count; i++)
{
var temp = LerpedSamples[i].DistanceTo(instanceSamples[i]);
distance += temp;
}
distance = distance / (float)LerpedSamples.Count;
return(distance);
}
// Finds the average number of samples for all dg instances.
// Lerps all DG instances to be this length, and creates a class for each index.
public List <DGClassSample> GetLerpedSamples(List <DGInstance> dgInstances)
{
var samples = new List <DGClassSample>();
int classSampleSize = getAverageSampleSize(dgInstances);
var lerpedDGInstances = new List <DGInstance>(); // lerps the static gesture instances contained within
foreach (var instance in dgInstances)
{
var resizedInstance = new DGInstance(instance.GetResizedSampleList(classSampleSize));
lerpedDGInstances.Add(resizedInstance);
}
// At this point, lerpedDGInstances() are all the same length.
for (int i = 0; i < classSampleSize; i++)
{
samples.Add(new DGClassSample(lerpedDGInstances.Select(dgi => dgi.Samples[i]).ToList()));
}
return(samples);
}
public float GetDTWDistance(DGInstance otherInstance, int windowSize = 3)
{
if (DtwSamples.Count == 0 || otherInstance.Samples.Count == 0)
{
return(float.PositiveInfinity);
}
float[,] distances = new float[DtwSamples.Count, otherInstance.Samples.Count];
for (int i = 0; i < DtwSamples.Count; i++)
{
for (int j = 0; j < otherInstance.Samples.Count; j++)
{
distances[i, j] = DtwSamples[i].DistanceTo(otherInstance.Samples[j]);
}
}
int window = Math.Max(windowSize, Math.Abs(DtwSamples.Count - otherInstance.Samples.Count));
float[,] dtw = new float[DtwSamples.Count, otherInstance.Samples.Count];
for (int i = 0; i < DtwSamples.Count; i++)
{
for (int j = 0; j < otherInstance.Samples.Count; j++)
{
dtw[i, j] = float.PositiveInfinity;
}
}
dtw[0, 0] = 0;
for (int i = 1; i < DtwSamples.Count; i++)
{
//for (int j = 0; j < otherInstance.Samples.Count; j++)
for (int j = Math.Max(1, i - window); j < Math.Min(otherInstance.Samples.Count, i + window); j++)
{
float cost = distances[i, j];
var precedingCosts = new List <float>()
{
dtw[Math.Max(i - 1, 0), j],
dtw[i, Math.Max(j - 1, 0)],
dtw[Math.Max(i - 1, 0), Math.Max(j - 1, 0)]
};
dtw[i, j] = cost + precedingCosts.Min();
}
}
// TODO: Backtrack through and find the path (particularly interested in its length)
var path = new Stack <Tuple <int, int> >();
int x = DtwSamples.Count - 1;
int y = otherInstance.Samples.Count - 1;
path.Push(new Tuple <int, int>(x, y));
while (x > 0 || y > 0)
{
if (x == 0)
{
y--;
}
else if (y == 0)
{
x--;
}
else
{
var neighboringCosts = new List <float>()
{
dtw[Math.Max(0, x - 1), Math.Max(0, y - 1)],
dtw[Math.Max(0, x - 1), y],
dtw[x, Math.Max(0, y - 1)]
};
if (dtw[Math.Max(0, x - 1), y] == neighboringCosts.Min())
{
x--;
}
else if (dtw[x, Math.Max(0, y - 1)] == neighboringCosts.Min())
{
y--;
}
else
{
x--; y--;
}
}
path.Push(new Tuple <int, int>(x, y));
}
float distance = dtw[DtwSamples.Count - 1, otherInstance.Samples.Count - 1] / (float)path.Count;
return(distance);
}
public float GetDtwDistance(DGInstance otherInstance, out DGInstance mappedInstance)
{
mappedInstance = null;
if (Samples.Count == 0 || otherInstance.Samples.Count == 0)
{
return(float.PositiveInfinity);
}
float[,] distances = new float[Samples.Count, otherInstance.Samples.Count];
for (int i = 0; i < Samples.Count; i++)
{
for (int j = 0; j < otherInstance.Samples.Count; j++)
{
distances[i, j] = Samples[i].DistanceTo(otherInstance.Samples[j]);
}
}
float[,] dtw = new float[Samples.Count, otherInstance.Samples.Count];
for (int i = 0; i < Samples.Count; i++)
{
for (int j = 0; j < otherInstance.Samples.Count; j++)
{
float cost = distances[i, j];
var precedingCosts = new List <float>()
{
dtw[Math.Max(i - 1, 0), j],
dtw[i, Math.Max(j - 1, 0)],
dtw[Math.Max(i - 1, 0), Math.Max(j - 1, 0)]
};
dtw[i, j] = cost + precedingCosts.Min();
}
}
// TODO: Backtrack through and find the path (particularly interested in its length)
var path = new Stack <Tuple <int, int> >();
int x = Samples.Count - 1;
int y = otherInstance.Samples.Count - 1;
path.Push(new Tuple <int, int>(x, y));
while (x > 0 || y > 0)
{
if (x == 0)
{
y--;
}
else if (y == 0)
{
x--;
}
else
{
var neighboringCosts = new List <float>()
{
dtw[x - 1, y - 1],
dtw[x - 1, y],
dtw[x, y - 1]
};
if (dtw[x - 1, y] == neighboringCosts.Min())
{
x--;
}
else if (dtw[x, y - 1] == neighboringCosts.Min())
{
y--;
}
else
{
x--; y--;
}
}
path.Push(new Tuple <int, int>(x, y));
}
// x (Item1) is index of this instance; y (Item2) is index of other instance
var mappedSamples = new List <DGInstanceSample>();
while (path.Count > 0)
{
int index = path.Pop().Item1;
mappedSamples.Add(Samples[index]);
}
mappedInstance = new DGInstance(mappedSamples);
float distance = dtw[Samples.Count - 1, otherInstance.Samples.Count - 1] / path.Count;
return(distance);
}
public void ProcessFrame(Frame frame)
{
if (_lastFrame == null)
{
_lastFrame = frame;
return;
}
bool handsStill = handsAreStill(frame);
if (frame.Hands.Count == 0)
{
State = DGRecorderState.WaitingForHands;
}
switch (State)
{
case DGRecorderState.WaitingForHands:
if (frame.Hands.Count > 0)
{
_stillGesture = new SGInstance(frame);
State = DGRecorderState.WaitingToStart;
}
break;
case DGRecorderState.WaitingToStart:
if (handsStill)
{
_startOfGesture = new DGInstanceSample(frame);
_gestureSamples = new List <DGInstanceSample>();
_gestureSamples.Add(_startOfGesture);
State = DGRecorderState.InStartPosition;
}
break;
case DGRecorderState.InStartPosition:
if (!handsStill)
{
State = DGRecorderState.RecordingGesture;
}
break;
case DGRecorderState.RecordingGesture:
if (handsStill)
{
// Trim the extra samples in back (from holding hand still for X seconds)
int stillFrames = (int)(_frameRate * _stillSeconds);
_gestureSamples.RemoveRange(_gestureSamples.Count - stillFrames, stillFrames);
MostRecentInstance = new DGInstance(_gestureSamples);
if (_inRecordMode)
{
Instances.Add(MostRecentInstance);
}
State = DGRecorderState.RecordingJustFinished; // Put this first so "InEndPosition" is printed while we process the frames
}
else
{
_gestureSamples.Add(new DGInstanceSample(frame));
}
break;
case DGRecorderState.RecordingJustFinished:
State = DGRecorderState.InEndPosition;
break;
case DGRecorderState.InEndPosition:
if (!handsStill)
{
State = DGRecorderState.WaitingToStart;
}
break;
}
}
