protected virtual Frame GetMinSumDistAssignment(Frame lastFrame, Frame currentFrame,
KdTree kdTreeCurrentTouches)
{
int max = lastFrame.Count > currentFrame.Count ? lastFrame.Count : currentFrame.Count;
double[,] matchingMatrix = new double[max, max];
for (int i = 0; i < lastFrame.Count; i++)
{
foreach (var current in currentFrame)
{
matchingMatrix[i, current.ID] =
MetricDistances.EuclideanDistance(new Vertex(current.X, current.Y),
new Vertex(lastFrame[i].X, lastFrame[i].Y));
}
for (int j = currentFrame.Count; j < max; j++)
{
matchingMatrix[i, j] = -1000;
}
}
var result = GreedySolver(matchingMatrix);
Frame trackedFrame = new Frame(currentFrame.TimeStamp);
for (int i = 0; i < lastFrame.Count; i++)
{
Touch current = currentFrame.GetTouch(result[i]);
if (current == null)
{
continue;
}
trackedFrame.AddTouch(new Touch(lastFrame[i].ID, current.X, current.Y,
current.DimX, current.DimY, current.Intense));
kdTreeCurrentTouches.Delete(current.ID);
}
return(trackedFrame);
}
private bool CheckMaxDistanceFromLineKriterion(Frame[] frameList, IVertex linePointA, IVertex linePointB, double maxDistanceFromLine)
{
var pA = linePointA;
var pB = linePointB;
double lengthC = MetricDistances.EuclideanDistance(pA, pB);
foreach (var frame in frameList)
{
foreach (var touch in frame)
{
var pC = new Vertex(touch.x, touch.y);
double lengthA = MetricDistances.EuclideanDistance(pB, pC);
double lengthB = MetricDistances.EuclideanDistance(pA, pC);
if (lengthA == 0 || lengthB == 0)
{
continue;
}
double cosA = (lengthB * lengthB + lengthC * lengthC - lengthA * lengthA) /
(2 * lengthB * lengthC);
double sinA = Math.Sin(Math.Acos(cosA));
double sinB = lengthB * sinA / lengthA;
double distC = lengthA * sinB;
if (distC > maxDistanceFromLine)
{
return(false);
}
}
}
return(true);
}
protected virtual Frame GetNearestNeighbourAssignment(Frame lastFrame, Frame currentFrame,
KdTree kdTreeCurrentTouches)
{
Frame trackedFrame = new Frame(currentFrame.TimeStamp);
int fibHeapCount = 0;
fibHeap.Initialize(MAXBLOBS);
foreach (var lastTouch in lastFrame)
{
IVertex lastVertex = new Vertex(lastTouch.X, lastTouch.Y);
int nn = kdTreeCurrentTouches.TopDownNearestNeighbour(lastVertex);
Touch currentTouch = currentFrame.GetTouch(nn);
IVertex currentVertex = new Vertex(currentTouch.X, currentTouch.Y);
lastToCurrentAssignment[lastTouch.ID] = nn;
fibHeap.InsertKey(lastTouch.ID, MetricDistances.EuclideanDistance(
lastVertex, currentVertex));
fibHeapCount++;
}
int min;
int tempCounter = 0;
while ((min = fibHeap.Minimum) > -1 && (min = fibHeap.DeleteMin()) != -1)
{
tempCounter++;
if (lastFrame.GetTouch(min) == null)
{
}
if (!kdTreeCurrentTouches.Included(lastToCurrentAssignment[min]))
{
Touch lastTouch = lastFrame.GetTouch(min);
IVertex lastVertex = new Vertex(lastTouch.X, lastTouch.Y);
int nn = kdTreeCurrentTouches.TopDownNearestNeighbour(lastVertex);
Touch currentTouch = currentFrame.GetTouch(nn);
if (currentTouch == null) // trajectory ended in last frame
{
return(trackedFrame); // if no radii search there is no point left to connect
}
else
{
IVertex currentVertex = new Vertex(currentTouch.X, currentTouch.Y);
lastToCurrentAssignment[lastTouch.ID] = currentTouch.ID;
fibHeap.InsertKey(lastTouch.ID, MetricDistances.EuclideanDistance(
lastVertex, currentVertex));
}
continue;
}
Touch t = currentFrame.GetTouch(lastToCurrentAssignment[min]);
kdTreeCurrentTouches.Delete(lastToCurrentAssignment[min]);
Touch newTouch = new Touch(min, t.X, t.Y, t.DimX, t.DimY, t.Intense);
trackedFrame.AddTouch(newTouch);
}
return(trackedFrame);
}
private IClassificationResult RecognizeBlurryCircleGesture(IList <Frame> frames)
{
String resultString = null;
if (!CheckTokenCriterion(frames, 1))
{
return(null);
}
int startPointIndex = FindFirstFrameIndexWithCorrectCount(frames, 1, true);
int endPointIndex = FindFirstFrameIndexWithCorrectCount(frames, 1, false);
if (startPointIndex == -1 || endPointIndex == -1)
{
return(null);
}
Vertex startPoint = new Vertex(frames[startPointIndex][0].X, frames[startPointIndex][0].Y);
Vertex endPoint = new Vertex(frames[endPointIndex][0].X, frames[endPointIndex][0].Y);
//search endpoint es point with max distance from start point
Size dim;
var centre = GetCentre(frames, out dim); //centre of all blobs
if (MetricDistances.EuclideanDistance(centre, startPoint)
> MetricDistances.EuclideanDistance(startPoint, endPoint))
{
int endPointBlobIndex = GetBlobMaxDistantFromPointIndex(frames, startPoint, out endPointIndex);
}
bool fullCircle = false;
if (CheckCircleFormKriterion(frames))
{
if (CheckForFullCircle(frames, startPoint, endPoint))
{
resultString = "circle"; fullCircle = true;
}
else
{
resultString = "semi circle";
}
}
else
{
return(null);
}
return(new ClassificationResult(
resultString,
fullCircle ? 0.7 : 0.6,
new Sample[] { new Sample(DateTime.Now, startPoint) },
new Dictionary <String, Object>()
{
{ "FirstTouch", startPoint },
{ "LastTouch", endPoint },
{ "direction", GetCircleDirection(frames) ? 1.0 : -1.0 },
{ "dimensions", dim }
}));
}
private IClassificationResult RecognizeBlurryLineGesture(IList <Frame> frames)
{
if (!CheckTokenCriterion(frames, 1))
{
return(null);
}
Vertex startPoint, endPoint;
int startPointIndex = FindFirstFrameIndexWithCorrectCount(frames, 1, true);
int endPointIndex = FindFirstFrameIndexWithCorrectCount(frames, 1, false);
Touch t = frames[startPointIndex][0];
startPoint = new Vertex(t.X, t.Y);
int maxDistantBlobFrameIndex;
//search endpoint es point with max distance from start point
//accounts for misleading data and offers robust identification of lines with disadvantage of
//more wrong positive classifications
int maxDistantBlobIndex = GetBlobMaxDistantFromPointIndex(frames, startPoint, out maxDistantBlobFrameIndex);
if (startPointIndex == -1 || endPointIndex == -1 ||
startPointIndex == endPointIndex || maxDistantBlobFrameIndex == -1)
{
return(null);
}
t = frames[endPointIndex][0];
endPoint = new Vertex(t.X, t.Y);
IClassificationResult result = null;
if (CheckMaxDistanceFromLineKriterion(frames, startPoint, endPoint, MAXDISTFROMLINE) &&
MetricDistances.EuclideanDistance(startPoint, endPoint) > MINLINELENGTH)
{
//return RecognizeDirectionalLine(startPoint, endPoint);
result = new ClassificationResult(
"line",
0.9,
new Sample[] {
new Sample(DateTime.Now, startPoint),
new Sample(DateTime.Now, endPoint)
},
new Dictionary <String, Object>()
{
{ "FirstTouch", startPoint },
{ "LastTouch", endPoint },
{ "angle", GetAngle(startPoint, endPoint) }
});
}
return(result);
}
/// <summary>
/// Checks for tap gestures.
/// </summary>
/// <param className="clusteredSamples">The clustered samples.</param>
/// <returns>Number of Taps detected, -1 if no tap gesture could be recognized.</returns>
private int CheckForTapGestures(IList <Sample> inputData, out IVertex tappedPos)
{
tappedPos = null;
if (inputData == null || inputData.Count == 0)
{
return(-1);
}
int tapCount = 0;
bool empty = true;
IVertex mean = new Vertex(0, 0);
for (int i = 0; i < inputData.Count; i++)
{
if (inputData[i] != null)
{
if (tappedPos != null)
{
double dist = MetricDistances.EuclideanDistance(inputData[i], tappedPos);
if (dist > maxTapDistance)
{
return(-1);
}
}
if (empty)
{
tapCount++; empty = false;
if (tappedPos == null)
{
tappedPos = inputData[i];
}
mean[0] += tappedPos[0];
mean[1] += tappedPos[1];
}
}
else
{
empty = true;
}
}
mean[0] /= (double)tapCount;
mean[1] /= (double)tapCount;
// tappedPos = mean;
if (inputData[0] != null)
{
tappedPos = inputData[0];
}
else
{//TODO: two samples with zero clusters within...maybe lost signals from brailledis
return(-1);
}
return(tapCount);
}
private IClassificationResult RecognizeBlurryCircleGesture(TrackedGesture inputData)
{
String resultString = null;
if (!CheckTokenCriterion(inputData.FrameList, 1))
{
return(null);
}
int startPointIndex = FindFirstFrameIndexWithCorrectCount(inputData.FrameList, 1, true);
int endPointIndex = FindFirstFrameIndexWithCorrectCount(inputData.FrameList, 1, false);
if (startPointIndex == -1 || endPointIndex == -1)
{
return(null);
}
Vertex startPoint = new Vertex(inputData.FrameList[startPointIndex][0].x, inputData.FrameList[startPointIndex][0].y);
Vertex endPoint = new Vertex(inputData.FrameList[endPointIndex][0].x, inputData.FrameList[endPointIndex][0].y);
//search endpoint es point with max distance from start point
Size dim;
var centre = GetCentre(inputData.FrameList, out dim); //centre of all blobs
if (MetricDistances.EuclideanDistance(centre, startPoint)
> MetricDistances.EuclideanDistance(startPoint, endPoint))
{
int endPointBlobIndex = GetBlobMaxDistantFromPointIndex(inputData.FrameList, startPoint, out endPointIndex);
}
if (CheckCircleFormKriterion(inputData.FrameList))
{
if (CheckForFullCircle(inputData.FrameList, startPoint, endPoint))
{
resultString = "circle";
}
else
{
resultString = "semi circle";
}
}
else
{
return(null);
}
return(new ClassificationResult(
resultString,
100.0,
new Sample[] { new Sample(DateTime.Now, startPoint) },
new KeyValuePair <String, double>("direction", GetCircleDirection(inputData.FrameList) ? 1.0 : -1.0),
new KeyValuePair <String, Size>("dimensions", dim)
));
}
/// <summary>
/// Checks for tap gestures.
/// </summary>
/// <param className="clusteredSamples">The clustered samples.</param>
/// <returns>Number of Taps detected, -1 if no tap gesture could be recognized.</returns>
private int CheckForTapGestures(TrackedGesture inputData, ref Vertex tapedPos)
{
tapedPos[0] = -1;
if (inputData == null || inputData.Count == 0)
{
return(-1);
}
var frameList = inputData.FrameList;
int framesWithBlobs = 0;
int taps = 1;
int frameCounter = 0;
foreach (var frame in frameList)
{
if (frame.Count > 1)
{
return(-1);
}
if (frame.Count == 1)
{
framesWithBlobs++;
if (!(frame[0].cx < MAXTAPDISTANCE && frame[0].cy < MAXTAPDISTANCE)) //to big blobs
{
return(-1);
}
if (tapedPos[0] != -1.0 &&
MetricDistances.EuclideanDistance(tapedPos,
new Vertex(frame[0].x, frame[0].y)) > MAXTAPDISTANCE) //to much moving while tapping
{
return(-1);
}
if (tapedPos[0] < 0.0)
{
tapedPos = new Vertex(frame[0].x, frame[0].y); //first tap contact is tap position
}
}
else //frame without blobs -> touch left surface
{
if (frameCounter != 0 && frameCounter != frameList.Length - 1)
{
taps++;
}
}
++frameCounter;
}
return(taps);
}
private Frame GetMinSumDistAssignment(Frame lastFrame, Frame currentFrame,
KdTree kdTreeCurrentTouches)
{
int max = lastFrame.Count > currentFrame.Count ? lastFrame.Count : currentFrame.Count;
max = max > 7 ? 7 : max; //Maximize to a dimension of 7x7 to maximize calculation time
double[,] matchingMatrix = new double[max, max];
for (int i = 0; i < lastFrame.Count && i < max; i++)
{
foreach (var current in currentFrame)
{
if (current.id >= max)
{
continue;
}
matchingMatrix[i, current.id] =
MetricDistances.EuclideanDistance(new Vertex(current.x, current.y),
new Vertex(lastFrame[i].x, lastFrame[i].y));
}
for (int j = currentFrame.Count; j < max; j++)
{
matchingMatrix[i, j] = -1000;
}
}
var result = GreedySolver(matchingMatrix);
Frame trackedFrame = new Frame(currentFrame.TimeStamp);
for (int i = 0; i < lastFrame.Count; i++)
{
if (result.ContainsKey(i))
{
Touch current = currentFrame.GetTouch(result[i]);
if (current == null)
{
continue;
}
trackedFrame.AddTouch(new Touch(lastFrame[i].id, current.x, current.y,
current.cx, current.cy, current.intense));
kdTreeCurrentTouches.Delete(current.id);
}
}
return(trackedFrame);
}
private IClassificationResult RecognizeBlurryLineGesture(TrackedGesture inputData)
{
if (!CheckTokenCriterion(inputData.FrameList, 1))
{
return(null);
}
Vertex startPoint, endPoint;
int startPointIndex = FindFirstFrameIndexWithCorrectCount(inputData.FrameList, 1, true);
int endPointIndex = FindFirstFrameIndexWithCorrectCount(inputData.FrameList, 1, false);
Touch t = inputData.FrameList[startPointIndex][0];
startPoint = new Vertex(t.x, t.y);
int maxDistantBlobFrameIndex;
//search endpoint es point with max distance from start point
//accounts for misleading data and offers robust identification of lines with disadvantage of
//more wrong positive classifications
int maxDistantBlobIndex = GetBlobMaxDistantFromPointIndex(inputData.FrameList, startPoint, out maxDistantBlobFrameIndex);
if (startPointIndex == -1 || endPointIndex == -1 ||
startPointIndex == endPointIndex || maxDistantBlobFrameIndex == -1)
{
return(null);
}
t = inputData.FrameList[endPointIndex][0];
endPoint = new Vertex(t.x, t.y);
IClassificationResult result = null;
if (CheckMaxDistanceFromLineKriterion(inputData.FrameList, startPoint, endPoint, MAXDISTFROMLINE) &&
MetricDistances.EuclideanDistance(startPoint, endPoint) > MINLINELENGTH)
{
//return RecognizeDirectionalLine(startPoint, endPoint);
result = new ClassificationResult("line", 100.0, new Sample[] { new Sample(DateTime.Now, startPoint), new Sample(DateTime.Now, endPoint) },
new KeyValuePair <String, double>("angle", GetAngle(startPoint, endPoint)));
}
return(result);
}
/// <summary>
/// Checks for tap gestures.
/// </summary>
/// <param className="inputData">The input data .</param>
/// <returns>Number of Taps detected, -1 if no tap gesture could be recognized.</returns>F:\Material\Gesture_Source\Gesten\Classifiers\DTWClassifier.cs
private int CheckForTapGestures2(TrackedGesture inputData, ref Vertex tapedPos)
{
tapedPos[0] = -1;
if (inputData == null || inputData.Count == 0)
{
return(-1);
}
var frameList = inputData.FrameList;
int framesWithBlobs = 0;
int taps = 1;
foreach (var frame in frameList)
{
if (frame.Count > 1)
{
return(-1);
}
if (frame.Count == 1)
{
framesWithBlobs++;
if (!(frame[0].DimX < maxTapDistance && frame[0].DimY < maxTapDistance)) //to big blobs
{
return(-1);
}
if (tapedPos[0] != -1.0 &&
MetricDistances.EuclideanDistance(tapedPos,
new Vertex(frame[0].X, frame[0].Y)) > maxTapDistance) //to much moving while tapping
{
return(-1);
}
if (tapedPos[0] < 0.0)
{
tapedPos = new Vertex(frame[0].X, frame[0].Y); //first tap contact is tap position
}
}
else //frame without blobs -> touch left surface
{
taps++;
}
}
return(taps);
}
private bool CheckCircleFormKriterion(IList <Frame> frameList)
{
//all points must lay on a circular shape around the center
//so check variance of distance from center
Size dim;
var midPoint = GetCentre(frameList, out dim);
int blobCount = 0;
double averageDist = 0.0;
for (int i = 0; i < frameList.Count; i++)
{
for (int j = 0; j < frameList[i].Count; j++)
{
averageDist += MetricDistances.EuclideanDistance(new Vertex(frameList[i][j].X, frameList[i][j].Y), midPoint);
blobCount++;
}
}
averageDist /= blobCount;
double diff = 0.0;
for (int i = 0; i < frameList.Count; i++)
{
for (int j = 0; j < frameList[i].Count; j++)
{
double dist = MetricDistances.EuclideanDistance(new Vertex(frameList[i][j].X, frameList[i][j].Y), midPoint) - averageDist;
diff += dist * dist;
}
}
diff = Math.Sqrt(diff);
diff /= blobCount;
if (diff > MAXCIRCLEVARIANCE * averageDist)
{
return(false);
}
return(true);
}
private int GetBlobMaxDistantFromPointIndex(Frame[] frameList, IVertex point, out int frameIndex)
{
double maxDist = 0;
frameIndex = -1;
int blobIndex = 0;
for (int f = 0; f < frameList.Length; f++)
{
for (int blob = 0; blob < frameList[f].Count; blob++)
{
double temp = MetricDistances.EuclideanDistance(point,
new Vertex(frameList[f][blob].x, frameList[f][blob].y));
if (temp > maxDist)
{
maxDist = temp;
frameIndex = f;
blobIndex = blob;
}
}
}
return(frameIndex != -1 ? blobIndex : -1);
}
private IClassificationResult RecognizeBlurryPinchGesture(TrackedGesture inputData)
{
double maxTapDistance = 2;
double startDist = double.MinValue,
endDist = double.MinValue;
Sample startNode1 = new Sample(), startNode2 = new Sample();
bool semi1 = true;
bool semi2 = true;
//check for min blob relation
int equalToX = 0, lessThanX = 0, moreThanX = 0;
int i = 0;
while (i < inputData.FrameList.Length)
{
if (inputData.FrameList[i].Count == 2)
{
equalToX++;
//store the first two blobs occurring together as start blobs for the pinch gesture
if (startDist < 0.0)
{
//if more than 1/3 of the frames belongs to single line movement, than it is no pinch anymore
if (i <= inputData.FrameList.Length / 3)
{
//first fingers start contact for the pinch
startNode1 = new Sample(
inputData.FrameList[i].TimeStamp,
inputData.FrameList[i][0].x,
inputData.FrameList[i][0].y);
//second fingers start contact for the pinch
startNode2 = new Sample(
inputData.FrameList[i].TimeStamp,
inputData.FrameList[i][1].x,
inputData.FrameList[i][1].y);
//pinching fingers distance at the beginning
startDist =
MetricDistances.EuclideanDistance(startNode1, startNode2);
}
}
//check for a steady first finger (blobs stay close to first contact of one finger)
semi1 = semi1 &&
(
(Math.Abs(startNode1[0] - inputData.FrameList[i][0].x) < maxTapDistance
&&
Math.Abs(startNode1[1] - inputData.FrameList[i][0].y) < maxTapDistance
)
||
(Math.Abs(startNode1[0] - inputData.FrameList[i][1].x) < maxTapDistance
&&
Math.Abs(startNode1[1] - inputData.FrameList[i][1].y) < maxTapDistance
)
);
//check for a steady second finger (blobs stay close to first contact of one finger)
semi2 = semi2 &&
(
(Math.Abs(startNode2[0] - inputData.FrameList[i][0].x) < maxTapDistance
&&
Math.Abs(startNode2[1] - inputData.FrameList[i][0].y) < maxTapDistance
)
||
(Math.Abs(startNode2[0] - inputData.FrameList[i][1].x) < maxTapDistance
&&
Math.Abs(startNode2[1] - inputData.FrameList[i][1].y) < maxTapDistance
)
);
}
else
{
//count cases of more or less than two blobs in one frame
//can be caused by performing another gesture, flickering or touch unsensitive modules
if (inputData.FrameList[i].Count > 2)
{
moreThanX++;
}
else
{
lessThanX++;
}
}
i++;
}
//if there are to many frames with less or more than two blobs, reject pinch gesture assumption
if ((double)equalToX / (lessThanX + moreThanX) < MINBLOBRELATION)
{
return(null);
}
//get last frame with two blobs for finding end positions of both fingers
i = inputData.FrameList.Length - 1;
while (endDist < 0.0 && i >= 0)
{
//check if last frame with two fingers contact is within last third of frame-range
//if more than 1/3 of the frames belongs to single line movement, than it is no pinch anymore
if (i >= inputData.FrameList.Length * 2 / 3 && inputData.FrameList[i].Count == 2)
{
IVertex endPoint1 = new Vertex(inputData.FrameList[i][0].x, inputData.FrameList[i][0].y);
IVertex endPoint2 = new Vertex(inputData.FrameList[i][1].x, inputData.FrameList[i][1].y);
endDist = MetricDistances.EuclideanDistance(endPoint1, endPoint2);
if (startDist > endDist)
{
endPoint1 = startNode1;
endPoint2 = startNode2;
}
//if to much scattering of blobs along a given line between the outmost two fingers contact, reject pinch gesture assumption
if (!CheckMaxDistanceFromLineKriterion(inputData.FrameList, endPoint1, endPoint2, MAXDISTFROMLINE))
{
return(null);
}
}
i--;
}
//get direction out of change of start to end distances between the two fingers contacts
bool direction = startDist > endDist; //pinch or reverse pinch
// GestureToken token = ConnectTokens(inputData);
// if (!CheckMaxDistanceFromLineKriterion(token, 2 * MAXDISTFROMLINE)) { return null; }
bool pinch = startDist > 0.0 && endDist > 0.0 &&
((direction ? (startDist / endDist) : endDist / startDist) > 0.5);
return(!pinch ? null :
new ClassificationResult(
(semi2 || semi1) ? "one finger pinch" : "pinch",
1.0, new Sample[] {
semi1?startNode1: (semi2 ? startNode2 :
new Sample(startNode2.TimeStamp,
(startNode1[0] + startNode2[0]) / 2,
(startNode1[1] + startNode2[1]) / 2))
},
new KeyValuePair <String, double>("expanding",
(direction) ? -1.0 : 1.0)));
}
