public string recognizeGesture(string gestureName, BaseTrajectory trace, out TrajectoryModel.ReasonForFail failReason)
{
//GestureRepresentation.ReasonForFail failReason;
//var calculations = knownGestures.Select((gest,i) => new { GestureName = gest.Key, Similarity = gest.Value.validateGestureTrace(trace, out failReasons[i]) });
double best = 0;
string winner = "";
TrajectoryModel.ReasonForFail tmp;
failReason = TrajectoryModel.ReasonForFail.UNDEFINED;
foreach (var gest in knownGestures)
{
var sim = gest.Value.validateGestureTrace(trace, out tmp);
if (gest.Key == gestureName)
{
failReason = tmp;
}
if (sim > best)
{
best = sim;
winner = gest.Key;
}
}
if (best == 0)
{
return(null);
}
else
{
return(winner);
}
}
public string recognizeGesture(BaseTrajectory trace)
{
var points = trace.TrajectoryPoints.Select(p => new WobbrockLib.TimePointF(p.X, p.Y, p.Time)).ToList();
var results = recognizer.Recognize(points, true);
return(results.Name);
}
public string recognizeGesture(BaseTrajectory trace)
{
var t = new Template(trace.TrajectoryPoints);
var best = knownTrajectories.MinBy(kt => kt.getBestCost(t, withTime));
return(best.Name);
}
public double validateGestureTrace(BaseTrajectory trace, out ReasonForFail failReason)
{
failReason = ReasonForFail.NoFail;
var O = strokeMap.getSymbolTrace(trace);
if (O == null)
{
failReason = ReasonForFail.MissedArea;
return(0);
}
//insert dummy symbol
//O = new Observation[] { new Observation("Dummy", O[0].Time) }.Concat(O).ToArray();
//also check time dynamics
var eval = model.evaluate(O, true);
if (eval == 0)
{
failReason = ReasonForFail.MissedTransition;
}
//normalize
return(eval /*/ model.MaxEvaluation*/);
}
public string recognizeGesture(BaseTrajectory trace)
{
var calculations = knownGestures.Select(gest => new { GestureName = gest.Key, Similarity = gest.Value.validateGestureTrace(trace) });
//var bestGesture = calculations.MaxBy(g => g.Similarity);
double maxSim = 0;
string bestGesture = null;
foreach (var calc in calculations)
{
//Console.WriteLine(calc.GestureName + "---" + calc.Similarity);
if (calc.Similarity > maxSim)
{
maxSim = calc.Similarity;
bestGesture = calc.GestureName;
}
}
if (maxSim == 0)
{
return(null);
}
else
{
return(bestGesture + ":" + maxSim);
}
}
public double getSimilarity(string userName, BaseTrajectory trace)
{
var targetGesture = knownGestures[userName];
var similarity = targetGesture.validateGestureTrace(trace);
return(similarity);
}
//public override bool strokeFitsMap(Stroke s)
//{
// var areaPoints = LinearInterpolation.getPointsByDistance(new LinearInterpolation(s.Points), AreaPointDistance);
// return !areaPoints.Zip(areas,
// (p, a) => a.PointInToleranceArea(p.X, p.Y)
// ).Any(b => !b);
//}
public override Observation[] getSymbolTrace(BaseTrajectory s)
{
var strokeInterpolation = (translationInvariant) ?
new LinearInterpolation(s.getInvariantPoints()) :
new LinearInterpolation(s.TrajectoryPoints);
var areaPoints = LinearInterpolation.getPointsByDistance(strokeInterpolation, AreaPointDistance);
//ToDo: tolerate longer inputs than trained?
//if (areaPoints.Length > Areas.Length) return null;
//ToDo: tolerate shorter inputs?
if (areaPoints.Length < Areas.Length)
{
return(null);
}
var observations = new Observation[Areas.Length + 2]; //+ GS and GE
int iObs = 1;
string baseSym = "S" + ID + "_A";
string symbol;
long time;
for (int iArea = 0; iArea < Areas.Length; iArea++)
{
var area = Areas[iArea];
if (iArea >= areaPoints.Length)
{
break;
}
var point = areaPoints[iArea];
if (area.PointInArea(point.X, point.Y))
{
symbol = baseSym + (iArea + 1) + "_Hit";
time = point.Time;
}
else if (area.PointInToleranceArea(point.X, point.Y))
{
symbol = baseSym + (iArea + 1) + "_Tolerance";
time = point.Time;
}
else
{
return(null);
}
observations[iObs++] = new Observation(symbol, time);
}
//GestureStart symbol
observations[0] = new Observation("GestureStart", s.TrajectoryPoints[0].Time);
//GestureEnd symbol
observations[observations.Length - 1] = new Observation("GestureEnd", s.TrajectoryPoints[s.TrajectoryPoints.Length - 1].Time);
return(observations);
}
public override Observation[] getSymbolTrace(BaseTrajectory s)
{
int current_stroke = s.TrajectoryPoints.First().StrokeNum;
var accumulate = new List <TrajectoryPoint>();
var gesture_AreaPoints = new List <TrajectoryPoint[]>();
foreach (var point in s.TrajectoryPoints)
{
if (point.StrokeNum == current_stroke)
{
accumulate.Add(point);
}
else
{
var trajectory = new LinearInterpolation(accumulate.ToArray());
gesture_AreaPoints.Add(LinearInterpolation.getEquidistantPoints(trajectory, nAreas));
accumulate.Clear();
current_stroke = point.StrokeNum;
}
}
gesture_AreaPoints.Add(LinearInterpolation.getEquidistantPoints(new LinearInterpolation(accumulate.ToArray()), nAreas));
var symbolTrace = new Observation[Areas.Length + gesture_AreaPoints.Count];
int counter = 0;
int stroke_counter = 0;
foreach (var fStroke in gesture_AreaPoints)
{
var Areas_fraction = Areas.Skip(stroke_counter * fStroke.Length).Take(nAreas);
var query = fStroke.Zip(Areas_fraction, (p, a) =>
{
var sym = a.CreateSymbol(p);
if (sym == null)
{
return(null);
}
string symbol = "S" + ID + "_" + sym;
return(new Observation(symbol, p.Time));
});
//GestureStart symbol
var startO = new Observation("GestureStart", s.TrajectoryPoints[0].Time);
foreach (var single_observation in SingleItemAsEnumerable(startO).Concat(query.TakeWhile(o => o != null)).ToArray())
{
symbolTrace[counter] = single_observation;
counter++;
}
stroke_counter++;
}
//wurden alle Areas erwischt?
if (symbolTrace.Length < Areas.Length)
{
return(null);
}
return(symbolTrace);
}
public bool verifyGesture(string userName, BaseTrajectory trace)
{
var targetGesture = knownGestures[userName];
var similarity = targetGesture.validateGestureTrace(trace);
//ToDo: evtl. Schwellwerte hier prüfen?
return(similarity > 0);
}
protected IStrokeInterpolation CreateInterpolation(BaseTrajectory stroke)
{
var points = (translationInvariant) ? stroke.getInvariantPoints() : stroke.TrajectoryPoints;
if (points[0] is TrajectoryPoint3D)
{
return(new LinearInterpolation3D(points.Cast <TrajectoryPoint3D>().ToArray()));
}
else
{
return(new LinearInterpolation(points));
}
}
public int checkMultiStrokeFeasibility(string name, BaseTrajectory candidate, int nArea_count)
{
var length = 0;
var targetGesture = knownGestures[name];
var sim_length = targetGesture.getTrace_match(candidate);
if (sim_length > nArea_count)
{
length = sim_length / nArea_count;
}
return(length);
}
public List <KeyValuePair <string, double> > recognizeMultiStroke(BaseTrajectory trace)
{
var calculations = knownGestures.Select(gest => new { GestureName = gest.Key, Similarity = gest.Value.validateGestureTrace(trace) });
var result = new List <KeyValuePair <string, double> >();
foreach (var calc in calculations)
{
if (calc.Similarity > 0)
{
result.Add(new KeyValuePair <string, double>(calc.GestureName, calc.Similarity));
}
}
return(result);
}
public string recognizeGesture(BaseTrajectory trace)
{
var calculations = knownGestures.Select(gest => new { GestureName = gest.Key, Similarity = gest.Value.validateGestureTrace(trace) });
var bestGesture = calculations.MaxBy(g => g.Similarity);
if (bestGesture.Similarity == 0)
{
return(null);
}
else
{
return(bestGesture.GestureName);
}
}
public int getTrace_match(BaseTrajectory trace)
{
var O = strokeMap.getSymbolTrace(trace);
int last_nonZero = 0;
for (int i = 0; i < O.Length; i++)
{
if (O[i] == null)
{
return(i);
}
last_nonZero = i + 1;
}
return(last_nonZero);
}
public int checkFeasibility(int prev_length, BaseTrajectory candidate, int nArea_count)
{
var length = 0;
foreach (var targetGesture in knownGestures)
{
var sim_length = targetGesture.Value.getTrace_match(candidate);
if (sim_length > nArea_count && (sim_length / nArea_count) > prev_length)
{
length = sim_length / nArea_count;
break;
}
}
return(length);
}
/// <summary>
/// determines whether the given trace fits the gesture representation
/// </summary>
/// <param name="trace"></param>
/// <returns>an arbitrary double value giving a measure for the similarity; 0 states that the given trace is no fitting example of the gesture </returns>
public double validateGestureTrace(BaseTrajectory trace)
{
var O = strokeMap.getSymbolTrace(trace);
if (O == null)
{
return(0);
}
//insert dummy symbol
//O = new Observation[] { new Observation("Dummy", O[0].Time) }.Concat(O).ToArray();
//also check time dynamics
var eval = model.evaluate(O, true);
//normalize
return(eval /*/ model.MaxEvaluation*/);
}
public override Observation[] getSymbolTrace(BaseTrajectory s)
{
var strokeInterpolation = (translationInvariant) ?
new LinearInterpolation(s.getInvariantPoints()) :
new LinearInterpolation(s.TrajectoryPoints);
var areaPoints = LinearInterpolation.getEquidistantPoints(strokeInterpolation, Areas.Length);
int counter = 1;
var query = areaPoints.Zip(Areas, (p, a) =>
{
string sym = "S" + ID + "_A";
long time;
if (a.PointInArea(p.X, p.Y))
{
sym += counter++ + "_Hit";
time = p.Time;
}
else if (a.PointInToleranceArea(p.X, p.Y))
{
sym += counter++ + "_Tolerance";
time = p.Time;
}
else
{
return(null);
}
return(new Observation(sym, time));
});
//GestureStart symbol
var startO = new Observation("GestureStart", s.TrajectoryPoints[0].Time);
var symbolTrace = SingleItemAsEnumerable(startO).Concat(query.TakeWhile(o => o != null)).ToArray();
//wurden alle Areas erwischt?
if (symbolTrace.Length < Areas.Length)
{
return(null);
}
return(symbolTrace);
}
//Maybe we should format the data here so that we don't have to do it in the debugger
public void PreformatData()
{
if (basePoses.Length != baseTrajectory.Length)
{
Debug.Log("Pose and trajectory data does not match!");
return;
}
BaseTrajectory[] formattedBaseTrajectories = new BaseTrajectory[baseTrajectory.Length];
Pose[] formattedBasePoses = new Pose[basePoses.Length];
for (int i = 0; i < baseTrajectory.Length; i++)
{
//Formatting base trajectories so they begin at (0,0,0);(0,0,1).
/*Vector3 localPos = baseTrajectory[0].rootWorldToLocalMatrix.MultiplyPoint3x4(baseTrajectory[i].position);
* Vector3 localFwd = baseTrajectory[0].rootWorldToLocalMatrix.MultiplyVector(baseTrajectory[i].forward);
* Matrix4x4 localMatrix = baseTrajectory[0].rootWorldToLocalMatrix.inverse * baseTrajectory[i].rootWorldToLocalMatrix;
* formattedBaseTrajectories[i] =
* new BaseTrajectory(localMatrix, localPos, localFwd, baseTrajectory[i].timeStamp);*/
//Formatting base poses so they correspond to the new trajectory
List <Vector3> localBonePositionList = new List <Vector3>();
List <Vector3> localBoneVelocityList = new List <Vector3>();
for (int j = 0; j < basePoses[i].jointPositions.Count; j++)
{
Vector3 localBonePos = basePoses[i].jointPositions[j];
Vector3 localBoneVel = i == 0
? Vector3.zero
: (basePoses[i].jointPositions[j] -
basePoses[i - 1].jointPositions[j]) * frameRate;
localBonePositionList.Add(localBonePos);
localBoneVelocityList.Add(localBoneVel);
}
formattedBasePoses[i] = new Pose(localBonePositionList, localBoneVelocityList);
}
//baseTrajectory = formattedBaseTrajectories;
basePoses = formattedBasePoses;
}
public string recognizeGesture(BaseTrajectory trace)
{
var t = new Template(trace.TrajectoryPoints);
//var best = knownTrajectories.MinBy(kt => kt.getBestCost(t, withTime));
double minCost = double.MaxValue;
TrajectoryModel best = null;
foreach (var kt in knownTrajectories)
{
var cost = kt.getBestCost(t, withTime);
if (cost < minCost)
{
minCost = cost;
best = kt;
}
}
return(best.Name);
}
// public string[] tags = new string[32];
public BaseTrajectory[] CreateBaseTrajectoryArray(AnimationClip animClip)
{
if (animClip == null)
{
return(new BaseTrajectory[0]);
}
float animLength = animClip.length;
float timeStep = 1f / config.frameRate;
float sampleTime = 0f;
BaseTrajectory[] baseTrajectory = new BaseTrajectory[(int)animLength * config.frameRate];
for (int i = 0; i < baseTrajectory.Length; i++)
{
animClip.SampleAnimation(mmAvatar, sampleTime);
Transform temp = rootJoint.transform;
baseTrajectory[i] = new BaseTrajectory(temp.worldToLocalMatrix, temp.position, temp.forward, sampleTime);
sampleTime += timeStep;
}
return(baseTrajectory);
}
/// <summary>
/// determines whether the given trace fits the gesture representation
/// </summary>
/// <param name="trace"></param>
/// <returns>an arbitrary double value giving a measure for the similarity; 0 states that the given trace is no fitting example of the gesture </returns>
public double validateGestureTrace(BaseTrajectory trace)
{
var O = strokeMap.getSymbolTrace(trace);
foreach (var symbol_idx in O)
{
if (symbol_idx == null)
{
return(0);
}
}
O = O.Where(w => w.Symbol != "GestureStart").ToArray();
//return 1;
//insert dummy symbol
//O = new Observation[] { new Observation("Dummy", O[0].Time) }.Concat(O).ToArray();
//also check time dynamics
var eval = model.evaluate(O, true);
//normalize
return(eval /*/ model.MaxEvaluation*/);
}
public abstract Observation[] getSymbolTrace(BaseTrajectory s);
