public Dictionary <string, Vector3> Predict(Dictionary <string, Vector3> availableMarkers, HashSet <string> occludedMarkers,
Dictionary <string, Vector3> oldMarkers, HashSet <string> oldAvailableMarkers, float timeSinceLastFrame)
{
var availableMarkersSet = new HashSet <string>(availableMarkers.Keys);
var markerList = handTemplate.MarkerList;
double[,] R;
double[] shift;
// Put the hand in object space
float[] input = ExtractInputFeatures(availableMarkers, out R, out shift);
Dictionary <string, Vector3> output;
if (occludedMarkers.Count > 0 || model.IsRecurrent())
{
// Predict using the neural network
output = ExtractOutput(model.Predict(input));
}
else
{
// No need to predict using the neural network (there are no occlusions and the model is not recurrent)
output = new Dictionary <string, Vector3>();
}
foreach (var marker in markerList)
{
if (availableMarkersSet.Contains(marker))
{
output[marker] = Utils.RotateAndShift(availableMarkers[marker], R, shift);
}
}
// If the set of occlusions has changed, the offsets must be recomputed
if (!model.IsRecurrent() && oldMarkers.Keys.Count == markerList.Count)
{
if (!availableMarkersSet.SetEquals(oldAvailableMarkers))
{
// Compute new offsets, using the data from the previous frame
var X = new Dictionary <string, Vector3>();
foreach (var marker in availableMarkersSet)
{
X[marker] = oldMarkers[marker];
}
double[,] RX;
double[] offsetX;
float[] inputX = ExtractInputFeatures(X, out RX, out offsetX);
var outputX = ExtractOutput(model.Predict(inputX));
var Y = new Dictionary <string, Vector3>();
foreach (var marker in markerList)
{
Y[marker] = oldMarkers[marker];
}
foreach (var marker in markerList)
{
if (availableMarkersSet.Contains(marker) && oldAvailableMarkers.Contains(marker))
{
// Do nothing
}
else if (availableMarkersSet.Contains(marker) && !oldAvailableMarkers.Contains(marker))
{
// Re-entry discontinuity
Vector3 y = Utils.RotateAndShift(Y[marker], R, shift);
offsets[marker] = y - output[marker];
}
else
{
// Occlusion discontinuity
Vector3 y = Utils.RotateAndShift(Y[marker], RX, offsetX);
offsets[marker] = y - outputX[marker];
}
}
}
foreach (var marker in markerList)
{
if (smoothing)
{
// Decay offsets for re-entry discontinuities
if (availableMarkersSet.Contains(marker) && offsets[marker] != Vector3.zero)
{
float oldMagnitude = offsets[marker].sqrMagnitude;
offsets[marker] -= SmoothingConstant * timeSinceLastFrame * offsets[marker].normalized;
if (offsets[marker].sqrMagnitude >= oldMagnitude)
{
offsets[marker] = Vector3.zero;
}
}
}
else
{
if (availableMarkersSet.Contains(marker))
{
offsets[marker] = Vector3.zero;
}
}
// Add offsets
if (useOffset)
{
// Limit offset
if (offsets[marker].magnitude > OffsetLimit)
{
offsets[marker] = offsets[marker].normalized * OffsetLimit;
}
output[marker] += offsets[marker];
}
}
}
// Move the hand back in world space
handTemplate.InverseAlignTransform(output, R, shift);
// This part is not really necessary, but it helps with improving numerical stability
foreach (var marker in markerList)
{
if (availableMarkersSet.Contains(marker) && offsets[marker] == Vector3.zero)
{
output[marker] = availableMarkers[marker];
}
}
return(output);
}
