public bool PredictExpression()
{
if (openSee == null)
{
return(false);
}
if (!simpleMode && !(modelReady && model.Ready()))
{
ResetInfo();
return(false);
}
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.time <= lastCapture)
{
return(false);
}
if (simpleMode)
{
ThresholdDetection();
return(true);
}
float[] faceData = GetData(openSeeData);
float[] predictionData = new float[cols];
for (int i = 0; i < cols; i++)
{
predictionData[i] = faceData[indices[i]];
}
float[] prediction = model.Predict(predictionData, out probabilities, 1);
int predictedExpression = (int)Mathf.Round(prediction[0]);
if (predictedExpression == lastPrediction)
{
lastPredictionCount++;
}
else
{
lastPrediction = predictedExpression;
lastPredictionCount = 1;
}
if (lastPredictionCount > expressionStabilizer)
{
currentPrediction = lastPrediction;
expressionTime = Time.time;
}
if (currentPrediction >= 0 && currentPrediction <= classLabels.Length)
{
expression = classLabels[currentPrediction];
}
else
{
expression = "";
}
return(true);
}
void RunUpdate()
{
var openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError)
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
trackingFrames++;
if (trackingFrames < skipFirst)
{
return;
}
}
else
{
Interpolate();
return;
}
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
Vector3 t = openSeeData.translation;
t.x = -t.x;
t.z = -t.z;
Vector3 convertedTranslation = new Vector3(t.x, t.y, t.z);
// Check for angular outliers
if (gotAccepted)
{
float angularDifference = Quaternion.Angle(lastAccepted, convertedQuaternion);
if (angularDifference >= outlierThresholdAngle)
{
if (!skipped)
{
if (outlierSkipPeriod >= 0.0000001f)
{
skipStartTime = Time.time;
skipped = true;
}
}
else
{
if (Time.time > skipStartTime + outlierSkipPeriod)
{
lastAccepted = convertedQuaternion;
skipped = false;
}
}
}
else
{
skipped = false;
lastAccepted = convertedQuaternion;
}
}
else
{
lastAccepted = convertedQuaternion;
skipped = false;
}
gotAccepted = true;
if (skipped)
{
outlierSkips += 1;
return;
}
// Check for positional outliers
bool keepOffset = false;
if (gotAcceptedPosition)
{
float distance = Vector3.Distance(t, lastAcceptedPosition);
if (distance > outlierThresholdDistance)
{
if (!skippedPosition)
{
if (outlierSkipPeriodDistance >= 0.0000001f)
{
skipStartTimePosition = Time.time;
skippedPosition = true;
}
}
else
{
if (Time.time > skipStartTimePosition + outlierSkipPeriodDistance)
{
lastAcceptedPosition = t;
skippedPosition = false;
if (outlierRecalibrate)
{
calibrate = true;
keepOffset = outlierKeepOffset;
outlierCalibrations += 1;
}
}
}
}
else
{
skippedPosition = false;
lastAcceptedPosition = t;
}
}
else
{
skippedPosition = false;
lastAcceptedPosition = t;
}
gotAcceptedPosition = true;
if (skippedPosition)
{
outlierSkipsDistance += 1;
return;
}
if (calibrate)
{
dR = convertedQuaternion;
dR = Quaternion.Inverse(dR);
dT = t;
rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
translationOffset = new Vector3(dT.x, dT.y, dT.z);
lastAcceptedPosition = translationOffset;
skippedPosition = false;
}
if (mirrorMotion != lastMirror || (mirrorMotion && calibrate))
{
dR = Quaternion.Inverse(MirrorQuaternion(Quaternion.Inverse(dR)));
dT = MirrorTranslation(dT);
lastMirror = mirrorMotion;
}
if (calibrate && keepOffset)
{
dR = Quaternion.Inverse(Quaternion.Inverse(transform.localRotation) * Quaternion.Inverse(dR));
dT -= transform.localPosition;
}
calibrate = false;
if (mirrorMotion)
{
convertedQuaternion = MirrorQuaternion(convertedQuaternion);
t = MirrorTranslation(t);
}
if (interpolateState > 1)
{
avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f);
}
interpolationCount = 0;
averageInterpolations = avgInterps;
lastT = transform.localPosition;
lastR = transform.localRotation;
if (interpolateState > 0)
{
currentT = Vector3.Lerp(currentT, (t - dT) * translationScale, 1f - smooth);
currentR = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
}
else
{
currentT = (t - dT) * translationScale;
currentR = convertedQuaternion * dR;
}
if (interpolateState < 2)
{
interpolateState++;
}
if (interpolate)
{
Interpolate();
}
else
{
transform.localPosition = Vector3.Lerp(transform.localPosition, (t - dT) * translationScale, 1f - smooth);
transform.localRotation = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
}
if (kinematicInterpolation != null && kinematicInterpolation.gameObject != gameObject)
{
kinematicInterpolation.UpdateKI(updated, transform.localPosition, transform.localRotation);
}
if (driftBack)
{
if (mirrorMotion)
{
dT = Vector3.Lerp(dT, MirrorTranslation(convertedTranslation), driftFactor);
dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
}
else
{
dT = Vector3.Lerp(dT, convertedTranslation, driftFactor);
dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
}
rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
translationOffset = new Vector3(dT.x, dT.y, dT.z);
}
}
void RunUpdate()
{
var openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError)
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
}
else
{
Interpolate();
return;
}
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w);
Vector3 t = openSeeData.translation;
t.x = -t.x;
t.z = -t.z;
Vector3 convertedTranslation = new Vector3(t.x, t.y, t.z);
if (calibrate)
{
dR = convertedQuaternion;
dR = Quaternion.Inverse(dR);
dT = t;
rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
translationOffset = new Vector3(dT.x, dT.y, dT.z);
}
if (mirrorMotion != lastMirror || (mirrorMotion && calibrate))
{
dR = Quaternion.Inverse(MirrorQuaternion(Quaternion.Inverse(dR)));
dT = MirrorTranslation(dT);
lastMirror = mirrorMotion;
}
calibrate = false;
if (mirrorMotion)
{
convertedQuaternion = MirrorQuaternion(convertedQuaternion);
t = MirrorTranslation(t);
}
if (interpolateState > 1)
{
avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f);
}
interpolationCount = 0;
averageInterpolations = avgInterps;
lastT = currentT;
lastR = currentR;
if (interpolateState > 0)
{
currentT = Vector3.Lerp(currentT, (t - dT) * translationScale, 1f - smooth);
currentR = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
}
else
{
currentT = (t - dT) * translationScale;
currentR = convertedQuaternion * dR;
}
if (interpolateState < 2)
{
interpolateState++;
}
if (interpolate)
{
Interpolate();
}
else
{
transform.localPosition = Vector3.Lerp(transform.localPosition, (t - dT) * translationScale, 1f - smooth);
transform.localRotation = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth);
}
if (driftBack)
{
if (mirrorMotion)
{
dT = Vector3.Lerp(dT, MirrorTranslation(convertedTranslation), driftFactor);
dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
}
else
{
dT = Vector3.Lerp(dT, convertedTranslation, driftFactor);
dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor);
}
rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z);
translationOffset = new Vector3(dT.x, dT.y, dT.z);
}
}
void Update()
{
if (!openSee)
{
return;
}
/*openSeeData = openSee.trackingData;
* if (openSeeData == null || openSeeData.Length < 1)
* return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || (show3DPoints && openSeeData.fit3DError > openSee.maxFit3DError))
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
}
else
{
return;
}
if (show3DPoints)
{
centerBall.gameObject.SetActive(false);
for (int i = 0; i < total; i++)
{
if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence))
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt;
if (i < 68)
{
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
}
else
{
if (i == 68)
{
gameObjects[i].transform.localRotation = openSeeData.rightGaze;
}
else
{
gameObjects[i].transform.localRotation = openSeeData.leftGaze;
}
}
}
else
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.cyan);
}
}
if (applyTranslation)
{
Vector3 v = openSeeData.translation;
v.x = -v.x;
v.z = -v.z;
transform.localPosition = v;
}
if (applyRotation)
{
Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion;
}
}
else
{
centerBall.gameObject.SetActive(false);
Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
float minX = 10000.0f;
float minY = 10000.0f;
float maxX = -1.0f;
float maxY = -1.0f;
for (int i = 0; i < 66; i++)
{
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
if (minX > x)
{
minX = x;
}
if (minY > y)
{
minY = y;
}
if (maxX < x)
{
maxX = x;
}
if (maxY < y)
{
maxY = y;
}
center += new Vector3(x, y, 0.0f);
}
center = center / 66;
center = center - new Vector3(minX, minY, 0.0f);
center.x = center.x / (maxX - minX);
center.y = center.y / (maxX - minX);
center.z = 0.5f;
centerBall.transform.localPosition = center;
for (int i = 0; i < 66; i++)
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX);
position.y = position.y / (maxX - minX);
gameObjects[i].transform.localPosition = position;
}
}
for (int i = 0; i < 68; i++)
{
if (lines[i] == -1)
{
continue;
}
if (!showLines || lineMaterial == null)
{
lineRenderers[i].enabled = false;
}
else
{
int a = i;
int b = lines[i];
if (i == 66)
{
a = 48;
}
if (i == 67)
{
a = 53;
}
Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b]));
lineRenderers[i].enabled = true;
lineRenderers[i].widthMultiplier = lineWidth;
//lineRenderers[i].material = lineMaterial;
lineRenderers[i].material.SetColor("_Color", color);
lineRenderers[i].startColor = color;
lineRenderers[i].endColor = color;
lineRenderers[i].SetPosition(0, gameObjects[a].transform.position);
lineRenderers[i].SetPosition(1, gameObjects[b].transform.position);
}
}
}
void Update()
{
if (!openSee)
{
return;
}
/*openSeeData = openSee.trackingData;
* if (openSeeData == null || openSeeData.Length < 1)
* return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null)
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
}
else
{
return;
}
if (show3DPoints)
{
centerBall.gameObject.SetActive(false);
for (int i = 0; i < total; i++)
{
if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence))
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt;
if (i < 68)
{
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
}
else
{
if (i == 68)
{
gameObjects[i].transform.localRotation = openSeeData.rightGaze;
}
else
{
gameObjects[i].transform.localRotation = openSeeData.leftGaze;
}
}
}
else
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.cyan);
}
}
if (applyTranslation)
{
Vector3 v = openSeeData.translation;
v.x = -v.x;
v.z = -v.z;
transform.localPosition = v;
}
if (applyRotation)
{
Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion;
}
}
else
{
centerBall.gameObject.SetActive(false);
Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
float minX = 10000.0f;
float minY = 10000.0f;
float maxX = -1.0f;
float maxY = -1.0f;
for (int i = 0; i < 66; i++)
{
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
if (minX > x)
{
minX = x;
}
if (minY > y)
{
minY = y;
}
if (maxX < x)
{
maxX = x;
}
if (maxY < y)
{
maxY = y;
}
center += new Vector3(x, y, 0.0f);
}
center = center / 66;
center = center - new Vector3(minX, minY, 0.0f);
center.x = center.x / (maxX - minX);
center.y = center.y / (maxX - minX);
center.z = 0.5f;
centerBall.transform.localPosition = center;
for (int i = 0; i < 66; i++)
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX);
position.y = position.y / (maxX - minX);
gameObjects[i].transform.localPosition = position;
}
}
}
