void RunUpdates()
{
if (openSeeExpression != null && openSeeExpression.openSee != null)
{
openSeeData = openSeeExpression.openSee.GetOpenSeeData(openSeeExpression.faceId);
if (openSeeData.fit3DError > openSeeExpression.openSee.maxFit3DError)
{
openSeeData = null;
}
}
if (initializeLipSync)
{
InitializeLipSync();
initializeLipSync = false;
}
UpdateExpression();
UpdateGaze();
BlinkEyes();
ReadAudio();
if (lipSync)
{
ApplyVisemes();
}
else
{
ApplyMouthShape();
}
}
public bool PredictExpression()
{
if (openSee == null)
{
return(false);
}
if (!simpleMode && !(modelReady && model.Ready()))
{
ResetInfo();
return(false);
}
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.time <= lastCapture)
{
return(false);
}
if (simpleMode)
{
ThresholdDetection();
return(true);
}
float[] faceData = GetData(openSeeData);
float[] predictionData = new float[cols];
for (int i = 0; i < cols; i++)
{
predictionData[i] = faceData[indices[i]];
}
float[] prediction = model.Predict(predictionData, out probabilities, 1);
int predictedExpression = (int)Mathf.Round(prediction[0]);
if (predictedExpression == lastPrediction)
{
lastPredictionCount++;
}
else
{
lastPrediction = predictedExpression;
lastPredictionCount = 1;
}
if (lastPredictionCount > expressionStabilizer)
{
currentPrediction = lastPrediction;
expressionTime = Time.time;
}
if (currentPrediction >= 0 && currentPrediction <= classLabels.Length)
{
expression = classLabels[currentPrediction];
}
else
{
expression = "";
}
return(true);
}
private float[] GetData(OpenSee.OpenSeeData t)
{
float[] data = new float[colsFull];
data[0] = t.rightEyeOpen;
data[1] = t.leftEyeOpen;
data[2] = t.translation.x;
data[3] = t.translation.y;
data[4] = t.translation.z;
data[5] = t.rawQuaternion.x;
data[6] = t.rawQuaternion.y;
data[7] = t.rawQuaternion.z;
data[8] = t.rawQuaternion.w;
data[9] = t.rawEuler.x;
data[10] = t.rawEuler.y;
data[11] = t.rawEuler.z;
/*for (int i = 0; i < 66; i++)
* data[12 + i] = t.confidence[i];
* for (int i = 0; i < 66; i++) {
* data[12 + 66 + 2 * i] = t.points[i].x / t.cameraResolution.x;
* data[12 + 66 + 2 * i + 1] = t.points[i].y / t.cameraResolution.y;
* }*/
for (int i = 0; i < 66; i++)
{
data[12 + /*66 + 2 * 66 +*/ 3 * i] = t.points3D[i].x;
data[12 + /*66 + 2 * 66 +*/ 3 * i + 1] = t.points3D[i].y;
data[12 + /*66 + 2 * 66 +*/ 3 * i + 2] = t.points3D[i].z;
}
data[12 + 66 * 3 + 0] = t.features.EyeLeft;
data[12 + 66 * 3 + 1] = t.features.EyeRight;
data[12 + 66 * 3 + 2] = t.features.EyebrowSteepnessLeft;
data[12 + 66 * 3 + 3] = t.features.EyebrowUpDownLeft;
data[12 + 66 * 3 + 4] = t.features.EyebrowQuirkLeft;
data[12 + 66 * 3 + 5] = t.features.EyebrowSteepnessRight;
data[12 + 66 * 3 + 6] = t.features.EyebrowUpDownRight;
data[12 + 66 * 3 + 7] = t.features.EyebrowQuirkRight;
data[12 + 66 * 3 + 8] = t.features.MouthCornerUpDownLeft;
data[12 + 66 * 3 + 9] = t.features.MouthCornerInOutLeft;
data[12 + 66 * 3 + 10] = t.features.MouthCornerUpDownRight;
data[12 + 66 * 3 + 11] = t.features.MouthCornerInOutRight;
data[12 + 66 * 3 + 12] = t.features.MouthOpen;
data[12 + 66 * 3 + 13] = t.features.MouthWide;
return(data);
}
public bool CaptureCalibrationData()
{
if (openSee == null)
{
ResetInfo();
return(false);
}
if (recordingSkip < 1)
{
recordingSkip = 1;
}
if (calibrationExpression == "")
{
return(false);
}
if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10)
{
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || openSeeData.time <= lastCapture)
{
return(false);
}
if (!expressions.ContainsKey(calibrationExpression))
{
expressions.Add(calibrationExpression, new List <float[]>());
}
List <float[]> list = expressions[calibrationExpression];
if (list.Count >= maxSamples)
{
percentRecorded = 100f;
recording = allowOverRecording;
if (!recording)
{
return(false);
}
}
int skip = recordingSkip;
if (percentRecorded >= 100f)
{
skip = overRecordingSkip;
}
if (frameCount % skip != 0)
{
return(true);
}
if (percentRecorded >= 100f)
{
list[rnd.Next(maxSamples)] = GetData(openSeeData);
}
else
{
list.Add(GetData(openSeeData));
}
percentRecorded = 100f * (float)list.Count / (float)maxSamples;
return(true);
}
else
{
recording = false;
calibrationExpression = "";
return(false);
}
}
void Update()
{
if (!openSee)
{
return;
}
/*openSeeData = openSee.trackingData;
* if (openSeeData == null || openSeeData.Length < 1)
* return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null || (show3DPoints && openSeeData.fit3DError > openSee.maxFit3DError))
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
}
else
{
return;
}
if (show3DPoints)
{
centerBall.gameObject.SetActive(false);
for (int i = 0; i < total; i++)
{
if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence))
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt;
if (i < 68)
{
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
}
else
{
if (i == 68)
{
gameObjects[i].transform.localRotation = openSeeData.rightGaze;
}
else
{
gameObjects[i].transform.localRotation = openSeeData.leftGaze;
}
}
}
else
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.cyan);
}
}
if (applyTranslation)
{
Vector3 v = openSeeData.translation;
v.x = -v.x;
v.z = -v.z;
transform.localPosition = v;
}
if (applyRotation)
{
Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion;
}
}
else
{
centerBall.gameObject.SetActive(false);
Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
float minX = 10000.0f;
float minY = 10000.0f;
float maxX = -1.0f;
float maxY = -1.0f;
for (int i = 0; i < 66; i++)
{
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
if (minX > x)
{
minX = x;
}
if (minY > y)
{
minY = y;
}
if (maxX < x)
{
maxX = x;
}
if (maxY < y)
{
maxY = y;
}
center += new Vector3(x, y, 0.0f);
}
center = center / 66;
center = center - new Vector3(minX, minY, 0.0f);
center.x = center.x / (maxX - minX);
center.y = center.y / (maxX - minX);
center.z = 0.5f;
centerBall.transform.localPosition = center;
for (int i = 0; i < 66; i++)
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX);
position.y = position.y / (maxX - minX);
gameObjects[i].transform.localPosition = position;
}
}
for (int i = 0; i < 68; i++)
{
if (lines[i] == -1)
{
continue;
}
if (!showLines || lineMaterial == null)
{
lineRenderers[i].enabled = false;
}
else
{
int a = i;
int b = lines[i];
if (i == 66)
{
a = 48;
}
if (i == 67)
{
a = 53;
}
Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b]));
lineRenderers[i].enabled = true;
lineRenderers[i].widthMultiplier = lineWidth;
//lineRenderers[i].material = lineMaterial;
lineRenderers[i].material.SetColor("_Color", color);
lineRenderers[i].startColor = color;
lineRenderers[i].endColor = color;
lineRenderers[i].SetPosition(0, gameObjects[a].transform.position);
lineRenderers[i].SetPosition(1, gameObjects[b].transform.position);
}
}
}
void Update()
{
if (!openSee)
{
return;
}
/*openSeeData = openSee.trackingData;
* if (openSeeData == null || openSeeData.Length < 1)
* return;*/
openSeeData = openSee.GetOpenSeeData(faceId);
if (openSeeData == null)
{
return;
}
if (openSeeData.time > updated)
{
updated = openSeeData.time;
}
else
{
return;
}
if (show3DPoints)
{
centerBall.gameObject.SetActive(false);
for (int i = 0; i < total; i++)
{
if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence))
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
Vector3 pt = openSeeData.points3D[i];
pt.x = -pt.x;
gameObjects[i].transform.localPosition = pt;
if (i < 68)
{
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
}
else
{
if (i == 68)
{
gameObjects[i].transform.localRotation = openSeeData.rightGaze;
}
else
{
gameObjects[i].transform.localRotation = openSeeData.leftGaze;
}
}
}
else
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.cyan);
}
}
if (applyTranslation)
{
Vector3 v = openSeeData.translation;
v.x = -v.x;
v.z = -v.z;
transform.localPosition = v;
}
if (applyRotation)
{
Quaternion offset = Quaternion.Euler(0f, 0f, -90f);
Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset;
transform.localRotation = convertedQuaternion;
}
}
else
{
centerBall.gameObject.SetActive(false);
Vector3 center = new Vector3(0.0f, 0.0f, 0.0f);
float minX = 10000.0f;
float minY = 10000.0f;
float maxX = -1.0f;
float maxY = -1.0f;
for (int i = 0; i < 66; i++)
{
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
if (minX > x)
{
minX = x;
}
if (minY > y)
{
minY = y;
}
if (maxX < x)
{
maxX = x;
}
if (maxY < y)
{
maxY = y;
}
center += new Vector3(x, y, 0.0f);
}
center = center / 66;
center = center - new Vector3(minX, minY, 0.0f);
center.x = center.x / (maxX - minX);
center.y = center.y / (maxX - minX);
center.z = 0.5f;
centerBall.transform.localPosition = center;
for (int i = 0; i < 66; i++)
{
Renderer renderer = gameObjects[i].GetComponent <Renderer>();
renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i]));
float x = openSeeData.points[i].x;
float y = -openSeeData.points[i].y;
Vector3 position = new Vector3(x, y, 0.0f);
position = position - new Vector3(minX, minY, 0.0f);
position.x = position.x / (maxX - minX);
position.y = position.y / (maxX - minX);
gameObjects[i].transform.localPosition = position;
}
}
}
