void RunUpdates() { if (openSeeExpression != null && openSeeExpression.openSee != null) { openSeeData = openSeeExpression.openSee.GetOpenSeeData(openSeeExpression.faceId); if (openSeeData.fit3DError > openSeeExpression.openSee.maxFit3DError) { openSeeData = null; } } if (initializeLipSync) { InitializeLipSync(); initializeLipSync = false; } UpdateExpression(); UpdateGaze(); BlinkEyes(); ReadAudio(); if (lipSync) { ApplyVisemes(); } else { ApplyMouthShape(); } }
public bool PredictExpression() { if (openSee == null) { return(false); } if (!simpleMode && !(modelReady && model.Ready())) { ResetInfo(); return(false); } openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || openSeeData.time <= lastCapture) { return(false); } if (simpleMode) { ThresholdDetection(); return(true); } float[] faceData = GetData(openSeeData); float[] predictionData = new float[cols]; for (int i = 0; i < cols; i++) { predictionData[i] = faceData[indices[i]]; } float[] prediction = model.Predict(predictionData, out probabilities, 1); int predictedExpression = (int)Mathf.Round(prediction[0]); if (predictedExpression == lastPrediction) { lastPredictionCount++; } else { lastPrediction = predictedExpression; lastPredictionCount = 1; } if (lastPredictionCount > expressionStabilizer) { currentPrediction = lastPrediction; expressionTime = Time.time; } if (currentPrediction >= 0 && currentPrediction <= classLabels.Length) { expression = classLabels[currentPrediction]; } else { expression = ""; } return(true); }
private float[] GetData(OpenSee.OpenSeeData t) { float[] data = new float[colsFull]; data[0] = t.rightEyeOpen; data[1] = t.leftEyeOpen; data[2] = t.translation.x; data[3] = t.translation.y; data[4] = t.translation.z; data[5] = t.rawQuaternion.x; data[6] = t.rawQuaternion.y; data[7] = t.rawQuaternion.z; data[8] = t.rawQuaternion.w; data[9] = t.rawEuler.x; data[10] = t.rawEuler.y; data[11] = t.rawEuler.z; /*for (int i = 0; i < 66; i++) * data[12 + i] = t.confidence[i]; * for (int i = 0; i < 66; i++) { * data[12 + 66 + 2 * i] = t.points[i].x / t.cameraResolution.x; * data[12 + 66 + 2 * i + 1] = t.points[i].y / t.cameraResolution.y; * }*/ for (int i = 0; i < 66; i++) { data[12 + /*66 + 2 * 66 +*/ 3 * i] = t.points3D[i].x; data[12 + /*66 + 2 * 66 +*/ 3 * i + 1] = t.points3D[i].y; data[12 + /*66 + 2 * 66 +*/ 3 * i + 2] = t.points3D[i].z; } data[12 + 66 * 3 + 0] = t.features.EyeLeft; data[12 + 66 * 3 + 1] = t.features.EyeRight; data[12 + 66 * 3 + 2] = t.features.EyebrowSteepnessLeft; data[12 + 66 * 3 + 3] = t.features.EyebrowUpDownLeft; data[12 + 66 * 3 + 4] = t.features.EyebrowQuirkLeft; data[12 + 66 * 3 + 5] = t.features.EyebrowSteepnessRight; data[12 + 66 * 3 + 6] = t.features.EyebrowUpDownRight; data[12 + 66 * 3 + 7] = t.features.EyebrowQuirkRight; data[12 + 66 * 3 + 8] = t.features.MouthCornerUpDownLeft; data[12 + 66 * 3 + 9] = t.features.MouthCornerInOutLeft; data[12 + 66 * 3 + 10] = t.features.MouthCornerUpDownRight; data[12 + 66 * 3 + 11] = t.features.MouthCornerInOutRight; data[12 + 66 * 3 + 12] = t.features.MouthOpen; data[12 + 66 * 3 + 13] = t.features.MouthWide; return(data); }
public bool CaptureCalibrationData() { if (openSee == null) { ResetInfo(); return(false); } if (recordingSkip < 1) { recordingSkip = 1; } if (calibrationExpression == "") { return(false); } if (expressions.ContainsKey(calibrationExpression) || expressions.Keys.Count < 10) { openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || openSeeData.time <= lastCapture) { return(false); } if (!expressions.ContainsKey(calibrationExpression)) { expressions.Add(calibrationExpression, new List <float[]>()); } List <float[]> list = expressions[calibrationExpression]; if (list.Count >= maxSamples) { percentRecorded = 100f; recording = allowOverRecording; if (!recording) { return(false); } } int skip = recordingSkip; if (percentRecorded >= 100f) { skip = overRecordingSkip; } if (frameCount % skip != 0) { return(true); } if (percentRecorded >= 100f) { list[rnd.Next(maxSamples)] = GetData(openSeeData); } else { list.Add(GetData(openSeeData)); } percentRecorded = 100f * (float)list.Count / (float)maxSamples; return(true); } else { recording = false; calibrationExpression = ""; return(false); } }
void Update() { if (!openSee) { return; } /*openSeeData = openSee.trackingData; * if (openSeeData == null || openSeeData.Length < 1) * return;*/ openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || (show3DPoints && openSeeData.fit3DError > openSee.maxFit3DError)) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; } else { return; } if (show3DPoints) { centerBall.gameObject.SetActive(false); for (int i = 0; i < total; i++) { if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); Vector3 pt = openSeeData.points3D[i]; pt.x = -pt.x; gameObjects[i].transform.localPosition = pt; if (i < 68) { renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); } else { if (i == 68) { gameObjects[i].transform.localRotation = openSeeData.rightGaze; } else { gameObjects[i].transform.localRotation = openSeeData.leftGaze; } } } else { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.cyan); } } if (applyTranslation) { Vector3 v = openSeeData.translation; v.x = -v.x; v.z = -v.z; transform.localPosition = v; } if (applyRotation) { Quaternion offset = Quaternion.Euler(0f, 0f, -90f); Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset; transform.localRotation = convertedQuaternion; } } else { centerBall.gameObject.SetActive(false); Vector3 center = new Vector3(0.0f, 0.0f, 0.0f); float minX = 10000.0f; float minY = 10000.0f; float maxX = -1.0f; float maxY = -1.0f; for (int i = 0; i < 66; i++) { float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; if (minX > x) { minX = x; } if (minY > y) { minY = y; } if (maxX < x) { maxX = x; } if (maxY < y) { maxY = y; } center += new Vector3(x, y, 0.0f); } center = center / 66; center = center - new Vector3(minX, minY, 0.0f); center.x = center.x / (maxX - minX); center.y = center.y / (maxX - minX); center.z = 0.5f; centerBall.transform.localPosition = center; for (int i = 0; i < 66; i++) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; Vector3 position = new Vector3(x, y, 0.0f); position = position - new Vector3(minX, minY, 0.0f); position.x = position.x / (maxX - minX); position.y = position.y / (maxX - minX); gameObjects[i].transform.localPosition = position; } } for (int i = 0; i < 68; i++) { if (lines[i] == -1) { continue; } if (!showLines || lineMaterial == null) { lineRenderers[i].enabled = false; } else { int a = i; int b = lines[i]; if (i == 66) { a = 48; } if (i == 67) { a = 53; } Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b])); lineRenderers[i].enabled = true; lineRenderers[i].widthMultiplier = lineWidth; //lineRenderers[i].material = lineMaterial; lineRenderers[i].material.SetColor("_Color", color); lineRenderers[i].startColor = color; lineRenderers[i].endColor = color; lineRenderers[i].SetPosition(0, gameObjects[a].transform.position); lineRenderers[i].SetPosition(1, gameObjects[b].transform.position); } } }
void Update() { if (!openSee) { return; } /*openSeeData = openSee.trackingData; * if (openSeeData == null || openSeeData.Length < 1) * return;*/ openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; } else { return; } if (show3DPoints) { centerBall.gameObject.SetActive(false); for (int i = 0; i < total; i++) { if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); Vector3 pt = openSeeData.points3D[i]; pt.x = -pt.x; gameObjects[i].transform.localPosition = pt; if (i < 68) { renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); } else { if (i == 68) { gameObjects[i].transform.localRotation = openSeeData.rightGaze; } else { gameObjects[i].transform.localRotation = openSeeData.leftGaze; } } } else { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.cyan); } } if (applyTranslation) { Vector3 v = openSeeData.translation; v.x = -v.x; v.z = -v.z; transform.localPosition = v; } if (applyRotation) { Quaternion offset = Quaternion.Euler(0f, 0f, -90f); Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset; transform.localRotation = convertedQuaternion; } } else { centerBall.gameObject.SetActive(false); Vector3 center = new Vector3(0.0f, 0.0f, 0.0f); float minX = 10000.0f; float minY = 10000.0f; float maxX = -1.0f; float maxY = -1.0f; for (int i = 0; i < 66; i++) { float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; if (minX > x) { minX = x; } if (minY > y) { minY = y; } if (maxX < x) { maxX = x; } if (maxY < y) { maxY = y; } center += new Vector3(x, y, 0.0f); } center = center / 66; center = center - new Vector3(minX, minY, 0.0f); center.x = center.x / (maxX - minX); center.y = center.y / (maxX - minX); center.z = 0.5f; centerBall.transform.localPosition = center; for (int i = 0; i < 66; i++) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; Vector3 position = new Vector3(x, y, 0.0f); position = position - new Vector3(minX, minY, 0.0f); position.x = position.x / (maxX - minX); position.y = position.y / (maxX - minX); gameObjects[i].transform.localPosition = position; } } }