public bool PredictExpression() { if (openSee == null) { return(false); } if (!simpleMode && !(modelReady && model.Ready())) { ResetInfo(); return(false); } openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || openSeeData.time <= lastCapture) { return(false); } if (simpleMode) { ThresholdDetection(); return(true); } float[] faceData = GetData(openSeeData); float[] predictionData = new float[cols]; for (int i = 0; i < cols; i++) { predictionData[i] = faceData[indices[i]]; } float[] prediction = model.Predict(predictionData, out probabilities, 1); int predictedExpression = (int)Mathf.Round(prediction[0]); if (predictedExpression == lastPrediction) { lastPredictionCount++; } else { lastPrediction = predictedExpression; lastPredictionCount = 1; } if (lastPredictionCount > expressionStabilizer) { currentPrediction = lastPrediction; expressionTime = Time.time; } if (currentPrediction >= 0 && currentPrediction <= classLabels.Length) { expression = classLabels[currentPrediction]; } else { expression = ""; } return(true); }
void RunUpdate() { var openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; trackingFrames++; if (trackingFrames < skipFirst) { return; } } else { Interpolate(); return; } Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w); Vector3 t = openSeeData.translation; t.x = -t.x; t.z = -t.z; Vector3 convertedTranslation = new Vector3(t.x, t.y, t.z); // Check for angular outliers if (gotAccepted) { float angularDifference = Quaternion.Angle(lastAccepted, convertedQuaternion); if (angularDifference >= outlierThresholdAngle) { if (!skipped) { if (outlierSkipPeriod >= 0.0000001f) { skipStartTime = Time.time; skipped = true; } } else { if (Time.time > skipStartTime + outlierSkipPeriod) { lastAccepted = convertedQuaternion; skipped = false; } } } else { skipped = false; lastAccepted = convertedQuaternion; } } else { lastAccepted = convertedQuaternion; skipped = false; } gotAccepted = true; if (skipped) { outlierSkips += 1; return; } // Check for positional outliers bool keepOffset = false; if (gotAcceptedPosition) { float distance = Vector3.Distance(t, lastAcceptedPosition); if (distance > outlierThresholdDistance) { if (!skippedPosition) { if (outlierSkipPeriodDistance >= 0.0000001f) { skipStartTimePosition = Time.time; skippedPosition = true; } } else { if (Time.time > skipStartTimePosition + outlierSkipPeriodDistance) { lastAcceptedPosition = t; skippedPosition = false; if (outlierRecalibrate) { calibrate = true; keepOffset = outlierKeepOffset; outlierCalibrations += 1; } } } } else { skippedPosition = false; lastAcceptedPosition = t; } } else { skippedPosition = false; lastAcceptedPosition = t; } gotAcceptedPosition = true; if (skippedPosition) { outlierSkipsDistance += 1; return; } if (calibrate) { dR = convertedQuaternion; dR = Quaternion.Inverse(dR); dT = t; rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z); translationOffset = new Vector3(dT.x, dT.y, dT.z); lastAcceptedPosition = translationOffset; skippedPosition = false; } if (mirrorMotion != lastMirror || (mirrorMotion && calibrate)) { dR = Quaternion.Inverse(MirrorQuaternion(Quaternion.Inverse(dR))); dT = MirrorTranslation(dT); lastMirror = mirrorMotion; } if (calibrate && keepOffset) { dR = Quaternion.Inverse(Quaternion.Inverse(transform.localRotation) * Quaternion.Inverse(dR)); dT -= transform.localPosition; } calibrate = false; if (mirrorMotion) { convertedQuaternion = MirrorQuaternion(convertedQuaternion); t = MirrorTranslation(t); } if (interpolateState > 1) { avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f); } interpolationCount = 0; averageInterpolations = avgInterps; lastT = transform.localPosition; lastR = transform.localRotation; if (interpolateState > 0) { currentT = Vector3.Lerp(currentT, (t - dT) * translationScale, 1f - smooth); currentR = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth); } else { currentT = (t - dT) * translationScale; currentR = convertedQuaternion * dR; } if (interpolateState < 2) { interpolateState++; } if (interpolate) { Interpolate(); } else { transform.localPosition = Vector3.Lerp(transform.localPosition, (t - dT) * translationScale, 1f - smooth); transform.localRotation = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth); } if (kinematicInterpolation != null && kinematicInterpolation.gameObject != gameObject) { kinematicInterpolation.UpdateKI(updated, transform.localPosition, transform.localRotation); } if (driftBack) { if (mirrorMotion) { dT = Vector3.Lerp(dT, MirrorTranslation(convertedTranslation), driftFactor); dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor); } else { dT = Vector3.Lerp(dT, convertedTranslation, driftFactor); dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor); } rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z); translationOffset = new Vector3(dT.x, dT.y, dT.z); } }
void RunUpdate() { var openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || openSeeData.fit3DError > openSee.maxFit3DError) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; } else { Interpolate(); return; } Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w); Vector3 t = openSeeData.translation; t.x = -t.x; t.z = -t.z; Vector3 convertedTranslation = new Vector3(t.x, t.y, t.z); if (calibrate) { dR = convertedQuaternion; dR = Quaternion.Inverse(dR); dT = t; rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z); translationOffset = new Vector3(dT.x, dT.y, dT.z); } if (mirrorMotion != lastMirror || (mirrorMotion && calibrate)) { dR = Quaternion.Inverse(MirrorQuaternion(Quaternion.Inverse(dR))); dT = MirrorTranslation(dT); lastMirror = mirrorMotion; } calibrate = false; if (mirrorMotion) { convertedQuaternion = MirrorQuaternion(convertedQuaternion); t = MirrorTranslation(t); } if (interpolateState > 1) { avgInterps = Mathf.Lerp(avgInterps, (float)interpolationCount, 0.15f); } interpolationCount = 0; averageInterpolations = avgInterps; lastT = currentT; lastR = currentR; if (interpolateState > 0) { currentT = Vector3.Lerp(currentT, (t - dT) * translationScale, 1f - smooth); currentR = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth); } else { currentT = (t - dT) * translationScale; currentR = convertedQuaternion * dR; } if (interpolateState < 2) { interpolateState++; } if (interpolate) { Interpolate(); } else { transform.localPosition = Vector3.Lerp(transform.localPosition, (t - dT) * translationScale, 1f - smooth); transform.localRotation = Quaternion.Lerp(transform.localRotation, convertedQuaternion * dR, 1f - smooth); } if (driftBack) { if (mirrorMotion) { dT = Vector3.Lerp(dT, MirrorTranslation(convertedTranslation), driftFactor); dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor); } else { dT = Vector3.Lerp(dT, convertedTranslation, driftFactor); dR = Quaternion.Lerp(dR, Quaternion.Inverse(convertedQuaternion), driftFactor); } rotationOffset = new Vector3(dR.eulerAngles.x, dR.eulerAngles.y, dR.eulerAngles.z); translationOffset = new Vector3(dT.x, dT.y, dT.z); } }
void Update() { if (!openSee) { return; } /*openSeeData = openSee.trackingData; * if (openSeeData == null || openSeeData.Length < 1) * return;*/ openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null || (show3DPoints && openSeeData.fit3DError > openSee.maxFit3DError)) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; } else { return; } if (show3DPoints) { centerBall.gameObject.SetActive(false); for (int i = 0; i < total; i++) { if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); Vector3 pt = openSeeData.points3D[i]; pt.x = -pt.x; gameObjects[i].transform.localPosition = pt; if (i < 68) { renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); } else { if (i == 68) { gameObjects[i].transform.localRotation = openSeeData.rightGaze; } else { gameObjects[i].transform.localRotation = openSeeData.leftGaze; } } } else { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.cyan); } } if (applyTranslation) { Vector3 v = openSeeData.translation; v.x = -v.x; v.z = -v.z; transform.localPosition = v; } if (applyRotation) { Quaternion offset = Quaternion.Euler(0f, 0f, -90f); Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset; transform.localRotation = convertedQuaternion; } } else { centerBall.gameObject.SetActive(false); Vector3 center = new Vector3(0.0f, 0.0f, 0.0f); float minX = 10000.0f; float minY = 10000.0f; float maxX = -1.0f; float maxY = -1.0f; for (int i = 0; i < 66; i++) { float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; if (minX > x) { minX = x; } if (minY > y) { minY = y; } if (maxX < x) { maxX = x; } if (maxY < y) { maxY = y; } center += new Vector3(x, y, 0.0f); } center = center / 66; center = center - new Vector3(minX, minY, 0.0f); center.x = center.x / (maxX - minX); center.y = center.y / (maxX - minX); center.z = 0.5f; centerBall.transform.localPosition = center; for (int i = 0; i < 66; i++) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; Vector3 position = new Vector3(x, y, 0.0f); position = position - new Vector3(minX, minY, 0.0f); position.x = position.x / (maxX - minX); position.y = position.y / (maxX - minX); gameObjects[i].transform.localPosition = position; } } for (int i = 0; i < 68; i++) { if (lines[i] == -1) { continue; } if (!showLines || lineMaterial == null) { lineRenderers[i].enabled = false; } else { int a = i; int b = lines[i]; if (i == 66) { a = 48; } if (i == 67) { a = 53; } Color color = Color.Lerp(Color.red, Color.green, Mathf.Lerp(0.5f, openSeeData.confidence[a], openSeeData.confidence[b])); lineRenderers[i].enabled = true; lineRenderers[i].widthMultiplier = lineWidth; //lineRenderers[i].material = lineMaterial; lineRenderers[i].material.SetColor("_Color", color); lineRenderers[i].startColor = color; lineRenderers[i].endColor = color; lineRenderers[i].SetPosition(0, gameObjects[a].transform.position); lineRenderers[i].SetPosition(1, gameObjects[b].transform.position); } } }
void Update() { if (!openSee) { return; } /*openSeeData = openSee.trackingData; * if (openSeeData == null || openSeeData.Length < 1) * return;*/ openSeeData = openSee.GetOpenSeeData(faceId); if (openSeeData == null) { return; } if (openSeeData.time > updated) { updated = openSeeData.time; } else { return; } if (show3DPoints) { centerBall.gameObject.SetActive(false); for (int i = 0; i < total; i++) { if (openSeeData.got3DPoints && (i >= 68 || openSeeData.confidence[i] > minConfidence)) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); Vector3 pt = openSeeData.points3D[i]; pt.x = -pt.x; gameObjects[i].transform.localPosition = pt; if (i < 68) { renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); } else { if (i == 68) { gameObjects[i].transform.localRotation = openSeeData.rightGaze; } else { gameObjects[i].transform.localRotation = openSeeData.leftGaze; } } } else { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.cyan); } } if (applyTranslation) { Vector3 v = openSeeData.translation; v.x = -v.x; v.z = -v.z; transform.localPosition = v; } if (applyRotation) { Quaternion offset = Quaternion.Euler(0f, 0f, -90f); Quaternion convertedQuaternion = new Quaternion(-openSeeData.rawQuaternion.y, -openSeeData.rawQuaternion.x, openSeeData.rawQuaternion.z, openSeeData.rawQuaternion.w) * offset; transform.localRotation = convertedQuaternion; } } else { centerBall.gameObject.SetActive(false); Vector3 center = new Vector3(0.0f, 0.0f, 0.0f); float minX = 10000.0f; float minY = 10000.0f; float maxX = -1.0f; float maxY = -1.0f; for (int i = 0; i < 66; i++) { float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; if (minX > x) { minX = x; } if (minY > y) { minY = y; } if (maxX < x) { maxX = x; } if (maxY < y) { maxY = y; } center += new Vector3(x, y, 0.0f); } center = center / 66; center = center - new Vector3(minX, minY, 0.0f); center.x = center.x / (maxX - minX); center.y = center.y / (maxX - minX); center.z = 0.5f; centerBall.transform.localPosition = center; for (int i = 0; i < 66; i++) { Renderer renderer = gameObjects[i].GetComponent <Renderer>(); renderer.material.SetColor("_Color", Color.Lerp(Color.red, Color.green, openSeeData.confidence[i])); float x = openSeeData.points[i].x; float y = -openSeeData.points[i].y; Vector3 position = new Vector3(x, y, 0.0f); position = position - new Vector3(minX, minY, 0.0f); position.x = position.x / (maxX - minX); position.y = position.y / (maxX - minX); gameObjects[i].transform.localPosition = position; } } }