// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); //detect face rects List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect(); foreach (var rect in detectResult) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); //detect face rects List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect(); if (detectResult.Count > 0) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]); if (isShowingFacePoints) { OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); } imagePoints.fromArray( new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye new Point(points [33].x, points [33].y), //nose new Point(points [48].x, points [48].y), //l mouth new Point(points [54].x, points [54].y) //r mouth , new Point(points [0].x, points [0].y), //l ear new Point(points [16].x, points [16].y) //r ear ); Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec); if (tvec.get(2, 0) [0] > 0) { if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 6.0) { if (isShowingEffects) { rightEye.SetActive(true); } } if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 6.0) { if (isShowingEffects) { leftEye.SetActive(true); } } if (isShowingHead) { head.SetActive(true); } if (isShowingAxes) { axes.SetActive(true); } float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y)); float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y)); if (mouseDistance > noseDistance / 5.0) { if (isShowingEffects) { mouth.SetActive(true); foreach (ParticleSystem ps in mouthParticleSystem) { ps.enableEmission = true; ps.startSize = 40 * (mouseDistance / noseDistance); } } } else { if (isShowingEffects) { foreach (ParticleSystem ps in mouthParticleSystem) { ps.enableEmission = false; } } } // position Vector4 pos = new Vector4((float)tvec.get(0, 0) [0], (float)tvec.get(1, 0) [0], (float)tvec.get(2, 0) [0], 0); // from OpenCV // right-handed coordinates system (OpenCV) to left-handed one (Unity) ARGameObject.transform.localPosition = new Vector3(pos.x, -pos.y, pos.z); // rotation Calib3d.Rodrigues(rvec, rotMat); Vector3 forward = new Vector3((float)rotMat.get(0, 2) [0], (float)rotMat.get(1, 2) [0], (float)rotMat.get(2, 2) [0]); // from OpenCV Vector3 up = new Vector3((float)rotMat.get(0, 1) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(2, 1) [0]); // from OpenCV // right-handed coordinates system (OpenCV) to left-handed one (Unity) Quaternion rot = Quaternion.LookRotation(new Vector3(forward.x, -forward.y, forward.z), new Vector3(up.x, -up.y, up.z)); ARGameObject.transform.localRotation = rot; // Apply Z axis inverted matrix. invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1)); transformationM = ARGameObject.transform.localToWorldMatrix * invertZM; // Apply camera transform matrix. transformationM = ARCamera.transform.localToWorldMatrix * transformationM; ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref transformationM); } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); //detect face rects List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect(); if (detectResult.Count > 0) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]); if (displayFacePoints) { OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); } imagePoints.fromArray( new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth) new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth) new Point(points [30].x, points [30].y), //nose (Nose top) new Point(points [48].x, points [48].y), //l mouth (Mouth breadth) new Point(points [54].x, points [54].y), //r mouth (Mouth breadth) new Point(points [0].x, points [0].y), //l ear (Bitragion breadth) new Point(points [16].x, points [16].y) //r ear (Bitragion breadth) ); // Estimate head pose. if (rvec == null || tvec == null) { rvec = new Mat(3, 1, CvType.CV_64FC1); tvec = new Mat(3, 1, CvType.CV_64FC1); Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec); } double tvec_z = tvec.get(2, 0) [0]; if (double.IsNaN(tvec_z) || tvec_z < 0) // if tvec is wrong data, do not use extrinsic guesses. { Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec); } else { Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE); } // Debug.Log (tvec.dump()); if (!double.IsNaN(tvec_z)) { if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 6.0) { if (displayEffects) { rightEye.SetActive(true); } } if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 6.0) { if (displayEffects) { leftEye.SetActive(true); } } if (displayHead) { head.SetActive(true); } if (displayAxes) { axes.SetActive(true); } float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y)); float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y)); if (mouseDistance > noseDistance / 5.0) { if (displayEffects) { mouth.SetActive(true); foreach (ParticleSystem ps in mouthParticleSystem) { var em = ps.emission; em.enabled = true; ps.startSize = 40 * (mouseDistance / noseDistance); } } } else { if (displayEffects) { foreach (ParticleSystem ps in mouthParticleSystem) { var em = ps.emission; em.enabled = false; } } } Calib3d.Rodrigues(rvec, rotMat); transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0) [0], (float)rotMat.get(0, 1) [0], (float)rotMat.get(0, 2) [0], (float)tvec.get(0, 0) [0])); transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(1, 2) [0], (float)tvec.get(1, 0) [0])); transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0) [0], (float)rotMat.get(2, 1) [0], (float)rotMat.get(2, 2) [0], (float)tvec.get(2, 0) [0])); transformationM.SetRow(3, new Vector4(0, 0, 0, 1)); // right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z axis inverted matrix. ARM = ARM * invertZM; if (shouldMoveARCamera) { ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse; ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM); } else { ARM = ARCamera.transform.localToWorldMatrix * ARM; ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM); } } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } }