// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); } // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } // face tracking. rectangleTracker.UpdateTrackedObjects(detectResult); List <TrackedRect> trackedRects = new List <TrackedRect> (); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict [openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict [openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // create LUT texture. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { faceMaskColorCorrector.CreateLUTTex(openCVRect.id); } else if (openCVRect.state == TrackedState.DELETED) { faceMaskColorCorrector.DeleteLUTTex(openCVRect.id); } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); for (int i = 0; i < trackedRects.Count; i++) { TrackedRect tr = trackedRects [i]; UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED) { opticalFlowFilterDict [tr.id].Process(rgbaMat, points, points); lowPassFilterDict [tr.id].Process(rgbaMat, points, points); } } if (extendForehead) { AddForeheadPoints(points); } landmarkPoints.Add(points); } // face masking. if (faceMaskTexture != null && landmarkPoints.Count >= 1) // Apply face masking between detected faces and a face mask image. { float maskImageWidth = faceMaskTexture.width; float maskImageHeight = faceMaskTexture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, faceMaskTexture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, faceMaskMat, rgbaMat, faceLandmarkPointsInMask, landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } else if (landmarkPoints.Count >= 1) // Apply face masking between detected faces. { float maskImageWidth = texture.width; float maskImageHeight = texture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, texture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], landmarkPoints [0], maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, rgbaMat, rgbaMat, landmarkPoints [0], landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); } for (int i = 0; i < trackedRects.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 255, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // draw face points. if (displayDebugFacePoints) { for (int i = 0; i < landmarkPoints.Count; i++) { DrawFaceLandmark(rgbaMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2); } } // display face mask image. if (faceMaskTexture != null && faceMaskMat != null) { if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2); } if (displayDebugFacePoints) { DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2); } float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePointsToggle) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } // Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); } // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } // face traking. rectangleTracker.UpdateTrackedObjects(detectResult); List <TrackedRect> trackedRects = new List <TrackedRect> (); rectangleTracker.GetObjects(trackedRects, true); // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); for (int i = 0; i < trackedRects.Count; i++) { TrackedRect tr = trackedRects [i]; UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); if (extendForehead) { AddForeheadPoints(points); } landmarkPoints.Add(points); } // face masking. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); float imageWidth = meshOverlay.width; float imageHeight = meshOverlay.height; float maskImageWidth = faceMaskTexture.width; float maskImageHeight = faceMaskTexture.height; TrackedRect tr; TrackedMesh tm; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, faceMaskTexture); } if (tr.state < TrackedState.DELETED) { tm = meshOverlay.GetObjectById(tr.id); Vector3[] vertices = tm.meshFilter.mesh.vertices; if (vertices.Length == landmarkPoints [i].Count) { for (int j = 0; j < vertices.Length; j++) { vertices [j].x = landmarkPoints [i] [j].x / imageWidth - 0.5f; vertices [j].y = 0.5f - landmarkPoints [i] [j].y / imageHeight; } } Vector2[] uv = tm.meshFilter.mesh.uv; if (uv.Length == faceLandmarkPointsInMask.Count) { for (int jj = 0; jj < uv.Length; jj++) { uv [jj].x = faceLandmarkPointsInMask [jj].x / maskImageWidth; uv [jj].y = (maskImageHeight - faceLandmarkPointsInMask [jj].y) / maskImageHeight; } } meshOverlay.UpdateObject(tr.id, vertices, null, uv); if (tr.numFramesNotDetected > 3) { tm.material.SetFloat(shader_FadeID, 1f); } else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3) { tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected); } else { tm.material.SetFloat(shader_FadeID, 0.3f); } // filter non frontal faces. if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { tm.material.SetFloat(shader_FadeID, 1f); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } else if (landmarkPoints.Count >= 1) { float imageWidth = meshOverlay.width; float imageHeight = meshOverlay.height; float maskImageWidth = texture.width; float maskImageHeight = texture.height; TrackedRect tr; TrackedMesh tm; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, texture); } if (tr.state < TrackedState.DELETED) { tm = meshOverlay.GetObjectById(tr.id); Vector3[] vertices = tm.meshFilter.mesh.vertices; if (vertices.Length == landmarkPoints [i].Count) { for (int j = 0; j < vertices.Length; j++) { vertices [j].x = landmarkPoints[i][j].x / imageWidth - 0.5f; vertices [j].y = 0.5f - landmarkPoints[i][j].y / imageHeight; } } Vector2[] uv = tm.meshFilter.mesh.uv; if (uv.Length == landmarkPoints [0].Count) { for (int jj = 0; jj < uv.Length; jj++) { uv [jj].x = landmarkPoints[0][jj].x / maskImageWidth; uv [jj].y = (maskImageHeight - landmarkPoints[0][jj].y) / maskImageHeight; } } meshOverlay.UpdateObject(tr.id, vertices, null, uv); if (tr.numFramesNotDetected > 3) { tm.material.SetFloat(shader_FadeID, 1f); } else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3) { tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected); } else { tm.material.SetFloat(shader_FadeID, 0.3f); } // filter nonsfrontal faces. if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { tm.material.SetFloat(shader_FadeID, 1f); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); } for (int i = 0; i < trackedRects.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 255, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // draw face points. if (displayDebugFacePoints) { for (int i = 0; i < landmarkPoints.Count; i++) { DrawFaceLandmark(rgbaMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2); } } // display face mask image. if (faceMaskTexture != null && faceMaskMat != null) { if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2); } if (displayDebugFacePoints) { DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2); } float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors); } }