// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face traking. if (enableTracking) { rectangleTracker.UpdateTrackedObjects(detectResult); detectResult = new List <OpenCVForUnity.Rect> (); rectangleTracker.GetObjects(detectResult, true); } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); foreach (var openCVRect in detectResult) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); landmarkPoints.Add(points); } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face swapping. if (landmarkPoints.Count >= 2) { int ann = 0, bob = 1; for (int i = 0; i < landmarkPoints.Count - 1; i += 2) { ann = i; bob = i + 1; faceSwapper.SwapFaces(rgbaMat, landmarkPoints [ann], landmarkPoints [bob], 1); } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face traking. if (enableTracking) { rectangleTracker.UpdateTrackedObjects(detectResult); detectResult = new List <OpenCVForUnity.Rect> (); rectangleTracker.GetObjects(detectResult, true); } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); foreach (var openCVRect in detectResult) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); landmarkPoints.Add(points); } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face changeing. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f) { if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); if (result.Count >= 1) { detectedFaceRect = result [0]; } } else { using (Mat grayMat = new Mat()) using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { //convert image to greyscale Imgproc.cvtColor(faceMaskMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); //detectResult = faces.toList (); List <OpenCVForUnity.Rect> faceList = faces.toList(); if (faceList.Count >= 1) { detectedFaceRect = new UnityEngine.Rect(faceList [0].x, faceList [0].y, faceList [0].width, faceList [0].height); // Adjust to Dilb's result. detectedFaceRect.y += detectedFaceRect.height * 0.1f; } } } } if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect); faceChanger.SetTargetImage(rgbaMat); for (int i = 0; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints [i], 1); } faceChanger.ChangeFace(); if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2); } } } else if (landmarkPoints.Count >= 2) { faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints [0], landmarkPoints [i], 1); } faceChanger.ChangeFace(); } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // display face mask image. if (faceMaskMat != null) { float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <Rect> detectResult; DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter); // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face changing. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { // Apply face changing between detected faces and a mask image. if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f) { DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter); if (detectResult.Count >= 1) { detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height); } } if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect); faceChanger.SetTargetImage(rgbaMat); for (int i = 0; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1); } faceChanger.ChangeFace(); if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2); } } } else if (landmarkPoints.Count >= 2) { // Apply face changing between detected faces. faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1); } faceChanger.ChangeFace(); } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // display face mask image. if (faceMaskMat != null) { float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePoints) { OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <Rect> detectResult = new List <Rect>(); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. foreach (Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face swapping. if (landmarkPoints.Count >= 2) { int ann = 0, bob = 1; for (int i = 0; i < landmarkPoints.Count - 1; i += 2) { ann = i; bob = i + 1; faceSwapper.SwapFaces(rgbaMat, landmarkPoints[ann], landmarkPoints[bob], 1); } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture); } }