private void Run() { faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); #if UNITY_ANDROID && !UNITY_EDITOR // Set the requestedFPS parameter to avoid the problem of the WebCamTexture image becoming low light on some Android devices. (Pixel, pixel 2) // https://forum.unity.com/threads/android-webcamtexture-in-low-light-only-some-models.520656/ // https://forum.unity.com/threads/released-opencv-for-unity.277080/page-33#post-3445178 rearCameraRequestedFPS = webCamTextureToMatHelper.requestedFPS; if (webCamTextureToMatHelper.requestedIsFrontFacing) { webCamTextureToMatHelper.requestedFPS = 15; webCamTextureToMatHelper.Initialize(); } else { webCamTextureToMatHelper.Initialize(); } #else webCamTextureToMatHelper.Initialize(); #endif }
private void Run() { if (string.IsNullOrEmpty(dlibShapePredictorFilePath)) { Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); webCamTextureToMatHelper.Initialize(); }
private void Run() { if (string.IsNullOrEmpty(dlibShapePredictorFilePath)) { Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); #if UNITY_ANDROID && !UNITY_EDITOR // Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2). webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true; #endif webCamTextureToMatHelper.Initialize(); }
private void Run() { if (string.IsNullOrEmpty(dlibShapePredictorFilePath)) { Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); if (string.IsNullOrEmpty(sourceToMatHelper.requestedVideoFilePath)) { sourceToMatHelper.requestedVideoFilePath = VIDEO_FILENAME; } sourceToMatHelper.outputColorFormat = VideoCaptureToMatHelper.ColorFormat.RGB; sourceToMatHelper.Initialize(); }
// Use this for initialization protected override void Start() { base.Start(); displayCameraPreviewToggle.isOn = displayCameraPreview; useSeparateDetectionToggle.isOn = useSeparateDetection; displayAxesToggle.isOn = displayAxes; displayHeadToggle.isOn = displayHead; displayEffectsToggle.isOn = displayEffects; enableOpticalFlowFilterToggle.isOn = enableOpticalFlowFilter; enableLowPassFilterToggle.isOn = enableLowPassFilter; imageOptimizationHelper = gameObject.GetComponent <ImageOptimizationHelper> (); webCamTextureToMatHelper = gameObject.GetComponent <HololensCameraStreamToMatHelper> (); #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API webCamTextureToMatHelper.frameMatAcquired += OnFrameMatAcquired; #endif webCamTextureToMatHelper.Initialize(); rectangleTracker = new RectangleTracker(); faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("sp_human_face_68.dat")); // faceLandmarkDetector = new FaceLandmarkDetector (DlibFaceLandmarkDetector.Utils.getFilePath ("sp_human_face_68_for_mobile.dat")); // The coordinates of the detection object on the real world space connected with the pixel coordinates.(mm) objectPoints = new MatOfPoint3f( new Point3(-34, 90, 83), //l eye (Interpupillary breadth) new Point3(34, 90, 83), //r eye (Interpupillary breadth) new Point3(0.0, 50, 120), //nose (Nose top) new Point3(-26, 15, 83), //l mouse (Mouth breadth) new Point3(26, 15, 83), //r mouse (Mouth breadth) new Point3(-79, 90, 0.0), //l ear (Bitragion breadth) new Point3(79, 90, 0.0) //r ear (Bitragion breadth) ); imagePoints = new MatOfPoint2f(); rotMat = new Mat(3, 3, CvType.CV_64FC1); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter.diffDlib /= imageOptimizationHelper.downscaleRatio; }
// Update is called once per frame void Update() { // loop play. if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT)) { capture.set(Videoio.CAP_PROP_POS_FRAMES, 0); } if (capture.grab()) { capture.retrieve(rgbMat, 0); Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB); //Debug.Log ("Mat toString " + rgbMat.ToString ()); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face tracking. rectangleTracker.UpdateTrackedObjects(detectResult); List <TrackedRect> trackedRects = new List <TrackedRect> (); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict [openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict [openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // create LUT texture. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { faceMaskColorCorrector.CreateLUTTex(openCVRect.id); } else if (openCVRect.state == TrackedState.DELETED) { faceMaskColorCorrector.DeleteLUTTex(openCVRect.id); } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); for (int i = 0; i < trackedRects.Count; i++) { TrackedRect tr = trackedRects [i]; UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED) { opticalFlowFilterDict [tr.id].Process(rgbMat, points, points); lowPassFilterDict [tr.id].Process(rgbMat, points, points); } } landmarkPoints.Add(points); } // face masking. if (faceMaskTexture != null && landmarkPoints.Count >= 1) // Apply face masking between detected faces and a face mask image. { float maskImageWidth = faceMaskTexture.width; float maskImageHeight = faceMaskTexture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, faceMaskTexture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, faceMaskMat, rgbMat, faceLandmarkPointsInMask, landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } else if (landmarkPoints.Count >= 1) // Apply face masking between detected faces. { float maskImageWidth = texture.width; float maskImageHeight = texture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, texture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], landmarkPoints [0], maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, rgbMat, rgbMat, landmarkPoints [0], landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2); } for (int i = 0; i < trackedRects.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // draw face points. if (displayDebugFacePoints) { for (int i = 0; i < landmarkPoints.Count; i++) { OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2); } } // display face mask image. if (faceMaskTexture != null && faceMaskMat != null) { if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2); } if (displayDebugFacePoints) { OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2); } float scale = (rgbMat.width() / 4f) / faceMaskMat.width(); float tx = rgbMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePointsToggle) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } // Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture); } }
// Update is called once per frame void Update() { if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame()) { Mat rgbMat = sourceToMatHelper.GetMat(); // detect faces. List <Rect> detectResult = new List <Rect>(); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. foreach (Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face swapping. if (landmarkPoints.Count >= 2) { int ann = 0, bob = 1; for (int i = 0; i < landmarkPoints.Count - 1; i += 2) { ann = i; bob = i + 1; faceSwapper.SwapFaces(rgbMat, landmarkPoints[ann], landmarkPoints[bob], 1); } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } //Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <Rect> detectResult; DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter); // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face changing. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { // Apply face changing between detected faces and a mask image. if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f) { DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter); if (detectResult.Count >= 1) { detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height); } } if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect); faceChanger.SetTargetImage(rgbaMat); for (int i = 0; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1); } faceChanger.ChangeFace(); if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2); } } } else if (landmarkPoints.Count >= 2) { // Apply face changing between detected faces. faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1); } faceChanger.ChangeFace(); } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // display face mask image. if (faceMaskMat != null) { float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePoints) { OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture); } }
private void Run() { if (string.IsNullOrEmpty(dlibShapePredictorFilePath)) { Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); rgbMat = new Mat(); capture = new VideoCapture(); capture.open(video_filepath); if (!capture.isOpened()) { Debug.LogError("capture.isOpened() is false. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT)); Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC)); Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES)); Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO)); Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT)); Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS)); Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH)); Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT)); double ext = capture.get(Videoio.CAP_PROP_FOURCC); Debug.Log("CAP_PROP_FOURCC: " + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24)); capture.grab(); capture.retrieve(rgbMat, 0); int frameWidth = rgbMat.cols(); int frameHeight = rgbMat.rows(); texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false); gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1); float widthScale = (float)Screen.width / (float)frameWidth; float heightScale = (float)Screen.height / (float)frameHeight; if (widthScale < heightScale) { Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = (float)frameHeight / 2; } capture.set(Videoio.CAP_PROP_POS_FRAMES, 0); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; if (fpsMonitor != null) { fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName); fpsMonitor.Add("width", frameWidth.ToString()); fpsMonitor.Add("height", frameHeight.ToString()); fpsMonitor.Add("orientation", Screen.orientation.ToString()); } }
private void Run() { faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath); lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()); rgbMat = new Mat(); capture = new VideoCapture(); capture.open(dance_avi_filepath); if (capture.isOpened()) { Debug.Log("capture.isOpened() true"); } else { Debug.Log("capture.isOpened() false"); } Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT)); Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT)); Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC)); Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES)); Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO)); Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT)); Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS)); Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH)); Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT)); capture.grab(); capture.retrieve(rgbMat, 0); int frameWidth = rgbMat.cols(); int frameHeight = rgbMat.rows(); texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false); gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1); float widthScale = (float)Screen.width / (float)frameWidth; float heightScale = (float)Screen.height / (float)frameHeight; if (widthScale < heightScale) { Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = (float)frameHeight / 2; } capture.set(Videoio.CAP_PROP_POS_FRAMES, 0); gameObject.GetComponent <Renderer> ().material.mainTexture = texture; if (fpsMonitor != null) { fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName); fpsMonitor.Add("width", frameWidth.ToString()); fpsMonitor.Add("height", frameHeight.ToString()); fpsMonitor.Add("orientation", Screen.orientation.ToString()); } }