// Update is called once per frame void Update() { // loop play. if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT)) { capture.set(Videoio.CAP_PROP_POS_FRAMES, 0); } if (capture.grab()) { capture.retrieve(rgbMat, 0); Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB); //Debug.Log ("Mat toString " + rgbMat.ToString ()); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face traking. rectangleTracker.UpdateTrackedObjects(detectResult); List <TrackedRect> trackedRects = new List <TrackedRect> (); rectangleTracker.GetObjects(trackedRects, true); // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); for (int i = 0; i < trackedRects.Count; i++) { TrackedRect tr = trackedRects [i]; UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); landmarkPoints.Add(points); } // face masking. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); float imageWidth = meshOverlay.width; float imageHeight = meshOverlay.height; float maskImageWidth = faceMaskTexture.width; float maskImageHeight = faceMaskTexture.height; TrackedRect tr; TrackedMesh tm; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, faceMaskTexture); } if (tr.state < TrackedState.DELETED) { tm = meshOverlay.GetObjectById(tr.id); Vector3[] vertices = tm.meshFilter.mesh.vertices; if (vertices.Length == landmarkPoints [i].Count) { for (int j = 0; j < vertices.Length; j++) { vertices [j].x = landmarkPoints [i] [j].x / imageWidth - 0.5f; vertices [j].y = 0.5f - landmarkPoints [i] [j].y / imageHeight; } } Vector2[] uv = tm.meshFilter.mesh.uv; if (uv.Length == faceLandmarkPointsInMask.Count) { for (int jj = 0; jj < uv.Length; jj++) { uv [jj].x = faceLandmarkPointsInMask [jj].x / maskImageWidth; uv [jj].y = (maskImageHeight - faceLandmarkPointsInMask [jj].y) / maskImageHeight; } } meshOverlay.UpdateObject(tr.id, vertices, null, uv); if (tr.numFramesNotDetected > 3) { tm.material.SetFloat(shader_FadeID, 1f); } else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3) { tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected); } else { tm.material.SetFloat(shader_FadeID, 0.3f); } // filter non frontal faces. if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { tm.material.SetFloat(shader_FadeID, 1f); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } else if (landmarkPoints.Count >= 1) { float imageWidth = meshOverlay.width; float imageHeight = meshOverlay.height; float maskImageWidth = texture.width; float maskImageHeight = texture.height; TrackedRect tr; TrackedMesh tm; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, texture); } if (tr.state < TrackedState.DELETED) { tm = meshOverlay.GetObjectById(tr.id); Vector3[] vertices = tm.meshFilter.mesh.vertices; if (vertices.Length == landmarkPoints [i].Count) { for (int j = 0; j < vertices.Length; j++) { vertices [j].x = landmarkPoints[i][j].x / imageWidth - 0.5f; vertices [j].y = 0.5f - landmarkPoints[i][j].y / imageHeight; } } Vector2[] uv = tm.meshFilter.mesh.uv; if (uv.Length == landmarkPoints [0].Count) { for (int jj = 0; jj < uv.Length; jj++) { uv [jj].x = landmarkPoints[0][jj].x / maskImageWidth; uv [jj].y = (maskImageHeight - landmarkPoints[0][jj].y) / maskImageHeight; } } meshOverlay.UpdateObject(tr.id, vertices, null, uv); if (tr.numFramesNotDetected > 3) { tm.material.SetFloat(shader_FadeID, 1f); } else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3) { tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected); } else { tm.material.SetFloat(shader_FadeID, 0.3f); } // filter non frontal faces. if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { tm.material.SetFloat(shader_FadeID, 1f); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2); } for (int i = 0; i < trackedRects.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // draw face points. if (displayDebugFacePoints) { for (int i = 0; i < landmarkPoints.Count; i++) { OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2); } } // display face mask image. if (faceMaskTexture != null && faceMaskMat != null) { if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2); } if (displayDebugFacePoints) { OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2); } float scale = (rgbMat.width() / 4f) / faceMaskMat.width(); float tx = rgbMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); } Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbMat, texture); } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); position *= 1.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = bgraMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } int len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; if (enableDownScale) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } // draw face rect Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_BLUE, 1); } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GREEN, 1); } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; if (enableDownScale) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } // draw face rect Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2); } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
// Update is called once per frame void Update() { if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame()) { Mat rgbMat = sourceToMatHelper.GetMat(); // detect faces. List <Rect> detectResult = new List <Rect>(); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. foreach (Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face swapping. if (landmarkPoints.Count >= 2) { int ann = 0, bob = 1; for (int i = 0; i < landmarkPoints.Count - 1; i += 2) { ann = i; bob = i + 1; faceSwapper.SwapFaces(rgbMat, landmarkPoints[ann], landmarkPoints[bob], 1); } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } //Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture); } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); Mat bgraMat4preview = null; if (displayCameraPreview) { bgraMat4preview = new Mat(); downScaleFrameMat.copyTo(bgraMat4preview); } List <Vector2> points = null; Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); if (rects.Length > 0) { OpenCVForUnity.Rect rect = rects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(bgraMat, points, points, false); } if (displayCameraPreview && bgraMat4preview != null) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2); } } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (resultObjects.Count > 0) { OpenCVForUnity.Rect rect = resultObjects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(bgraMat, points, points, false); } if (displayCameraPreview && bgraMat4preview != null) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2); } } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && bgraMat4preview != null) { OpenCVForUnity.Utils.fastMatToTexture2D(bgraMat4preview, texture); } if (points != null) { UpdateARHeadTransform(points, cameraToWorldMatrix); } bgraMat.Dispose(); if (bgraMat4preview != null) { bgraMat4preview.Dispose(); } }, false); }
// Update is called once per frame void Update() { lock (sync) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enable && !detecting) { detecting = true; grayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } if (!isShowingWebCamImage) { // fill all black. Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!isUsingSeparateDetection) { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); // rects = rectangleTracker.CreateCorrectionBySpeedOfRects (); for (int i = 0; i < rects.Length; i++) { //Debug.Log ("detected face[" + i + "] " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2); } } else { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.toArray(); rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1); } } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //Debug.Log ("detected face[" + i + "] " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2); } } Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; texture.wrapMode = TextureWrapMode.Clamp; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <Rect> detectResult; DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter); // face tracking. List <TrackedRect> trackedRects = new List <TrackedRect>(); rectangleTracker.UpdateTrackedObjects(detectResult); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict[openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict[openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in trackedRects) { if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points); lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points); } landmarkPoints.Add(points); } } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { trackedRects.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face changing. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { // Apply face changing between detected faces and a mask image. if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f) { DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter); if (detectResult.Count >= 1) { detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height); } } if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect); faceChanger.SetTargetImage(rgbaMat); for (int i = 0; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1); } faceChanger.ChangeFace(); if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2); } } } else if (landmarkPoints.Count >= 2) { // Apply face changing between detected faces. faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1); } faceChanger.ChangeFace(); } // draw face rects. if (displayFaceRects) { for (int i = 0; i < trackedRects.Count; i++) { Rect openCVRect = trackedRects[i]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // display face mask image. if (faceMaskMat != null) { float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePoints) { OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } //Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture); } }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleMat, downScaleMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; if (useOpenCVDetector) { DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true); } else { DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread); } isThreadRunning = false; OnDetectionDone(); }); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); if (useOpenCVDetector) { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } else { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } } detectedObjectsInRegions.Clear(); foreach (Rect rect in rectsWhereRegions) { if (useOpenCVDetector) { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw previous rects DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(grayMat, texture); grayMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); isDetectingInFrameArrivedThread = false; }
// Update is called once per frame void Update() { lock (sync) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enable && !detecting) { detecting = true; grayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); Rect[] rects; if (!isUsingSeparateDetection) { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } rectangleTracker.GetObjects(resultObjects, true); rects = rectangleTracker.CreateCorrectionBySpeedOfRects(); if (rects.Length > 0) { OpenCVForUnity.Rect rect = rects [0]; // Adjust to Dilb's result. rect.y += (int)(rect.height * 0.1f); //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); UpdateARHeadTransform(points); } } else { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.toArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); if (resultObjects.Count > 0) { OpenCVForUnity.Rect rect = resultObjects [0]; // Adjust to Dilb's result. rect.y += (int)(rect.height * 0.1f); //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); UpdateARHeadTransform(points); } } } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } rects = rectsWhereRegions; } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } rects = rectsWhereRegions; } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } DrawRects(rects, bgraMat.width(), bgraMat.height()); bgraMat.Dispose(); Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); }, false); }
// Update is called once per frame void Update() { lock (sync) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); // rects = rectangleTracker.CreateCorrectionBySpeedOfRects (); DrawRects(rects, grayMat.width(), grayMat.height()); } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.toArray(); // rects = rectsWhereRegions; // for (int i = 0; i < rects.Length; i++) { // Imgproc.rectangle (rgbaMat, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (0, 0, 255, 255), 1); // } } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); // rects = rectsWhereRegions; // for (int i = 0; i < rects.Length; i++) { // Imgproc.rectangle (rgbaMat, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (0, 255, 0, 255), 1); // } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); DrawRects(rects, grayMat.width(), grayMat.height()); } } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;; Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); } }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw previous rect. DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(grayMat, texture); grayMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); isDetectingInFrameArrivedThread = false; }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face traking. if (enableTracking) { rectangleTracker.UpdateTrackedObjects(detectResult); detectResult = new List <OpenCVForUnity.Rect> (); rectangleTracker.GetObjects(detectResult, true); } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); foreach (var openCVRect in detectResult) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); landmarkPoints.Add(points); } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face changeing. if (faceMaskTexture != null && landmarkPoints.Count >= 1) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f) { if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); if (result.Count >= 1) { detectedFaceRect = result [0]; } } else { using (Mat grayMat = new Mat()) using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { //convert image to greyscale Imgproc.cvtColor(faceMaskMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); //detectResult = faces.toList (); List <OpenCVForUnity.Rect> faceList = faces.toList(); if (faceList.Count >= 1) { detectedFaceRect = new UnityEngine.Rect(faceList [0].x, faceList [0].y, faceList [0].width, faceList [0].height); // Adjust to Dilb's result. detectedFaceRect.y += detectedFaceRect.height * 0.1f; } } } } if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat); List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect); faceChanger.SetTargetImage(rgbaMat); for (int i = 0; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints [i], 1); } faceChanger.ChangeFace(); if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2); } } } else if (landmarkPoints.Count >= 2) { faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < landmarkPoints.Count; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints [0], landmarkPoints [i], 1); } faceChanger.ChangeFace(); } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // display face mask image. if (faceMaskMat != null) { float scale = (rgbaMat.width() / 4f) / faceMaskMat.width(); float tx = rgbaMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors); } }
// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face traking. if (enableTracking) { rectangleTracker.UpdateTrackedObjects(detectResult); detectResult = new List <OpenCVForUnity.Rect> (); rectangleTracker.GetObjects(detectResult, true); } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); foreach (var openCVRect in detectResult) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); landmarkPoints.Add(points); } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit) { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // face swapping. if (landmarkPoints.Count >= 2) { int ann = 0, bob = 1; for (int i = 0; i < landmarkPoints.Count - 1; i += 2) { ann = i; bob = i + 1; faceSwapper.SwapFaces(rgbaMat, landmarkPoints [ann], landmarkPoints [bob], 1); } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors); } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = bgraMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } if (enableDownScale) { int len = rects.Length; for (int i = 0; i < len; i++) { Rect rect = rects[i]; // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } DrawRects(rects, bgraMat.width(), bgraMat.height()); bgraMat.Dispose(); Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); }); }
// Update is called once per frame void Update() { // loop play. if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT)) { capture.set(Videoio.CAP_PROP_POS_FRAMES, 0); } if (capture.grab()) { capture.retrieve(rgbMat, 0); Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB); //Debug.Log ("Mat toString " + rgbMat.ToString ()); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY); using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) { Imgproc.equalizeHist(grayMat, equalizeHistMat); cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } } // face tracking. rectangleTracker.UpdateTrackedObjects(detectResult); List <TrackedRect> trackedRects = new List <TrackedRect> (); rectangleTracker.GetObjects(trackedRects, true); // create noise filter. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { if (!lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } if (!opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts())); } } else if (openCVRect.state == TrackedState.DELETED) { if (lowPassFilterDict.ContainsKey(openCVRect.id)) { lowPassFilterDict [openCVRect.id].Dispose(); lowPassFilterDict.Remove(openCVRect.id); } if (opticalFlowFilterDict.ContainsKey(openCVRect.id)) { opticalFlowFilterDict [openCVRect.id].Dispose(); opticalFlowFilterDict.Remove(openCVRect.id); } } } // create LUT texture. foreach (var openCVRect in trackedRects) { if (openCVRect.state == TrackedState.NEW) { faceMaskColorCorrector.CreateLUTTex(openCVRect.id); } else if (openCVRect.state == TrackedState.DELETED) { faceMaskColorCorrector.DeleteLUTTex(openCVRect.id); } } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> > (); for (int i = 0; i < trackedRects.Count; i++) { TrackedRect tr = trackedRects [i]; UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); // apply noise filter. if (enableNoiseFilter) { if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED) { opticalFlowFilterDict [tr.id].Process(rgbMat, points, points); lowPassFilterDict [tr.id].Process(rgbMat, points, points); } } landmarkPoints.Add(points); } // face masking. if (faceMaskTexture != null && landmarkPoints.Count >= 1) // Apply face masking between detected faces and a face mask image. { float maskImageWidth = faceMaskTexture.width; float maskImageHeight = faceMaskTexture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, faceMaskTexture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, faceMaskMat, rgbMat, faceLandmarkPointsInMask, landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } else if (landmarkPoints.Count >= 1) // Apply face masking between detected faces. { float maskImageWidth = texture.width; float maskImageHeight = texture.height; TrackedRect tr; for (int i = 0; i < trackedRects.Count; i++) { tr = trackedRects [i]; if (tr.state == TrackedState.NEW) { meshOverlay.CreateObject(tr.id, texture); } if (tr.state < TrackedState.DELETED) { MaskFace(meshOverlay, tr, landmarkPoints [i], landmarkPoints [0], maskImageWidth, maskImageHeight); if (enableColorCorrection) { CorrectFaceMaskColor(tr.id, rgbMat, rgbMat, landmarkPoints [0], landmarkPoints [i]); } } else if (tr.state == TrackedState.DELETED) { meshOverlay.DeleteObject(tr.id); } } } // draw face rects. if (displayFaceRects) { for (int i = 0; i < detectResult.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2); } for (int i = 0; i < trackedRects.Count; i++) { UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height); OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); //Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false); } } // draw face points. if (displayDebugFacePoints) { for (int i = 0; i < landmarkPoints.Count; i++) { OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2); } } // display face mask image. if (faceMaskTexture != null && faceMaskMat != null) { if (displayFaceRects) { OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2); } if (displayDebugFacePoints) { OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2); } float scale = (rgbMat.width() / 4f) / faceMaskMat.width(); float tx = rgbMat.width() - faceMaskMat.width() * scale; float ty = 0.0f; Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty); trans.put(0, 0, scale); trans.put(0, 1, 0.0f); trans.put(0, 2, tx); trans.put(1, 0, 0.0f); trans.put(1, 1, scale); trans.put(1, 2, ty); Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0)); if (displayFaceRects || displayDebugFacePointsToggle) { OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat); } } // Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture); } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleFrameMat = bgraMat; if (imageOptimizationHelper.downscaleRatio != 1) { downScaleFrameMat = new Mat(); Imgproc.resize(bgraMat, downScaleFrameMat, new Size(), 1.0 / imageOptimizationHelper.downscaleRatio, 1.0 / imageOptimizationHelper.downscaleRatio, Imgproc.INTER_LINEAR); } Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); if (!displayCameraImage) { // fill all black. Imgproc.rectangle(downScaleFrameMat, new Point(0, 0), new Point(downScaleFrameMat.width(), downScaleFrameMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); if (displayDetectedFaceRect) { for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2); } } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } if (displayDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } if (displayDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (displayDetectedFaceRect) { rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2); } } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } OpenCVForUnity.Utils.fastMatToTexture2D(downScaleFrameMat, texture); bgraMat.Dispose(); if (imageOptimizationHelper.downscaleRatio != 1) { downScaleFrameMat.Dispose(); } Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }