public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, CVTCOLOR_CODE); Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat); if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleGrayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; if (useOpenCVDetector) { DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true); } else { DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread); } isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); int len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2); if (displayDetectedFaceRect) { // draw face rect OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } if (displayDetectedFaceRect) { DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); if (useOpenCVDetector) { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } else { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } if (displayDetectedFaceRect) { DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { if (useOpenCVDetector) { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2); if (displayDetectedFaceRect) { // draw face rect OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleMat, downScaleMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; if (useOpenCVDetector) { DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true); } else { DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread); } isThreadRunning = false; OnDetectionDone(); }); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); if (useOpenCVDetector) { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } else { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } } detectedObjectsInRegions.Clear(); foreach (Rect rect in rectsWhereRegions) { if (useOpenCVDetector) { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw previous rects DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(grayMat, texture); grayMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); isDetectingInFrameArrivedThread = false; }