// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Mat rgbaMatClipROI = new Mat(rgbaMat, processingAreaRect); rgbaMatClipROI.copyTo(processingAreaMat); // fill all black. Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, processingAreaMat); // detect faces. List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> (); if (isUsingDlibFaceDetecter) { List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { // convert image to greyscale. Imgproc.cvtColor(processingAreaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); cascade.detectMultiScale(grayMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(grayMat.cols() * 0.15, grayMat.cols() * 0.15), new Size()); detectResult = faces.toList(); // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } } foreach (var rect in detectResult) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMatClipROI, points, new Scalar(0, 255, 0, 255), 2); //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMatClipROI, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2); } Imgproc.putText(rgbaMatClipROI, "W:" + rgbaMatClipROI.width() + " H:" + rgbaMatClipROI.height() + " SO:" + Screen.orientation, new Point(5, rgbaMatClipROI.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false); Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2); // Draw prosessing area rectangle. Imgproc.rectangle(rgbaMat, processingAreaRect.tl(), processingAreaRect.br(), new Scalar(255, 255, 0, 255), 2); OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture); rgbaMatClipROI.Dispose(); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix; ; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; texture.wrapMode = TextureWrapMode.Clamp; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
void OnPhotoCaptured(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame) { Matrix4x4 cameraToWorldMatrix; photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; Matrix4x4 projectionMatrix; photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix); photoCaptureFrame.UploadImageDataToTexture(m_Texture); OpenCVForUnity.Utils.texture2DToMat(m_Texture, rgbaMat); // fill all black. // Imgproc.rectangle (rgbaMat, new Point (0, 0), new Point (rgbaMat.width (), rgbaMat.height ()), new Scalar (0, 0, 0, 0), -1); // draw an edge lines. Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2); // draw a diagonal line. // Imgproc.line (rgbaMat, new Point (0, 0), new Point (rgbaMat.cols (), rgbaMat.rows ()), new Scalar (255, 0, 0, 255)); OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); //detect face List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection(); foreach (var r in detectResult) { Debug.Log("rect : " + r.rect); //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(r.rect); Debug.Log("face points count : " + points.Count); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, r.rect, new Scalar(255, 0, 0, 255), 2); } Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.5, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_AA, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, m_Texture, colors); m_Texture.wrapMode = TextureWrapMode.Clamp; m_CanvasRenderer.enabled = true; m_CanvasRenderer.sharedMaterial.SetTexture("_MainTex", m_Texture); m_CanvasRenderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); m_CanvasRenderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); m_CanvasRenderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); m_Canvas.transform.position = position; m_Canvas.transform.rotation = rotation; Debug.Log("Took picture!"); Debug.Log("projectionMatrix:\n" + projectionMatrix.ToString()); m_CapturingPhoto = false; }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); Mat bgraMat4preview = null; if (displayCameraPreview) { bgraMat4preview = new Mat(); downScaleFrameMat.copyTo(bgraMat4preview); } List <Vector2> points = null; Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); if (rects.Length > 0) { OpenCVForUnity.Rect rect = rects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(bgraMat, points, points, false); } if (displayCameraPreview && bgraMat4preview != null) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2); } } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (resultObjects.Count > 0) { OpenCVForUnity.Rect rect = resultObjects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(bgraMat, points, points, false); } if (displayCameraPreview && bgraMat4preview != null) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2); } } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && bgraMat4preview != null) { OpenCVForUnity.Utils.fastMatToTexture2D(bgraMat4preview, texture); } if (points != null) { UpdateARHeadTransform(points, cameraToWorldMatrix); } bgraMat.Dispose(); if (bgraMat4preview != null) { bgraMat4preview.Dispose(); } }, false); }
// Update is called once per frame void Update() { lock (sync) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = imageOptimizationHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } rectangleTracker.GetObjects(resultObjects, true); rects = rectangleTracker.CreateCorrectionBySpeedOfRects(); if (rects.Length > 0) { OpenCVForUnity.Rect rect = rects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(rgbaMat, points, points, false); } UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix); if (displayCameraPreview) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); } } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.toArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); if (resultObjects.Count > 0) { OpenCVForUnity.Rect rect = resultObjects [0]; // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. rect.y += (int)(rect.height * 0.1f); //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); if (enableOpticalFlowFilter) { opticalFlowFilter.Process(rgbaMat, points, points, false); } UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix); if (displayCameraPreview) { //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); } } } if (displayCameraPreview) { OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture); } } }
// Update is called once per frame void Update() { lock (ExecuteOnMainThread) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, CVTCOLOR_CODE); Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat); if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleGrayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } if (!displayCameraImage) { // fill all black Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult); } rectangleTracker.GetObjects(resultObjects, true); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); int len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { // draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.ToArray(); if (displayDetectedFaceRect) { DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1); } } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); if (useOpenCVDetector) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } else { rectsWhereRegions = rectangleTracker.CreateRawRects(); } if (displayDetectedFaceRect) { DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { if (useOpenCVDetector) { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } Utils.fastMatToTexture2D(rgbaMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, CVTCOLOR_CODE); Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat); if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleGrayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; if (useOpenCVDetector) { DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true); } else { DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread); } isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); int len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2); if (displayDetectedFaceRect) { // draw face rect OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } if (displayDetectedFaceRect) { DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); if (useOpenCVDetector) { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } else { lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } if (displayDetectedFaceRect) { DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { if (useOpenCVDetector) { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); } // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; // restore to original size rect float downscaleRatio = imageOptimizationHelper.downscaleRatio; rect.x = (int)(rect.x * downscaleRatio); rect.y = (int)(rect.y * downscaleRatio); rect.width = (int)(rect.width * downscaleRatio); rect.height = (int)(rect.height * downscaleRatio); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); // draw face landmark points OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2); if (displayDetectedFaceRect) { // draw face rect OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2); } } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
// Update is called once per frame void Update() { lock (sync) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat()); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enable && !detecting) { detecting = true; grayMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); if (!isShowingWebCamImage) { // fill all black. Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!isUsingSeparateDetection) { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } rectangleTracker.GetObjects(resultObjects, true); // rects = resultObjects.ToArray (); rects = rectangleTracker.CreateCorrectionBySpeedOfRects(); if (isShowingDetectedFaceRect) { for (int i = 0; i < rects.Length; i++) { //Debug.Log ("detected face[" + i + "] " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); if (isShowingDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2); } } } else { if (didUpdateTheDetectionResult) { didUpdateTheDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.toArray(); if (isShowingDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1); } } } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); if (isShowingDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); if (isShowingDetectedFaceRect) { rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //Debug.Log ("detected face[" + i + "] " + rects [i]); Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); if (isShowingDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2); } } } OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors()); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; texture.wrapMode = TextureWrapMode.Clamp; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
// Update is called once per frame void Update() { lock (ExecuteOnMainThread) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { DebugUtils.VideoTick(); Mat grayMat = webCamTextureToMatHelper.GetMat(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleMat, downScaleMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult); } rectangleTracker.GetObjects(resultObjects, true); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.ToArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); if (useOpenCVDetector) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } else { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } detectedObjectsInRegions.Clear(); foreach (Rect rect in rectsWhereRegions) { if (useOpenCVDetector) { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw previous rects DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } DebugUtils.TrackTick(); Utils.fastMatToTexture2D(grayMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleFrameMat = bgraMat; if (imageOptimizationHelper.downscaleRatio != 1) { downScaleFrameMat = new Mat(); Imgproc.resize(bgraMat, downScaleFrameMat, new Size(), 1.0 / imageOptimizationHelper.downscaleRatio, 1.0 / imageOptimizationHelper.downscaleRatio, Imgproc.INTER_LINEAR); } Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); if (!displayCameraImage) { // fill all black. Imgproc.rectangle(downScaleFrameMat, new Point(0, 0), new Point(downScaleFrameMat.width(), downScaleFrameMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); if (displayDetectedFaceRect) { for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2); } } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } if (displayDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } if (displayDetectedFaceRect) { rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (displayDetectedFaceRect) { rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } // Adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in resultObjects) { r.y += (int)(r.height * 0.1f); } foreach (var rect in resultObjects) { //detect landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); //draw landmark points OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2); if (displayDetectedFaceRect) { //draw face rect OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2); } } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } OpenCVForUnity.Utils.fastMatToTexture2D(downScaleFrameMat, texture); bgraMat.Dispose(); if (imageOptimizationHelper.downscaleRatio != 1) { downScaleFrameMat.Dispose(); } Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }