// Update is called once per frame void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { DebugUtils.VideoTick(); Mat grayMat = webCamTextureToMatHelper.GetMat(); comicFilter.Process(grayMat, grayMat, false); //Imgproc.putText(grayMat, "W:" + grayMat.width() + " H:" + grayMat.height() + " SO:" + Screen.orientation, new Point(5, grayMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255), 2, Imgproc.LINE_AA, false); DebugUtils.TrackTick(); Utils.fastMatToTexture2D(grayMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = webCamTextureToMatHelper.GetCameraToWorldMatrix(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
private void OnDetectionDone() { DebugUtils.TrackTick(); if (displayCameraPreview) { Imgproc.cvtColor(downScaleMat, rgbMat4preview, Imgproc.COLOR_GRAY2RGB); if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) { // In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue) Calib3d.drawFrameAxes(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f); } } } } Utils.fastMatToTexture2D(rgbMat4preview, texture); } if (applyEstimationPose) { if (hasUpdatedARTransformMatrix) { hasUpdatedARTransformMatrix = false; // Apply the cameraToWorld matrix with the Z-axis inverted. ARM = arCamera.cameraToWorldMatrix * invertZM * ARM; if (enableLerpFilter) { arGameObject.SetMatrix4x4(ARM); } else { ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM); } } } isDetecting = false; }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Mat camMatrix = null; MatOfDouble distCoeffs = null; if (useStoredCameraParameters) { camMatrix = this.camMatrix; distCoeffs = this.distCoeffs; } else { camMatrix = CreateCameraMatrix(cameraIntrinsics.FocalLengthX, cameraIntrinsics.FocalLengthY, cameraIntrinsics.PrincipalPointX / DOWNSCALE_RATIO, cameraIntrinsics.PrincipalPointY / DOWNSCALE_RATIO); distCoeffs = new MatOfDouble(cameraIntrinsics.RadialDistK1, cameraIntrinsics.RadialDistK2, cameraIntrinsics.RadialDistK3, cameraIntrinsics.TangentialDistP1, cameraIntrinsics.TangentialDistP2); } if (enableDetection) { // Detect markers and estimate Pose Aruco.detectMarkers(downScaleMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); if (applyEstimationPose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); for (int i = 0; i < ids.total(); i++) { //This example can display ARObject on only first detected marker. if (i == 0) { // Convert to unity pose data. double[] rvecArr = new double[3]; rvecs.get(0, 0, rvecArr); double[] tvecArr = new double[3]; tvecs.get(0, 0, tvecArr); tvecArr[2] /= DOWNSCALE_RATIO; PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr); // Create transform matrix. transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one); lock (sync) { // Right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z-axis inverted matrix. ARM = ARM * invertZM; } hasUpdatedARTransformMatrix = true; break; } } } } Mat rgbMat4preview = null; if (displayCameraPreview) { rgbMat4preview = new Mat(); Imgproc.cvtColor(downScaleMat, rgbMat4preview, Imgproc.COLOR_GRAY2RGB); if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) { // In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue) Calib3d.drawFrameAxes(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f); } } } } } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && rgbMat4preview != null) { Utils.fastMatToTexture2D(rgbMat4preview, texture); rgbMat4preview.Dispose(); } if (applyEstimationPose) { if (hasUpdatedARTransformMatrix) { hasUpdatedARTransformMatrix = false; lock (sync) { // Apply camera transform matrix. ARM = cameraToWorldMatrix * invertZM * ARM; if (enableLerpFilter) { arGameObject.SetMatrix4x4(ARM); } else { ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM); } } } } grayMat.Dispose(); }); isDetectingInFrameArrivedThread = false; }
// Update is called once per frame void Update() { lock (ExecuteOnMainThread) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { DebugUtils.VideoTick(); Mat grayMat = webCamTextureToMatHelper.GetMat(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult); } rectangleTracker.GetObjects(resultObjects, true); if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.ToArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw previous rect. DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } DebugUtils.TrackTick(); Utils.fastMatToTexture2D(grayMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw previous rect. DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(grayMat, texture); grayMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); isDetectingInFrameArrivedThread = false; }
// Update is called once per frame void Update() { lock (ExecuteOnMainThread) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { DebugUtils.VideoTick(); Mat grayMat = webCamTextureToMatHelper.GetMat(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult); } rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.ToArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); rects = resultObjects.ToArray(); } if (enableDownScale) { int len = rects.Length; for (int i = 0; i < len; i++) { Rect rect = rects[i]; // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } } // draw face rect. DrawRects(rects, grayMat.width(), grayMat.height()); DebugUtils.TrackTick(); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;; Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); } }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } if (enableDownScale) { int len = rects.Length; for (int i = 0; i < len; i++) { Rect rect = rects[i]; // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } // draw face rect. DrawRects(rects, grayMat.width(), grayMat.height()); grayMat.Dispose(); Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); }); isDetectingInFrameArrivedThread = false; }