public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); if (enableDetection) { Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); // Detect markers and estimate Pose Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); if (applyEstimationPose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); for (int i = 0; i < ids.total(); i++) { //This example can display ARObject on only first detected marker. if (i == 0) { // Convert to unity pose data. double[] rvecArr = new double[3]; rvecs.get(0, 0, rvecArr); double[] tvecArr = new double[3]; tvecs.get(0, 0, tvecArr); tvecArr[2] /= imageOptimizationHelper.downscaleRatio; PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr); // Changes in pos/rot below these thresholds are ignored. if (enableLowPassFilter) { ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass); } oldPoseData = poseData; // Create transform matrix. transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one); lock (sync){ // Right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z-axis inverted matrix. ARM = ARM * invertZM; } hasUpdatedARTransformMatrix = true; break; } } } } Mat rgbMat4preview = null; if (displayCameraPreview) { rgbMat4preview = new Mat(); Imgproc.cvtColor(downScaleFrameMat, rgbMat4preview, Imgproc.COLOR_BGRA2RGB); if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) { // In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue) Aruco.drawAxis(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f); } } } } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && rgbMat4preview != null) { Utils.fastMatToTexture2D(rgbMat4preview, texture); } if (applyEstimationPose) { if (hasUpdatedARTransformMatrix) { hasUpdatedARTransformMatrix = false; lock (sync){ // Apply camera transform matrix. ARM = cameraToWorldMatrix * invertZM * ARM; ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM); } } } bgraMat.Dispose(); if (rgbMat4preview != null) { rgbMat4preview.Dispose(); } }); }