public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { comicFilter.Process(bgraMat, bgraMat, true); //Imgproc.putText (bgraMat, "W:" + bgraMat.width () + " H:" + bgraMat.height () + " SO:" + Screen.orientation, new Point (5, bgraMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (0), 2, Imgproc.LINE_AA, false); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
//Update for the Hololens /*9n */ #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat bgraMatClipROI = new Mat(bgraMat, processingAreaRect); Imgproc.cvtColor(bgraMat /*bgraMatClipROI*/, grayMat, Imgproc.COLOR_BGRA2GRAY); //// Here goes the code //new HL UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); position *= 1.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }
// Update is called once per frame void Update() { if (isImagesInputMode) { return; } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat grayMat = webCamTextureToMatHelper.GetMat(); if (shouldCaptureFrame) { shouldCaptureFrame = false; Mat frameMat = grayMat.clone(); double e = CaptureFrame(frameMat); if (e > 0) { repErr = e; } } DrawFrame(grayMat, bgrMat); Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); Utils.fastMatToTexture2D(rgbaMat, texture); } }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics camIntrinsics) { // Implement the HoloLens process here. --> Vector3 hitPoint = new Vector3(0, 0, 0); var laserPointerPosition = FindLaserPointer(bgraMat); if (laserPointerPosition != null) { hitPoint = raycast(laserPointerPosition, cameraToWorldMatrix, camIntrinsics); } // Implement the HoloLens process here. <-- Enqueue(() => { var fps = showFPS ? getFPS() : 0; if (laserPointerPosition != null && (redSphere.transform.position - hitPoint).magnitude >= 0.01) { redSphere.transform.position = hitPoint; Vector3 cameraPos = cameraToWorldMatrix.GetColumn(3); toolTipText.text = (hitPoint - cameraPos).magnitude.ToString("0.00") + " m" + (showFPS ? fps.ToString(", 0.0 FPS") : ""); } if (!(webCamTextureToMatHelper.IsPlaying() && isVisibleImage)) { bgraMat.Dispose(); return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } rects = rectsWhereRegions; for (int i = 0; i < rects.Length; i++) { Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1); } } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); for (int i = 0; i < rects.Length; i++) { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log ("detected face[" + i + "] " + rects [i]); //}, true); Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3); } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); position *= 1.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); if (enableDetection) { Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY); // Detect markers and estimate Pose Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); if (applyEstimationPose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); for (int i = 0; i < ids.total(); i++) { //This example can display ARObject on only first detected marker. if (i == 0) { // Convert to unity pose data. double[] rvecArr = new double[3]; rvecs.get(0, 0, rvecArr); double[] tvecArr = new double[3]; tvecs.get(0, 0, tvecArr); tvecArr[2] /= imageOptimizationHelper.downscaleRatio; PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr); // Changes in pos/rot below these thresholds are ignored. if (enableLowPassFilter) { ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass); } oldPoseData = poseData; // Create transform matrix. transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one); lock (sync){ // Right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z-axis inverted matrix. ARM = ARM * invertZM; } hasUpdatedARTransformMatrix = true; break; } } } } Mat rgbMat4preview = null; if (displayCameraPreview) { rgbMat4preview = new Mat(); Imgproc.cvtColor(downScaleFrameMat, rgbMat4preview, Imgproc.COLOR_BGRA2RGB); if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.Rect(0, i, 1, 1))) using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.Rect(0, i, 1, 1))) { // In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue) Aruco.drawAxis(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f); } } } } } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && rgbMat4preview != null) { OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat4preview, texture); } if (applyEstimationPose) { if (hasUpdatedARTransformMatrix) { hasUpdatedARTransformMatrix = false; lock (sync){ // Apply camera transform matrix. ARM = cameraToWorldMatrix * invertZM * ARM; ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM); } } } bgraMat.Dispose(); if (rgbMat4preview != null) { rgbMat4preview.Dispose(); } }, false); }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Mat camMatrix = null; MatOfDouble distCoeffs = null; if (useStoredCameraParameters) { camMatrix = this.camMatrix; distCoeffs = this.distCoeffs; } else { camMatrix = CreateCameraMatrix(cameraIntrinsics.FocalLengthX, cameraIntrinsics.FocalLengthY, cameraIntrinsics.PrincipalPointX / DOWNSCALE_RATIO, cameraIntrinsics.PrincipalPointY / DOWNSCALE_RATIO); distCoeffs = new MatOfDouble(cameraIntrinsics.RadialDistK1, cameraIntrinsics.RadialDistK2, cameraIntrinsics.RadialDistK3, cameraIntrinsics.TangentialDistP1, cameraIntrinsics.TangentialDistP2); } if (enableDetection) { // Detect markers and estimate Pose Aruco.detectMarkers(downScaleMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); if (applyEstimationPose && ids.total() > 0) { Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs); for (int i = 0; i < ids.total(); i++) { //This example can display ARObject on only first detected marker. if (i == 0) { // Convert to unity pose data. double[] rvecArr = new double[3]; rvecs.get(0, 0, rvecArr); double[] tvecArr = new double[3]; tvecs.get(0, 0, tvecArr); tvecArr[2] /= DOWNSCALE_RATIO; PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr); // Create transform matrix. transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one); lock (sync) { // Right-handed coordinates system (OpenCV) to left-handed one (Unity) ARM = invertYM * transformationM; // Apply Z-axis inverted matrix. ARM = ARM * invertZM; } hasUpdatedARTransformMatrix = true; break; } } } } Mat rgbMat4preview = null; if (displayCameraPreview) { rgbMat4preview = new Mat(); Imgproc.cvtColor(downScaleMat, rgbMat4preview, Imgproc.COLOR_GRAY2RGB); if (ids.total() > 0) { Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0)); if (applyEstimationPose) { for (int i = 0; i < ids.total(); i++) { using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1))) { // In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue) Calib3d.drawFrameAxes(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f); } } } } } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } if (displayCameraPreview && rgbMat4preview != null) { Utils.fastMatToTexture2D(rgbMat4preview, texture); rgbMat4preview.Dispose(); } if (applyEstimationPose) { if (hasUpdatedARTransformMatrix) { hasUpdatedARTransformMatrix = false; lock (sync) { // Apply camera transform matrix. ARM = cameraToWorldMatrix * invertZM * ARM; if (enableLerpFilter) { arGameObject.SetMatrix4x4(ARM); } else { ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM); } } } } grayMat.Dispose(); }); isDetectingInFrameArrivedThread = false; }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat bgraMatClipROI = new Mat(bgraMat, processingAreaRect); Imgproc.cvtColor(bgraMatClipROI, grayMat, Imgproc.COLOR_BGRA2GRAY); bgMat.copyTo(dstMatClippingROI); Imgproc.GaussianBlur(grayMat, lineMat, new Size(3, 3), 0); grayMat.get(0, 0, grayPixels); for (int i = 0; i < grayPixels.Length; i++) { maskPixels [i] = 0; if (grayPixels [i] < 70) { grayPixels [i] = 0; maskPixels [i] = 1; } else if (70 <= grayPixels [i] && grayPixels [i] < 120) { grayPixels [i] = 100; } else { grayPixels [i] = 255; maskPixels [i] = 1; } } grayMat.put(0, 0, grayPixels); maskMat.put(0, 0, maskPixels); grayMat.copyTo(dstMatClippingROI, maskMat); Imgproc.Canny(lineMat, lineMat, 20, 120); lineMat.copyTo(maskMat); Core.bitwise_not(lineMat, lineMat); lineMat.copyTo(dstMatClippingROI, maskMat); //Imgproc.putText (dstMat, "W:" + dstMat.width () + " H:" + dstMat.height () + " SO:" + Screen.orientation, new Point (5, dstMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (0), 2, Imgproc.LINE_AA, false); Imgproc.cvtColor(dstMat, bgraMat, Imgproc.COLOR_GRAY2BGRA); // //Imgproc.rectangle (bgraMat, new Point (0, 0), new Point (bgraMat.width (), bgraMat.height ()), new Scalar (0, 0, 255, 255), 2); //Imgproc.rectangle (bgraMat, processingAreaRect.tl(), processingAreaRect.br(), new Scalar (0, 0, 255, 255), 2); // bgraMatClipROI.Dispose(); UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = bgraMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } int len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; if (enableDownScale) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } // draw face rect Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_BLUE, 1); } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GREEN, 1); } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } len = resultObjects.Count; for (int i = 0; i < len; i++) { Rect rect = resultObjects[i]; if (enableDownScale) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } // draw face rect Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2); } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = bgraMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } if (enableDownScale) { int len = rects.Length; for (int i = 0; i < len; i++) { Rect rect = rects[i]; // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); } } Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } DrawRects(rects, bgraMat.width(), bgraMat.height()); bgraMat.Dispose(); Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); }); }
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY); Imgproc.equalizeHist(grayMat, grayMat); if (enableDetection && !isDetecting) { isDetecting = true; grayMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(); isThreadRunning = false; OnDetectionDone(); }); } Rect[] rects; if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult.toList()); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } else { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}, true); lock (rectangleTracker) { rectsWhereRegions = detectionResult.toArray(); } rects = rectsWhereRegions; } else { //UnityEngine.WSA.Application.InvokeOnAppThread (() => { // Debug.Log("process: get rectsWhereRegions from previous positions"); //}, true); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } rects = rectsWhereRegions; } detectedObjectsInRegions.Clear(); if (rectsWhereRegions.Length > 0) { int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions); } } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } rects = resultObjects.ToArray(); } UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } DrawRects(rects, bgraMat.width(), bgraMat.height()); bgraMat.Dispose(); Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance)); Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance)); //position Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos); gameObject.transform.position = position; //scale Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1); gameObject.transform.localScale = scale; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.rotation = rotation; rectOverlay.UpdateOverlayTransform(gameObject.transform); }, false); }
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics) { isDetectingInFrameArrivedThread = true; DebugUtils.VideoTick(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } Imgproc.equalizeHist(downScaleMat, downScaleMat); if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); System.Threading.Tasks.Task.Run(() => { isThreadRunning = true; DetectObject(grayMat4Thread, out detectionResult, cascade4Thread); isThreadRunning = false; OnDetectionDone(); }); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectionResult); } } lock (rectangleTracker) { rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions were got from detectionResult"); //}); lock (rectangleTracker) { rectsWhereRegions = detectionResult.ToArray(); } } else { //Enqueue(() => //{ // Debug.Log("process: get rectsWhereRegions from previous positions"); //}); lock (rectangleTracker) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } } detectedObjectsInRegions.Clear(); int len = rectsWhereRegions.Length; for (int i = 0; i < len; i++) { DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade); } lock (rectangleTracker) { rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, true); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } // draw previous rect. DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rect. DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6); } DebugUtils.TrackTick(); Enqueue(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(grayMat, texture); grayMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }); isDetectingInFrameArrivedThread = false; }
//OnFrameMatAcquired == update pero para el HL builded /*20n versionskeleton */ #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix) { Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY); //Debug.Log("Entré al onframeaquired desde el tracker"); //26n Debug.Log("si hay puntos " + p1.x.ToString("0.00") + ";" + p1.y.ToString("0.00")); //yep //textDebug.text = "p2:"+p2.x +","+ p2.y; //never leave a textMesh inside a Hololen's Update //28n pm //if (p1 != null && p2 != null) { // Imgproc.circle(bgraMat, p1, 5, new Scalar(255, 0, 0, 255), 2); // Imgproc.circle(bgraMat, p2, 5, new Scalar(0, 255, 0, 255), 2); //} //Imgproc.rectangle(bgraMat, p1, p2, new Scalar(255, 255, 255, 255), 2, 1, 0); if (storedTouchPoint != null) //osea si el tipo es allguien { //28n selectedPointList.Add(storedTouchPoint); p1 = new Point(storedTouchPoint.x - 50, storedTouchPoint.y - 50); p2 = new Point(storedTouchPoint.x + 50, storedTouchPoint.y + 50); //#ToDo crete the ROI with two taps like MonoTrackingExample.cs selectedPointList.Add(p1); selectedPointList.Add(p2); //Imgproc.circle(bgraMat, p2, 6, new Scalar(0, 0, 0, 255), 2); using (MatOfPoint selectedPointMat = new MatOfPoint(selectedPointList.ToArray())) { OpenCVForUnity.Rect region = Imgproc.boundingRect(selectedPointMat); //si se necesita al fin y al cabo Debug.Log("region de ancho " + region.width); SelectTracker(tracker_type); //16n monotracker = TrackerKCF.create(); //8n trackerInitialized = monotracker.init(grayMat, new Rect2d(region.x, region.y, region.width, region.height)); } storedTouchPoint = null; } //21N //22nif (selectedPointList.Count != 0) //{ // //p1 = new Point(bgraMat.width() / 2, bgraMat.height() / 2); // //p2 = new Point(bgraMat.width() * 2 / 3, bgraMat.height() * 2 / 3); //21n vamos a reiniciar los puntos para ver si es cosa de coordenada o de que solo pinta por un ratico, yepsi los pinta // Debug.Log("si hay puntos "+ p1.x.ToString("0.00") +";"+ p1.y.ToString("0.00")); //yep // //textDebug.text = "p2:"+p2.x +","+ p2.y; //nunca maaaas mesh.text en un upate del HL // Imgproc.circle(bgraMat, p1, 6, new Scalar(255, 0, 0, 255), 2); // Imgproc.circle(bgraMat, p2, 6, new Scalar(0, 255, 0, 255), 2); // Imgproc.rectangle(bgraMat, p1, p2, new Scalar(255, 255, 255, 255), 2, 1, 0); // //pues debe ser que a este paso ya no hay punticos mijo! //} if (trackerInitialized) { Debug.Log("SI inicializo el tracking"); bool updated = monotracker.update(grayMat, bbox);//8n //pero nunca le pasa la bbox ni las coordenadas, unicamente al momento de tracker.init if (bbox.width != 0 && bbox.height != 0) { //Debug.Log("tracking por aca:" + bbox.x + ";" + bbox.y); Imgproc.rectangle(bgraMat, bbox.tl(), bbox.br(), new Scalar(255, 255, 255, 255), 2, 1, 0); previousBox = new Rect2d(bbox.x, bbox.y, bbox.width, bbox.height); // ------ 3D LOCATION ----------- #Todo //28n is like a task, take long time to response, then keep going with others //ray = cameraInstance.ScreenPointToRay(new Vector3((float)(bbox.x + bbox.width/2), (float)(bbox.y + bbox.height/2), 0)); //Debug.Log("dime que no te trabas ahi porfa" + ray.ToString("F2")); //if (Physics.Raycast(ray, out hit)) //{ // box3DPosition = hit.transform.position; // print("I'm looking at " + box3DPosition); // Debug.Log("the box is in" + box3DPosition); //} //else // print("I'm looking at nothing!"); } else { if (previousBox != null) { Debug.Log("Se perdio en:" + previousBox.x + ";" + previousBox.y); Imgproc.rectangle(bgraMat, previousBox.tl(), previousBox.br(), new Scalar(0, 0, 255, 255), 2, 1, 0); //8n } } } //new HL UnityEngine.WSA.Application.InvokeOnAppThread(() => { if (!webCamTextureToMatHelper.IsPlaying()) { return; } Utils.fastMatToTexture2D(bgraMat, texture); bgraMat.Dispose(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); position *= 1.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; }, false); }
// Update is called once per frame, in webcam void Update() { if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { //19n //Vector3 posWorld = GazeManager.Instance.HitPosition; //textMesh.text = posWorld + ""; //Vector2 posWorld2Screen = WorldCoordsToScreen(webCamTextureToMatHelper.GetCameraToWorldMatrix(), webCamTextureToMatHelper.GetProjectionMatrix(), posWorld /*not used because of gaze*/); //textScreenGazePos.text = posWorld2Screen + ""; Mat rgbaMat = webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); //textMesh.text = selectedPointList.Count+""; //Here starts the OpenCV script Vector2 posWorld2Screen = WorldCoordsToScreen(webCamTextureToMatHelper.GetCameraToWorldMatrix(), webCamTextureToMatHelper.GetProjectionMatrix(), sphere.transform.position);//19N storedTouchPoint = new Point(posWorld2Screen.x, posWorld2Screen.y); textMesh.text = storedTouchPoint.x + ";" + storedTouchPoint.y; //pintemelo en la screen /* * if (selectedPointList.Count < 2) * { * foreach (var point in selectedPointList) * { * Imgproc.circle(rgbaMat, point, 6, new Scalar(0, 0, 255), 2); * } * }*/ Utils.fastMatToTexture2D(rgbaMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = webCamTextureToMatHelper.GetCameraToWorldMatrix(); Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2); position *= 1.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }