/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API // HololensCameraStream always returns image data in BGRA format. texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.BGRA32, false); #else texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); #endif texture.wrapMode = TextureWrapMode.Clamp; Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); comicFilter = new ComicFilter(60, 120, 3); quad_renderer = gameObject.GetComponent <Renderer> () as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); Matrix4x4 projectionMatrix; #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", vignetteScale); float halfOfVerticalFov = Mathf.Atan(1.0f / projectionMatrix.m11); float aspectRatio = (1.0f / Mathf.Tan(halfOfVerticalFov)) / projectionMatrix.m00; Debug.Log("halfOfVerticalFov " + halfOfVerticalFov); Debug.Log("aspectRatio " + aspectRatio); }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); #else //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; #endif grayMat = new Mat(); cascade = new CascadeClassifier(); cascade.load(Utils.getFilePath("lbpcascade_frontalface.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif grayMat4Thread = new Mat(); cascade4Thread = new CascadeClassifier(); cascade4Thread.load(Utils.getFilePath("haarcascade_frontalface_alt.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade4Thread.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif }
/// <summary> /// Raises the webcam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); InitializeCalibraton(webCamTextureMat); // if WebCamera is frontFaceing, flip Mat. webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing(); }
/// <summary> /// Raises the webcam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); InitializeCalibraton(webCamTextureMat); // if WebCamera is frontFaceing, flip Mat. if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing) { webCamTextureToMatHelper.flipHorizontal = true; } }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API // HololensCameraStream always returns image data in BGRA format. texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.BGRA32, false); #else texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); #endif texture.wrapMode = TextureWrapMode.Clamp; Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); quad_renderer = gameObject.GetComponent <Renderer> () as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(0, 0)); Matrix4x4 projectionMatrix; #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f); grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1); cascade = new CascadeClassifier(); cascade.load(Utils.getFilePath("lbpcascade_frontalface.xml")); // cascade.load (Utils.getFilePath ("haarcascade_frontalface_alt.xml")); // "empty" method is not working on the UWP platform. // if (cascade.empty ()) { // Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); // } grayMat4Thread = new Mat(); cascade4Thread = new CascadeClassifier(); cascade4Thread.load(Utils.getFilePath("haarcascade_frontalface_alt.xml")); // "empty" method is not working on the UWP platform. // if (cascade4Thread.empty ()) { // Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); // } detectionResult = new MatOfRect(); }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat rawSizeMat = webCamTextureToMatHelper.GetMat(); float rawSizeWidth = rawSizeMat.width(); float rawSizeHeight = rawSizeMat.height(); Mat webCamTextureMat = imageOptimizationHelper.GetDownScaleMat(rawSizeMat); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); float width = webCamTextureMat.width(); float height = webCamTextureMat.height(); texture = new Texture2D((int)width, (int)height, TextureFormat.RGB24, false); previewQuad.GetComponent <MeshRenderer>().material.mainTexture = texture; previewQuad.transform.localScale = new Vector3(1, height / width, 1); previewQuad.SetActive(displayCameraPreview); // set camera parameters. double fx; double fy; double cx; double cy; string loadDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); string calibratonDirectoryName = "camera_parameters" + rawSizeWidth + "x" + rawSizeHeight; string loadCalibratonFileDirectoryPath = Path.Combine(loadDirectoryPath, calibratonDirectoryName); string loadPath = Path.Combine(loadCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); if (useStoredCameraParameters && File.Exists(loadPath)) { CameraParameters param; XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters)); using (var stream = new FileStream(loadPath, FileMode.Open)) { param = (CameraParameters)serializer.Deserialize(stream); } fx = param.camera_matrix[0]; fy = param.camera_matrix[4]; cx = param.camera_matrix[2] / imageOptimizationHelper.downscaleRatio; cy = param.camera_matrix[5] / imageOptimizationHelper.downscaleRatio; camMatrix = new Mat(3, 3, CvType.CV_64FC1); camMatrix.put(0, 0, fx); camMatrix.put(0, 1, 0); camMatrix.put(0, 2, cx); camMatrix.put(1, 0, 0); camMatrix.put(1, 1, fy); camMatrix.put(1, 2, cy); camMatrix.put(2, 0, 0); camMatrix.put(2, 1, 0); camMatrix.put(2, 2, 1.0f); distCoeffs = new MatOfDouble(param.GetDistortionCoefficients()); Debug.Log("Loaded CameraParameters from a stored XML file."); Debug.Log("loadPath: " + loadPath); } else { fx = this.fx; fy = this.fy; cx = this.cx / imageOptimizationHelper.downscaleRatio; cy = this.cy / imageOptimizationHelper.downscaleRatio; camMatrix = new Mat(3, 3, CvType.CV_64FC1); camMatrix.put(0, 0, fx); camMatrix.put(0, 1, 0); camMatrix.put(0, 2, cx); camMatrix.put(1, 0, 0); camMatrix.put(1, 1, fy); camMatrix.put(1, 2, cy); camMatrix.put(2, 0, 0); camMatrix.put(2, 1, 0); camMatrix.put(2, 2, 1.0f); distCoeffs = new MatOfDouble(this.distCoeffs1, this.distCoeffs2, this.distCoeffs3, this.distCoeffs4, this.distCoeffs5); Debug.Log("Created a dummy CameraParameters."); } Debug.Log("camMatrix " + camMatrix.dump()); Debug.Log("distCoeffs " + distCoeffs.dump()); //Calibration camera Size imageSize = new Size(width, height); double apertureWidth = 0; double apertureHeight = 0; double[] fovx = new double[1]; double[] fovy = new double[1]; double[] focalLength = new double[1]; Point principalPoint = new Point(0, 0); double[] aspectratio = new double[1]; Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); Debug.Log("imageSize " + imageSize.ToString()); Debug.Log("apertureWidth " + apertureWidth); Debug.Log("apertureHeight " + apertureHeight); Debug.Log("fovx " + fovx [0]); Debug.Log("fovy " + fovy [0]); Debug.Log("focalLength " + focalLength [0]); Debug.Log("principalPoint " + principalPoint.ToString()); Debug.Log("aspectratio " + aspectratio [0]); // Display objects near the camera. arCamera.nearClipPlane = 0.01f; grayMat = new Mat(); ids = new Mat(); corners = new List <Mat> (); rejectedCorners = new List <Mat> (); rvecs = new Mat(); tvecs = new Mat(); rotMat = new Mat(3, 3, CvType.CV_64FC1); transformationM = new Matrix4x4(); invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1)); Debug.Log("invertYM " + invertYM.ToString()); invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1)); Debug.Log("invertZM " + invertZM.ToString()); detectorParams = DetectorParameters.create(); dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_6X6_250); //If WebCamera is frontFaceing,flip Mat. if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing) { webCamTextureToMatHelper.flipHorizontal = true; } downScaleFrameMat = new Mat((int)height, (int)width, CvType.CV_8UC4); rgbMat4preview = new Mat(); }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat grayMat = webCamTextureToMatHelper.GetMat(); float rawFrameWidth = grayMat.width(); float rawFrameHeight = grayMat.height(); if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } float width = downScaleMat.width(); float height = downScaleMat.height(); texture = new Texture2D((int)width, (int)height, TextureFormat.RGB24, false); previewQuad.GetComponent <MeshRenderer>().material.mainTexture = texture; previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1); previewQuad.SetActive(displayCameraPreview); //Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); DebugUtils.AddDebugStr(webCamTextureToMatHelper.GetWidth() + " x " + webCamTextureToMatHelper.GetHeight() + " : " + webCamTextureToMatHelper.GetFPS()); if (enableDownScale) { DebugUtils.AddDebugStr("enableDownScale = true: " + DOWNSCALE_RATIO + " / " + width + " x " + height); } // create camera matrix and dist coeffs. string loadDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); string calibratonDirectoryName = "camera_parameters" + rawFrameWidth + "x" + rawFrameWidth; string loadCalibratonFileDirectoryPath = Path.Combine(loadDirectoryPath, calibratonDirectoryName); string loadPath = Path.Combine(loadCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); if (useStoredCameraParameters && File.Exists(loadPath)) { // If there is a camera parameters stored by HoloLensArUcoCameraCalibrationExample, use it CameraParameters param; XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters)); using (var stream = new FileStream(loadPath, FileMode.Open)) { param = (CameraParameters)serializer.Deserialize(stream); } double fx = param.camera_matrix[0]; double fy = param.camera_matrix[4]; double cx = param.camera_matrix[2]; double cy = param.camera_matrix[5]; camMatrix = CreateCameraMatrix(fx, fy, cx / DOWNSCALE_RATIO, cy / DOWNSCALE_RATIO); distCoeffs = new MatOfDouble(param.GetDistortionCoefficients()); Debug.Log("Loaded CameraParameters from a stored XML file."); Debug.Log("loadPath: " + loadPath); DebugUtils.AddDebugStr("Loaded CameraParameters from a stored XML file."); DebugUtils.AddDebugStr("loadPath: " + loadPath); } else { if (useStoredCameraParameters && !File.Exists(loadPath)) { DebugUtils.AddDebugStr("The CameraParameters XML file (" + loadPath + ") does not exist."); } #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API CameraIntrinsics cameraIntrinsics = webCamTextureToMatHelper.GetCameraIntrinsics(); camMatrix = CreateCameraMatrix(cameraIntrinsics.FocalLengthX, cameraIntrinsics.FocalLengthY, cameraIntrinsics.PrincipalPointX / DOWNSCALE_RATIO, cameraIntrinsics.PrincipalPointY / DOWNSCALE_RATIO); distCoeffs = new MatOfDouble(cameraIntrinsics.RadialDistK1, cameraIntrinsics.RadialDistK2, cameraIntrinsics.RadialDistK3, cameraIntrinsics.TangentialDistP1, cameraIntrinsics.TangentialDistP2); Debug.Log("Created CameraParameters from VideoMediaFrame.CameraIntrinsics on device."); DebugUtils.AddDebugStr("Created CameraParameters from VideoMediaFrame.CameraIntrinsics on device."); #else // The camera matrix value of Hololens camera 896x504 size. // For details on the camera matrix, please refer to this page. (http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) // These values are unique to my device, obtained from the "Windows.Media.Devices.Core.CameraIntrinsics" class. (https://docs.microsoft.com/en-us/uwp/api/windows.media.devices.core.cameraintrinsics) // Can get these values by using this helper script. (https://github.com/EnoxSoftware/HoloLensWithOpenCVForUnityExample/tree/master/Assets/HololensCameraIntrinsicsChecker/CameraIntrinsicsCheckerHelper) double fx = 1035.149; //focal length x. double fy = 1034.633; //focal length y. double cx = 404.9134; //principal point x. double cy = 236.2834; //principal point y. double distCoeffs1 = 0.2036923; //radial distortion coefficient k1. double distCoeffs2 = -0.2035773; //radial distortion coefficient k2. double distCoeffs3 = 0.0; //tangential distortion coefficient p1. double distCoeffs4 = 0.0; //tangential distortion coefficient p2. double distCoeffs5 = -0.2388065; //radial distortion coefficient k3. camMatrix = CreateCameraMatrix(fx, fy, cx / DOWNSCALE_RATIO, cy / DOWNSCALE_RATIO); distCoeffs = new MatOfDouble(distCoeffs1, distCoeffs2, distCoeffs3, distCoeffs4, distCoeffs5); Debug.Log("Created a dummy CameraParameters (896x504)."); DebugUtils.AddDebugStr("Created a dummy CameraParameters (896x504)."); #endif } Debug.Log("camMatrix " + camMatrix.dump()); Debug.Log("distCoeffs " + distCoeffs.dump()); //DebugUtils.AddDebugStr("camMatrix " + camMatrix.dump()); //DebugUtils.AddDebugStr("distCoeffs " + distCoeffs.dump()); //Calibration camera Size imageSize = new Size(width, height); double apertureWidth = 0; double apertureHeight = 0; double[] fovx = new double[1]; double[] fovy = new double[1]; double[] focalLength = new double[1]; Point principalPoint = new Point(0, 0); double[] aspectratio = new double[1]; Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); Debug.Log("imageSize " + imageSize.ToString()); Debug.Log("apertureWidth " + apertureWidth); Debug.Log("apertureHeight " + apertureHeight); Debug.Log("fovx " + fovx[0]); Debug.Log("fovy " + fovy[0]); Debug.Log("focalLength " + focalLength[0]); Debug.Log("principalPoint " + principalPoint.ToString()); Debug.Log("aspectratio " + aspectratio[0]); // Display objects near the camera. arCamera.nearClipPlane = 0.01f; ids = new Mat(); corners = new List <Mat>(); rejectedCorners = new List <Mat>(); rvecs = new Mat(); tvecs = new Mat(); rotMat = new Mat(3, 3, CvType.CV_64FC1); transformationM = new Matrix4x4(); invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1)); Debug.Log("invertYM " + invertYM.ToString()); invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1)); Debug.Log("invertZM " + invertZM.ToString()); detectorParams = DetectorParameters.create(); dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_6X6_250); //If WebCamera is frontFaceing, flip Mat. webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing(); rgbMat4preview = new Mat(); }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API // HololensCameraStream always returns image data in BGRA format. texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.BGRA32, false); #else texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); #endif texture.wrapMode = TextureWrapMode.Clamp; Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); processingAreaRect = new OpenCVForUnity.Rect((int)(webCamTextureMat.cols() * (outsideClippingRatio.x - clippingOffset.x)), (int)(webCamTextureMat.rows() * (outsideClippingRatio.y + clippingOffset.y)), (int)(webCamTextureMat.cols() * (1f - outsideClippingRatio.x * 2)), (int)(webCamTextureMat.rows() * (1f - outsideClippingRatio.y * 2))); processingAreaRect = processingAreaRect.intersect(new OpenCVForUnity.Rect(0, 0, webCamTextureMat.cols(), webCamTextureMat.rows())); dstMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1); dstMatClippingROI = new Mat(dstMat, processingAreaRect); // fill all black. //Imgproc.rectangle (dstMat, new Point (0, 0), new Point (dstMat.width (), dstMat.height ()), new Scalar (0, 0, 0, 0), -1); grayMat = new Mat(dstMatClippingROI.rows(), dstMatClippingROI.cols(), CvType.CV_8UC1); lineMat = new Mat(dstMatClippingROI.rows(), dstMatClippingROI.cols(), CvType.CV_8UC1); maskMat = new Mat(dstMatClippingROI.rows(), dstMatClippingROI.cols(), CvType.CV_8UC1); //create a striped background. bgMat = new Mat(dstMatClippingROI.rows(), dstMatClippingROI.cols(), CvType.CV_8UC1, new Scalar(255)); for (int i = 0; i < bgMat.rows() * 2.5f; i = i + 4) { Imgproc.line(bgMat, new Point(0, 0 + i), new Point(bgMat.cols(), -bgMat.cols() + i), new Scalar(0), 1); } grayPixels = new byte[grayMat.cols() * grayMat.rows() * grayMat.channels()]; maskPixels = new byte[maskMat.cols() * maskMat.rows() * maskMat.channels()]; quad_renderer = gameObject.GetComponent <Renderer> () as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(clippingOffset.x, clippingOffset.y)); Matrix4x4 projectionMatrix; #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", vignetteScale); float halfOfVerticalFov = Mathf.Atan(1.0f / projectionMatrix.m11); float aspectRatio = (1.0f / Mathf.Tan(halfOfVerticalFov)) / projectionMatrix.m00; Debug.Log("halfOfVerticalFov " + halfOfVerticalFov); Debug.Log("aspectRatio " + aspectRatio); // //Imgproc.rectangle (dstMat, new Point (0, 0), new Point (webCamTextureMat.width (), webCamTextureMat.height ()), new Scalar (126, 126, 126, 255), -1); // }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat grayMat = webCamTextureToMatHelper.GetMat(); texture = new Texture2D(grayMat.cols(), grayMat.rows(), TextureFormat.Alpha8, false); texture.wrapMode = TextureWrapMode.Clamp; quad_renderer = gameObject.GetComponent <Renderer>() as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(0, 0)); //Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); DebugUtils.AddDebugStr(webCamTextureToMatHelper.GetWidth() + " x " + webCamTextureToMatHelper.GetHeight() + " : " + webCamTextureToMatHelper.GetFPS()); if (enableDownScale) { DebugUtils.AddDebugStr("enableDownScale = true: " + imageOptimizationHelper.downscaleRatio + " / " + webCamTextureToMatHelper.GetWidth() / imageOptimizationHelper.downscaleRatio + " x " + webCamTextureToMatHelper.GetHeight() / imageOptimizationHelper.downscaleRatio); } Matrix4x4 projectionMatrix; #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else // This value is obtained from PhotoCapture's TryGetProjectionMatrix() method. I do not know whether this method is good. // Please see the discussion of this thread. Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f); cascade = new CascadeClassifier(); cascade.load(Utils.getFilePath("lbpcascade_frontalface.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif grayMat4Thread = new Mat(); cascade4Thread = new CascadeClassifier(); cascade4Thread.load(Utils.getFilePath("haarcascade_frontalface_alt.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade4Thread.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API // HololensCameraStream always returns image data in BGRA format. texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.BGRA32, false); #else texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); #endif texture.wrapMode = TextureWrapMode.Clamp; Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); Debug.Log("ScreenMat.width " + webCamTextureMat.width() + " ScreenMat.height " + webCamTextureMat.height()); processingAreaRect = new OpenCVForUnity.Rect((int)(webCamTextureMat.cols() * (outsideClippingRatio.x - clippingOffset.x)), (int)(webCamTextureMat.rows() * (outsideClippingRatio.y + clippingOffset.y)), (int)(webCamTextureMat.cols() * (1f - outsideClippingRatio.x * 2)), (int)(webCamTextureMat.rows() * (1f - outsideClippingRatio.y * 2))); processingAreaRect = processingAreaRect.intersect(new OpenCVForUnity.Rect(0, 0, webCamTextureMat.cols(), webCamTextureMat.rows())); grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1); // //HL quad_renderer = gameObject.GetComponent <Renderer>() as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(clippingOffset.x, clippingOffset.y)); Matrix4x4 projectionMatrix; #if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else //19n //Matrix4x4 projectionMatrix2 = webCamTextureToMatHelper.GetProjectionMatrix(); //Matrix4x4 camera2WorldMatrix = webCamTextureToMatHelper.GetCameraToWorldMatrix(); //HoloLensCameraStream.Resolution _resolution = CameraStreamHelper.Instance.GetLowestResolution(); //Vector3 imageCenterDirection = LocatableCameraUtils.PixelCoordToWorldCoord(camera2WorldMatrix, projectionMatrix2, _resolution, new Vector2(_resolution.width / 2, _resolution.height / 2)); //Vector3 imageBotRightDirection = LocatableCameraUtils.PixelCoordToWorldCoord(camera2WorldMatrix, projectionMatrix2, _resolution, new Vector2(_resolution.width, _resolution.height)); ////_laser.ShootLaserFrom(camera2WorldMatrix.GetColumn(3), imageBotRightDirection, 10f, _botRightMaterial); //Debug.Log(imageBotRightDirection); //HL //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", vignetteScale); float halfOfVerticalFov = Mathf.Atan(1.0f / projectionMatrix.m11); float aspectRatio = (1.0f / Mathf.Tan(halfOfVerticalFov)) / projectionMatrix.m00; Debug.Log("halfOfVerticalFov " + halfOfVerticalFov); Debug.Log("aspectRatio " + aspectRatio); //13n monotracker = TrackerKCF.create(); bbox = new Rect2d(); selectedPointList = new List <Point>(); SetupGestureRecognizer(); }