public SoftwareBitmap GetImage() { if (!webcamInitialized) { return(null); } SensorFrame latestFrame; latestFrame = _holoLensMediaFrameSourceGroup.GetLatestSensorFrame(SensorType.PhotoVideo); if (latestFrame == null || latestFrame.Timestamp == lastFrameTimestamp) { return(null); } lastFrameTimestamp = latestFrame.Timestamp; webcamToWorldMatrix.m00 = latestFrame.FrameToOrigin.M11; webcamToWorldMatrix.m01 = latestFrame.FrameToOrigin.M21; webcamToWorldMatrix.m02 = latestFrame.FrameToOrigin.M31; webcamToWorldMatrix.m10 = latestFrame.FrameToOrigin.M12; webcamToWorldMatrix.m11 = latestFrame.FrameToOrigin.M22; webcamToWorldMatrix.m12 = latestFrame.FrameToOrigin.M32; webcamToWorldMatrix.m20 = -latestFrame.FrameToOrigin.M13; webcamToWorldMatrix.m21 = -latestFrame.FrameToOrigin.M23; webcamToWorldMatrix.m22 = -latestFrame.FrameToOrigin.M33; webcamToWorldMatrix.m03 = latestFrame.FrameToOrigin.Translation.X; webcamToWorldMatrix.m13 = latestFrame.FrameToOrigin.Translation.Y; webcamToWorldMatrix.m23 = -latestFrame.FrameToOrigin.Translation.Z; webcamToWorldMatrix.m33 = 1; if (imageInitialized == false) { height = latestFrame.SoftwareBitmap.PixelHeight; width = latestFrame.SoftwareBitmap.PixelWidth; projectionMatrix = new Matrix4x4(); projectionMatrix.m00 = 2 * latestFrame.CameraIntrinsics.FocalLength.X / width; projectionMatrix.m11 = 2 * latestFrame.CameraIntrinsics.FocalLength.Y / height; projectionMatrix.m02 = -2 * (latestFrame.CameraIntrinsics.PrincipalPoint.X - width / 2) / width; projectionMatrix.m12 = 2 * (latestFrame.CameraIntrinsics.PrincipalPoint.Y - height / 2) / height; projectionMatrix.m22 = -1; projectionMatrix.m33 = -1; imageInitialized = true; } return(latestFrame.SoftwareBitmap); }
// Get the latest frame from hololens media // frame source group -- not needed unsafe void UpdateHoloLensMediaFrameSourceGroup() { #if ENABLE_WINMD_SUPPORT if (!_mediaFrameSourceGroupsStarted || _pvMediaFrameSourceGroup == null) { return; } // Destroy all marker gameobject instances from prior frames // otherwise game objects will pile on top of marker if (_markerGOs.Count != 0) { foreach (var marker in _markerGOs) { Destroy(marker); } } // Get latest sensor frames // Photo video SensorFrame latestPvCameraFrame = _pvMediaFrameSourceGroup.GetLatestSensorFrame( _sensorType); if (latestPvCameraFrame == null) { return; } // Detect ArUco markers in current frame // https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#void%20Rodrigues(InputArray%20src,%20OutputArray%20dst,%20OutputArray%20jacobian) IList <DetectedArUcoMarker> detectedArUcoMarkers = new List <DetectedArUcoMarker>(); detectedArUcoMarkers = _arUcoMarkerTracker.DetectArUcoMarkersInFrame(latestPvCameraFrame); // If we detect a marker, display if (detectedArUcoMarkers.Count != 0) { foreach (var detectedMarker in detectedArUcoMarkers) { // Get pose from OpenCV and format for Unity Vector3 position = CvUtils.Vec3FromFloat3(detectedMarker.Position); position.y *= -1f; Quaternion rotation = CvUtils.RotationQuatFromRodrigues(CvUtils.Vec3FromFloat3(detectedMarker.Rotation)); Matrix4x4 cameraToWorldUnity = CvUtils.Mat4x4FromFloat4x4(detectedMarker.CameraToWorldUnity); Matrix4x4 transformUnityCamera = CvUtils.TransformInUnitySpace(position, rotation); // Use camera to world transform to get world pose of marker Matrix4x4 transformUnityWorld = cameraToWorldUnity * transformUnityCamera; // Instantiate game object marker in world coordinates var thisGo = Instantiate( markerGo, CvUtils.GetVectorFromMatrix(transformUnityWorld), CvUtils.GetQuatFromMatrix(transformUnityWorld)) as GameObject; // Scale the game object to the size of the markers thisGo.transform.localScale = new Vector3(markerSize, markerSize, markerSize); _markerGOs.Add(thisGo); } } // Convert the frame to be unity viewable var pvFrame = SoftwareBitmap.Convert( latestPvCameraFrame.SoftwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Ignore); // Display the incoming pv frames as a texture. // Set texture to the desired renderer Destroy(_pvTexture); _pvTexture = new Texture2D( pvFrame.PixelWidth, pvFrame.PixelHeight, TextureFormat.BGRA32, false); // Get byte array, update unity material with texture (RGBA) byte *inBytesPV = GetByteArrayFromSoftwareBitmap(pvFrame); _pvTexture.LoadRawTextureData((IntPtr)inBytesPV, pvFrame.PixelWidth * pvFrame.PixelHeight * 4); _pvTexture.Apply(); _pvMaterial.mainTexture = _pvTexture; myText.text = "Began streaming sensor frames. Double tap to end streaming."; #endif }
// Get the latest frame from hololens media // frame source group -- not needed unsafe void UpdateHoloLensMediaFrameSourceGroup() { #if ENABLE_WINMD_SUPPORT if (!_mediaFrameSourceGroupsStarted || _pvMediaFrameSourceGroup == null || _shortDepthMediaFrameSourceGroup == null) { return; } // Get latest sensor frames // Photo video SensorFrame latestPvCameraFrame = _pvMediaFrameSourceGroup.GetLatestSensorFrame( _sensorType); // ToF Depth SensorFrame latestShortDepthCameraFrame = _shortDepthMediaFrameSourceGroup.GetLatestSensorFrame( _sensorTypeResearch); // Initialize depth pv mapper class to cache // the resulting depth transform. if (!_isDepthPvMapperInit && latestShortDepthCameraFrame != null) { _depthPvMapper = new DepthPvMapper( latestShortDepthCameraFrame); _isDepthPvMapperInit = true; } // Map depth frames to photo video camera // with from/to range and specified radius. SensorFrame latestPvDepthFrame = _depthPvMapper.MapDepthToPV( latestPvCameraFrame, latestShortDepthCameraFrame, depthRangeFrom, depthRangeTo, patchRadius); // Convert the frame to be unity viewable var pvDepthFrame = SoftwareBitmap.Convert( latestPvDepthFrame.SoftwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Ignore); // Display the incoming pv frames as a texture. // Set texture to the desired renderer Destroy(_pvDepthTexture); _pvDepthTexture = new Texture2D( pvDepthFrame.PixelWidth, pvDepthFrame.PixelHeight, TextureFormat.BGRA32, false); // Get byte array, update unity material with texture (RGBA) byte *inBytesPV = GetByteArrayFromSoftwareBitmap(pvDepthFrame); _pvDepthTexture.LoadRawTextureData((IntPtr)inBytesPV, pvDepthFrame.PixelWidth * pvDepthFrame.PixelHeight * 4); _pvDepthTexture.Apply(); _pvDepthMaterial.mainTexture = _pvDepthTexture; myText.text = "Began streaming sensor frames. Double tap to end streaming."; #endif }