void Start() { _KinectSensor = _AzureKinectManager.Sensor; if (_KinectSensor != null) { Debug.Log("ColorResolution: " + _KinectSensor.ColorImageWidth + "x" + _KinectSensor.ColorImageHeight); Debug.Log("DepthResolution: " + _KinectSensor.DepthImageWidth + "x" + _KinectSensor.DepthImageHeight); _TransformedColorImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.BGRA32, false); _DepthRawData = new byte[_KinectSensor.DepthImageWidth * _KinectSensor.DepthImageHeight * sizeof(ushort)]; _DepthImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false); _ColorImageTexture = new Texture2D(_KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight, TextureFormat.BGRA32, false); _TransformedDepthRawData = new byte[_KinectSensor.ColorImageWidth * _KinectSensor.ColorImageHeight * sizeof(ushort)]; _TransformedDepthImageTexture = new Texture2D(_KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight, TextureFormat.R16, false); _PointCloudRenderer = GetComponent <PointCloudRenderer>(); CameraCalibration deviceDepthCameraCalibration = _KinectSensor.DeviceCalibration.DepthCameraCalibration; CameraCalibration deviceColorCameraCalibration = _KinectSensor.DeviceCalibration.ColorCameraCalibration; K4A.Calibration calibration = new K4A.Calibration(); calibration.DepthCameraCalibration = CreateCalibrationCamera(deviceDepthCameraCalibration, _KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight); calibration.ColorCameraCalibration = CreateCalibrationCamera(deviceColorCameraCalibration, _KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight); #if COLOR_TO_DEPTH _PointCloudRenderer.GenerateMesh(calibration, K4A.CalibrationType.Depth); #else _PointCloudRenderer.GenerateMesh(calibration, K4A.CalibrationType.Color); #endif } }
void SendCalibrationToReceiver(int streamingClientId, int receiverClientId) { if (!_calibrationTypeDictionary.ContainsKey(streamingClientId) || !_calibrationDictionary.ContainsKey(streamingClientId)) { return; } K4A.CalibrationType calibrationType = _calibrationTypeDictionary[streamingClientId]; K4A.Calibration calibration = _calibrationDictionary[streamingClientId]; BinaryFormatter binaryFormatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(); binaryFormatter.Serialize(memoryStream, calibration); byte[] serializedCalibration = memoryStream.ToArray(); _dataWriter.Reset(); _dataWriter.Put((int)NetworkDataType.ReceiveCalibration); _dataWriter.Put((int)calibrationType); _dataWriter.Put(serializedCalibration.Length); _dataWriter.Put(serializedCalibration); _liteNetLibServer.SendData(receiverClientId, _dataWriter, DeliveryMethod.ReliableOrdered); Debug.Log("Send calibration of client: " + streamingClientId + " to client: " + receiverClientId); }
void Start() { List <AzureKinectSensor> kinectSensors = _AzureKinectManager.SensorList; if (_DeviceNumber < kinectSensors.Count) { _KinectSensor = _AzureKinectManager.SensorList[_DeviceNumber]; if (_KinectSensor != null) { Debug.Log("ColorResolution: " + _KinectSensor.ColorImageWidth + "x" + _KinectSensor.ColorImageHeight); Debug.Log("DepthResolution: " + _KinectSensor.DepthImageWidth + "x" + _KinectSensor.DepthImageHeight); _DepthRawData = new byte[_KinectSensor.DepthImageWidth * _KinectSensor.DepthImageHeight * sizeof(ushort)]; CameraCalibration deviceDepthCameraCalibration = _KinectSensor.DeviceCalibration.DepthCameraCalibration; CameraCalibration deviceColorCameraCalibration = _KinectSensor.DeviceCalibration.ColorCameraCalibration; K4A.Calibration calibration = new K4A.Calibration(); calibration.DepthCameraCalibration = CreateCalibrationCamera(deviceDepthCameraCalibration, _KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight); calibration.ColorCameraCalibration = CreateCalibrationCamera(deviceColorCameraCalibration, _KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight); _PointCloudRenderer = GetComponent <PointCloudRenderer>(); _PointCloudRenderer.GenerateMesh(calibration, K4A.CalibrationType.Depth); _AzureKinectManager.OpenDevice(_DeviceNumber); } } }
public void Initialize(AzureKinectSensor kinectSensor) { if (!_Initialized) { _KinectSensor = kinectSensor; if (_KinectSensor != null) { Debug.Log("ColorResolution: " + _KinectSensor.ColorImageWidth + "x" + _KinectSensor.ColorImageHeight); Debug.Log("DepthResolution: " + _KinectSensor.DepthImageWidth + "x" + _KinectSensor.DepthImageHeight); // _LowPassFilter = new ExponentialSmoothingLowPassFilter((uint)_Accel.Length, 0.05f); // _LowPassFilter = new DoubleExponentialSmoothingLowPassFilter((uint)_Accel.Length, 0.3f, 0.3f); _LowPassFilter = new ButterworthFilter(_Order, _SamplingFrequency, _CutoffFrequency, (uint)_Accel.Length); _DepthRawData = new byte[_KinectSensor.DepthImageWidth * _KinectSensor.DepthImageHeight * sizeof(ushort)]; _PointCloudRenderer = GetComponent <PointCloudRenderer>(); CameraCalibration deviceDepthCameraCalibration = _KinectSensor.DeviceCalibration.DepthCameraCalibration; CameraCalibration deviceColorCameraCalibration = _KinectSensor.DeviceCalibration.ColorCameraCalibration; K4A.Calibration calibration = new K4A.Calibration(); calibration.DepthCameraCalibration = CreateCalibrationCamera(deviceDepthCameraCalibration, _KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight); calibration.ColorCameraCalibration = CreateCalibrationCamera(deviceColorCameraCalibration, _KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight); _PointCloudRenderer.GenerateMesh(calibration, K4A.CalibrationType.Depth); _Initialized = true; } } }
void OnReceivedCalibration(K4A.CalibrationType calibrationType, K4A.Calibration calibration) { Debug.Log("CalibrationType: " + calibrationType); K4A.CalibrationCamera depthCalibrationCamera = calibration.DepthCameraCalibration; K4A.CalibrationCamera colorCalibrationCamera = calibration.ColorCameraCalibration; Debug.Log("Calibration.DepthImage: " + depthCalibrationCamera.resolutionWidth + "x" + depthCalibrationCamera.resolutionHeight); Debug.Log("Calibration.ColorImage: " + colorCalibrationCamera.resolutionWidth + "x" + colorCalibrationCamera.resolutionHeight); _PointCloudRenderer.GenerateMesh(calibration, calibrationType); }
public void SendCalibration(K4A.CalibrationType calibrationType, K4A.Calibration calibration) { BinaryFormatter binaryFormatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(); binaryFormatter.Serialize(memoryStream, calibration); byte[] serializedCalibration = memoryStream.ToArray(); _dataWriter.Reset(); _dataWriter.Put((int)NetworkDataType.SendCalibration); _dataWriter.Put((int)calibrationType); _dataWriter.Put(serializedCalibration.Length); _dataWriter.Put(serializedCalibration); _liteNetLibClient.SendData(_dataWriter, DeliveryMethod.ReliableOrdered); }
void OnReceivedCalibrationHandler(NetPeer peer, NetPacketReader reader) { Debug.Log("OnReceivedCalibration"); K4A.CalibrationType calibrationType = (K4A.CalibrationType)reader.GetInt(); Debug.Log("OnReceivedCalibrationType: " + calibrationType); int dataLength = reader.GetInt(); byte[] serializedCalibration = new byte[dataLength]; reader.GetBytes(serializedCalibration, dataLength); BinaryFormatter binaryFormatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(serializedCalibration); K4A.Calibration calibration = (K4A.Calibration)binaryFormatter.Deserialize(memoryStream); OnReceivedCalibration?.Invoke(calibrationType, calibration); }
void SetCalibration(int streamingClientId, NetPacketReader reader) { K4A.CalibrationType calibrationType = (K4A.CalibrationType)reader.GetInt(); int dataLength = reader.GetInt(); byte[] serializedCalibration = new byte[dataLength]; reader.GetBytes(serializedCalibration, dataLength); BinaryFormatter binaryFormatter = new BinaryFormatter(); MemoryStream memoryStream = new MemoryStream(serializedCalibration); K4A.Calibration calibration = (K4A.Calibration)binaryFormatter.Deserialize(memoryStream); if (!_calibrationTypeDictionary.ContainsKey(streamingClientId)) { _calibrationTypeDictionary.Add(streamingClientId, calibrationType); } if (!_calibrationDictionary.ContainsKey(streamingClientId)) { _calibrationDictionary.Add(streamingClientId, calibration); } }
public void Initialize() { _KinectSensor = _AzureKinectManager.Sensor; if (_KinectSensor != null) { Debug.Log("ColorResolution: " + _KinectSensor.ColorImageWidth + "x" + _KinectSensor.ColorImageHeight); Debug.Log("DepthResolution: " + _KinectSensor.DepthImageWidth + "x" + _KinectSensor.DepthImageHeight); _DepthImageSize = _KinectSensor.DepthImageWidth * _KinectSensor.DepthImageHeight; _DepthRawData = new byte[_DepthImageSize * sizeof(short)]; _Diff = new short[_DepthImageSize]; _EncodedColorImageData = new byte[_DepthImageSize]; _DepthImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false); _DecodedDepthImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false); _DiffImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false); _ColorImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.BGRA32, false); _TrvlEncoder = new TemporalRVLEncoder(_DepthImageSize, 10, 2); _TrvlDecoder = new TemporalRVLDecoder(_DepthImageSize); CameraCalibration deviceDepthCameraCalibration = _KinectSensor.DeviceCalibration.DepthCameraCalibration; CameraCalibration deviceColorCameraCalibration = _KinectSensor.DeviceCalibration.ColorCameraCalibration; _Calibration = new K4A.Calibration(); _Calibration.DepthCameraCalibration = CreateCalibrationCamera(deviceDepthCameraCalibration, _KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight); _Calibration.ColorCameraCalibration = CreateCalibrationCamera(deviceColorCameraCalibration, _KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight); _CalibrationType = K4A.CalibrationType.Depth; // Color to depth _Initialized = true; } else { Debug.LogError("KinectSensor is null!"); } }
public void GenerateMesh(K4A.Calibration calibration, K4A.CalibrationType calibrationType) { int width = 0; int height = 0; K4A.CalibrationExtrinsics extrinsics = null; switch (calibrationType) { case K4A.CalibrationType.Depth: width = calibration.DepthCameraCalibration.resolutionWidth; height = calibration.DepthCameraCalibration.resolutionHeight; extrinsics = calibration.DepthCameraCalibration.extrinsics; extrinsics = calibration.ColorCameraCalibration.extrinsics; extrinsics.translation[0] /= 1000.0f; // [millimeters] -> [meters] extrinsics.translation[1] /= 1000.0f; // [millimeters] -> [meters] extrinsics.translation[2] /= 1000.0f; // [millimeters] -> [meters] break; case K4A.CalibrationType.Color: width = calibration.ColorCameraCalibration.resolutionWidth; height = calibration.ColorCameraCalibration.resolutionHeight; extrinsics = calibration.DepthCameraCalibration.extrinsics; break; default: Debug.LogError("Unexpected camera calibration type," + "should either be K4A.CalibrationType.Depth or K4A.CalibrationType.Color"); return; } if (width <= 0 || height <= 0) { Debug.LogError("Camera resolution is invalid: " + width + "x" + height); return; } if (_Mesh != null) { _Mesh.Clear(); } else { _Mesh = new Mesh(); _Mesh.indexFormat = IndexFormat.UInt32; } float[] srcXYZ = new float[3]; float[] dstXYZ = new float[3]; Vector3[] vertices = new Vector3[width * height]; float[] xyTables; if (K4A.Transformation.InitXyTables(calibration, calibrationType, out xyTables)) { int pixelCount = 0; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { int index = 2 * pixelCount; srcXYZ[0] = xyTables[index]; srcXYZ[1] = xyTables[index + 1]; srcXYZ[2] = 1.0f; K4A.ExtrinsicTransformation.ApplyExtrinsicTransformation(extrinsics, srcXYZ, ref dstXYZ); vertices[x + y * width] = new Vector3(dstXYZ[0], dstXYZ[1], dstXYZ[2]); pixelCount++; } } } else { for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { vertices[x + y * width] = new Vector3(0f, 0f, 0f); } } } _Mesh.vertices = vertices; Vector2[] uv = new Vector2[width * height]; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { uv[x + y * width].x = x / (float)width; uv[x + y * width].y = y / (float)height; } } _Mesh.uv = uv; int[] indices = new int[vertices.Length]; for (int i = 0; i < vertices.Length; i++) { indices[i] = i; } _Mesh.SetIndices(indices, MeshTopology.Points, 0, false); GetComponent <MeshFilter>().mesh = _Mesh; InitializeColorTexture(width, height); InitializeDepthTexture(width, height); }