void SendCalibrationToReceiver(int streamingClientId, int receiverClientId)
{
if (!_calibrationTypeDictionary.ContainsKey(streamingClientId) ||
!_calibrationDictionary.ContainsKey(streamingClientId))
{
return;
}
K4A.CalibrationType calibrationType = _calibrationTypeDictionary[streamingClientId];
K4A.Calibration calibration = _calibrationDictionary[streamingClientId];
BinaryFormatter binaryFormatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
binaryFormatter.Serialize(memoryStream, calibration);
byte[] serializedCalibration = memoryStream.ToArray();
_dataWriter.Reset();
_dataWriter.Put((int)NetworkDataType.ReceiveCalibration);
_dataWriter.Put((int)calibrationType);
_dataWriter.Put(serializedCalibration.Length);
_dataWriter.Put(serializedCalibration);
_liteNetLibServer.SendData(receiverClientId, _dataWriter, DeliveryMethod.ReliableOrdered);
Debug.Log("Send calibration of client: " + streamingClientId + " to client: " + receiverClientId);
}
void OnReceivedCalibration(K4A.CalibrationType calibrationType, K4A.Calibration calibration)
{
Debug.Log("CalibrationType: " + calibrationType);
K4A.CalibrationCamera depthCalibrationCamera = calibration.DepthCameraCalibration;
K4A.CalibrationCamera colorCalibrationCamera = calibration.ColorCameraCalibration;
Debug.Log("Calibration.DepthImage: " + depthCalibrationCamera.resolutionWidth + "x" + depthCalibrationCamera.resolutionHeight);
Debug.Log("Calibration.ColorImage: " + colorCalibrationCamera.resolutionWidth + "x" + colorCalibrationCamera.resolutionHeight);
_PointCloudRenderer.GenerateMesh(calibration, calibrationType);
}
public void SendCalibration(K4A.CalibrationType calibrationType, K4A.Calibration calibration)
{
BinaryFormatter binaryFormatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream();
binaryFormatter.Serialize(memoryStream, calibration);
byte[] serializedCalibration = memoryStream.ToArray();
_dataWriter.Reset();
_dataWriter.Put((int)NetworkDataType.SendCalibration);
_dataWriter.Put((int)calibrationType);
_dataWriter.Put(serializedCalibration.Length);
_dataWriter.Put(serializedCalibration);
_liteNetLibClient.SendData(_dataWriter, DeliveryMethod.ReliableOrdered);
}
void OnReceivedCalibrationHandler(NetPeer peer, NetPacketReader reader)
{
Debug.Log("OnReceivedCalibration");
K4A.CalibrationType calibrationType = (K4A.CalibrationType)reader.GetInt();
Debug.Log("OnReceivedCalibrationType: " + calibrationType);
int dataLength = reader.GetInt();
byte[] serializedCalibration = new byte[dataLength];
reader.GetBytes(serializedCalibration, dataLength);
BinaryFormatter binaryFormatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream(serializedCalibration);
K4A.Calibration calibration = (K4A.Calibration)binaryFormatter.Deserialize(memoryStream);
OnReceivedCalibration?.Invoke(calibrationType, calibration);
}
void SetCalibration(int streamingClientId, NetPacketReader reader)
{
K4A.CalibrationType calibrationType = (K4A.CalibrationType)reader.GetInt();
int dataLength = reader.GetInt();
byte[] serializedCalibration = new byte[dataLength];
reader.GetBytes(serializedCalibration, dataLength);
BinaryFormatter binaryFormatter = new BinaryFormatter();
MemoryStream memoryStream = new MemoryStream(serializedCalibration);
K4A.Calibration calibration = (K4A.Calibration)binaryFormatter.Deserialize(memoryStream);
if (!_calibrationTypeDictionary.ContainsKey(streamingClientId))
{
_calibrationTypeDictionary.Add(streamingClientId, calibrationType);
}
if (!_calibrationDictionary.ContainsKey(streamingClientId))
{
_calibrationDictionary.Add(streamingClientId, calibration);
}
}
public void Initialize()
{
_KinectSensor = _AzureKinectManager.Sensor;
if (_KinectSensor != null)
{
Debug.Log("ColorResolution: " + _KinectSensor.ColorImageWidth + "x" + _KinectSensor.ColorImageHeight);
Debug.Log("DepthResolution: " + _KinectSensor.DepthImageWidth + "x" + _KinectSensor.DepthImageHeight);
_DepthImageSize = _KinectSensor.DepthImageWidth * _KinectSensor.DepthImageHeight;
_DepthRawData = new byte[_DepthImageSize * sizeof(short)];
_Diff = new short[_DepthImageSize];
_EncodedColorImageData = new byte[_DepthImageSize];
_DepthImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false);
_DecodedDepthImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false);
_DiffImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.R16, false);
_ColorImageTexture = new Texture2D(_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, TextureFormat.BGRA32, false);
_TrvlEncoder = new TemporalRVLEncoder(_DepthImageSize, 10, 2);
_TrvlDecoder = new TemporalRVLDecoder(_DepthImageSize);
CameraCalibration deviceDepthCameraCalibration = _KinectSensor.DeviceCalibration.DepthCameraCalibration;
CameraCalibration deviceColorCameraCalibration = _KinectSensor.DeviceCalibration.ColorCameraCalibration;
_Calibration = new K4A.Calibration();
_Calibration.DepthCameraCalibration = CreateCalibrationCamera(deviceDepthCameraCalibration, _KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight);
_Calibration.ColorCameraCalibration = CreateCalibrationCamera(deviceColorCameraCalibration, _KinectSensor.ColorImageWidth, _KinectSensor.ColorImageHeight);
_CalibrationType = K4A.CalibrationType.Depth; // Color to depth
_Initialized = true;
}
else
{
Debug.LogError("KinectSensor is null!");
}
}
public void GenerateMesh(K4A.Calibration calibration, K4A.CalibrationType calibrationType)
{
int width = 0;
int height = 0;
K4A.CalibrationExtrinsics extrinsics = null;
switch (calibrationType)
{
case K4A.CalibrationType.Depth:
width = calibration.DepthCameraCalibration.resolutionWidth;
height = calibration.DepthCameraCalibration.resolutionHeight;
extrinsics = calibration.DepthCameraCalibration.extrinsics;
extrinsics = calibration.ColorCameraCalibration.extrinsics;
extrinsics.translation[0] /= 1000.0f; // [millimeters] -> [meters]
extrinsics.translation[1] /= 1000.0f; // [millimeters] -> [meters]
extrinsics.translation[2] /= 1000.0f; // [millimeters] -> [meters]
break;
case K4A.CalibrationType.Color:
width = calibration.ColorCameraCalibration.resolutionWidth;
height = calibration.ColorCameraCalibration.resolutionHeight;
extrinsics = calibration.DepthCameraCalibration.extrinsics;
break;
default:
Debug.LogError("Unexpected camera calibration type," +
"should either be K4A.CalibrationType.Depth or K4A.CalibrationType.Color");
return;
}
if (width <= 0 || height <= 0)
{
Debug.LogError("Camera resolution is invalid: " + width + "x" + height);
return;
}
if (_Mesh != null)
{
_Mesh.Clear();
}
else
{
_Mesh = new Mesh();
_Mesh.indexFormat = IndexFormat.UInt32;
}
float[] srcXYZ = new float[3];
float[] dstXYZ = new float[3];
Vector3[] vertices = new Vector3[width * height];
float[] xyTables;
if (K4A.Transformation.InitXyTables(calibration, calibrationType, out xyTables))
{
int pixelCount = 0;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
int index = 2 * pixelCount;
srcXYZ[0] = xyTables[index];
srcXYZ[1] = xyTables[index + 1];
srcXYZ[2] = 1.0f;
K4A.ExtrinsicTransformation.ApplyExtrinsicTransformation(extrinsics, srcXYZ, ref dstXYZ);
vertices[x + y * width] = new Vector3(dstXYZ[0], dstXYZ[1], dstXYZ[2]);
pixelCount++;
}
}
}
else
{
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
vertices[x + y * width] = new Vector3(0f, 0f, 0f);
}
}
}
_Mesh.vertices = vertices;
Vector2[] uv = new Vector2[width * height];
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
uv[x + y * width].x = x / (float)width;
uv[x + y * width].y = y / (float)height;
}
}
_Mesh.uv = uv;
int[] indices = new int[vertices.Length];
for (int i = 0; i < vertices.Length; i++)
{
indices[i] = i;
}
_Mesh.SetIndices(indices, MeshTopology.Points, 0, false);
GetComponent <MeshFilter>().mesh = _Mesh;
InitializeColorTexture(width, height);
InitializeDepthTexture(width, height);
}
