void UpdateReceiving()
{
if (_DepthImageSize != _ReceiverClient.DepthImageSize)
{
_DepthImageSize = _ReceiverClient.DepthImageSize;
_TrvlDecoder = new TemporalRVLDecoder(_DepthImageSize);
_DepthImageRawData = new byte[_DepthImageSize * sizeof(short)];
}
if (_DepthImageTexture == null ||
_DepthImageTexture.width != _ReceiverClient.DepthWidth ||
_DepthImageTexture.height != _ReceiverClient.DepthHeight)
{
int width = _ReceiverClient.DepthWidth;
int height = _ReceiverClient.DepthHeight;
_DepthImageTexture = new Texture2D(width, height, TextureFormat.R16, false);
}
if (_ColorImageTexture == null ||
_ColorImageTexture.width != _ReceiverClient.ColorWidth ||
_ColorImageTexture.height != _ReceiverClient.ColorHeight)
{
int width = _ReceiverClient.ColorWidth;
int height = _ReceiverClient.ColorHeight;
_ColorImageTexture = new Texture2D(width, height, TextureFormat.BGRA32, false);
}
Frame frame = _ReceiverClient.GetFrame();
if (frame != null)
{
// Debug.Log("Frame count: " + frame.FrameCount);
bool isKeyFrame = frame.IsKeyFrame;
byte[] encodedDepthData = frame.EncodedDepthData;
if (frame.CompressionMethod == CompressionMethod.TemporalRVL)
{
// Temporal RVL decompression
_DecodedDepthData = _TrvlDecoder.Decode(encodedDepthData, isKeyFrame);
}
else if (frame.CompressionMethod == CompressionMethod.RVL)
{
// RVL decompression
RVL.DecompressRVL(encodedDepthData, _DecodedDepthData);
}
Buffer.BlockCopy(_DecodedDepthData, 0, _DepthImageRawData, 0, _DepthImageRawData.Length * sizeof(byte));
_DepthImageTexture.LoadRawTextureData(_DepthImageRawData);
_DepthImageTexture.Apply();
_ColorImageData = frame.ColorImageData;
_ColorImageTexture.LoadImage(_ColorImageData);
_ColorImageTexture.Apply();
}
}
void Update()
{
if (_KinectSensor.RawDepthImage != null)
{
// Visualize original depth image
short[] depthImage = _KinectSensor.RawDepthImage;
Buffer.BlockCopy(depthImage, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DepthImageTexture.LoadRawTextureData(_DepthRawData);
_DepthImageTexture.Apply();
_Stopwatch.Reset();
_Stopwatch.Start();
// RVL compression
int encodedDataBytes = RVL.CompressRVL(depthImage, _EncodedDepthData);
_Stopwatch.Stop();
long encodingTimeMillseconds = _Stopwatch.ElapsedMilliseconds;
_Stopwatch.Reset();
_Stopwatch.Start();
// RVL decompression
RVL.DecompressRVL(_EncodedDepthData, _DecodedDepthData);
_Stopwatch.Stop();
long decodingTimeMillseconds = _Stopwatch.ElapsedMilliseconds;
// Visualize decoded depth image
Buffer.BlockCopy(_DecodedDepthData, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DecodedDepthImageTexture.LoadRawTextureData(_DepthRawData);
_DecodedDepthImageTexture.Apply();
// Difference of depth images
for (int i = 0; i < depthImage.Length; i++)
{
_Diff[i] = (short)Math.Abs(depthImage[i] - _DecodedDepthData[i]);
}
// Visualize diff image
Buffer.BlockCopy(_Diff, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DiffImageTexture.LoadRawTextureData(_DepthRawData);
_DiffImageTexture.Apply();
// Display info
int originalDepthDataSize = depthImage.Length * sizeof(short);
int compressedDepthDataSize = encodedDataBytes;
float compressionRatio = originalDepthDataSize / compressedDepthDataSize;
_DepthImageSize.text = string.Format("Size: {2:#,0} [bytes] Resolution: {0}x{1}",
_DepthImageTexture.width, _DepthImageTexture.height, originalDepthDataSize);
_CompressedDepthImageSize.text = string.Format("Size: {0:#,0} [bytes] Data compression ratio: {1:F1}",
compressedDepthDataSize, compressionRatio);
_ProcessingTime.text = string.Format("Processing time:\n Encode: {0} [ms]\n Decode: {1} [ms]",
encodingTimeMillseconds, decodingTimeMillseconds);
}
if (_KinectSensor.RawColorImage != null)
{
_ColorImageTexture.LoadRawTextureData(_KinectSensor.RawColorImage);
_ColorImageTexture.Apply();
}
}
void UpdateStreaming()
{
if (_KinectSensor.RawDepthImage != null)
{
// Original depth image
short[] depthImage = _KinectSensor.RawDepthImage;
Buffer.BlockCopy(depthImage, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DepthImageTexture.LoadRawTextureData(_DepthRawData);
_DepthImageTexture.Apply();
_KeyFrame = (_FrameCount++ % _KeyFrameInterval == 0);
if (_DepthCompressionMethod == CompressionMethod.TemporalRVL)
{
// Temporal RVL compression
_EncodedDepthData = _TrvlEncoder.Encode(depthImage, _KeyFrame);
_CompressedDepthDataSize = _EncodedDepthData.Length;
// Temporal RVL decompression
_DecodedDepthData = _TrvlDecoder.Decode(_EncodedDepthData, _KeyFrame);
}
else if (_DepthCompressionMethod == CompressionMethod.RVL)
{
// RVL compression
_CompressedDepthDataSize = RVL.CompressRVL(depthImage, _EncodedDepthData);
// RVL decompression
RVL.DecompressRVL(_EncodedDepthData, _DecodedDepthData);
}
_OriginalDepthDataSize = depthImage.Length * sizeof(ushort);
_CompressionRatio = ((float)_OriginalDepthDataSize / _CompressedDepthDataSize);
// Decoded depth image
Buffer.BlockCopy(_DecodedDepthData, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DecodedDepthImageTexture.LoadRawTextureData(_DepthRawData);
_DecodedDepthImageTexture.Apply();
// Difference of original and decoded image
for (int i = 0; i < depthImage.Length; i++)
{
_Diff[i] = (short)Math.Abs(depthImage[i] - _DecodedDepthData[i]);
}
// Visualize diff image
Buffer.BlockCopy(_Diff, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DiffImageTexture.LoadRawTextureData(_DepthRawData);
_DiffImageTexture.Apply();
}
if (_KinectSensor.TransformedColorImage != null)
{
_ColorImageTexture.LoadRawTextureData(_KinectSensor.TransformedColorImage);
_ColorImageTexture.Apply();
_EncodedColorImageData = ImageConversion.EncodeToJPG(_ColorImageTexture);
_CompressedColorDataSize = _EncodedColorImageData.Length;
}
_StreamingClient.SendDepthAndColorData(_DepthCompressionMethod, _EncodedDepthData,
_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, _KeyFrame,
_EncodedColorImageData, _ColorImageTexture.width, _ColorImageTexture.height, _FrameCount);
}