void UpdateStreaming()
{
if (_KinectSensor.RawDepthImage != null)
{
// Original depth image
short[] depthImage = _KinectSensor.RawDepthImage;
Buffer.BlockCopy(depthImage, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DepthImageTexture.LoadRawTextureData(_DepthRawData);
_DepthImageTexture.Apply();
_KeyFrame = (_FrameCount++ % _KeyFrameInterval == 0);
if (_DepthCompressionMethod == CompressionMethod.TemporalRVL)
{
// Temporal RVL compression
_EncodedDepthData = _TrvlEncoder.Encode(depthImage, _KeyFrame);
_CompressedDepthDataSize = _EncodedDepthData.Length;
// Temporal RVL decompression
_DecodedDepthData = _TrvlDecoder.Decode(_EncodedDepthData, _KeyFrame);
}
else if (_DepthCompressionMethod == CompressionMethod.RVL)
{
// RVL compression
_CompressedDepthDataSize = RVL.CompressRVL(depthImage, _EncodedDepthData);
// RVL decompression
RVL.DecompressRVL(_EncodedDepthData, _DecodedDepthData);
}
_OriginalDepthDataSize = depthImage.Length * sizeof(ushort);
_CompressionRatio = ((float)_OriginalDepthDataSize / _CompressedDepthDataSize);
// Decoded depth image
Buffer.BlockCopy(_DecodedDepthData, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DecodedDepthImageTexture.LoadRawTextureData(_DepthRawData);
_DecodedDepthImageTexture.Apply();
// Difference of original and decoded image
for (int i = 0; i < depthImage.Length; i++)
{
_Diff[i] = (short)Math.Abs(depthImage[i] - _DecodedDepthData[i]);
}
// Visualize diff image
Buffer.BlockCopy(_Diff, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DiffImageTexture.LoadRawTextureData(_DepthRawData);
_DiffImageTexture.Apply();
}
if (_KinectSensor.TransformedColorImage != null)
{
_ColorImageTexture.LoadRawTextureData(_KinectSensor.TransformedColorImage);
_ColorImageTexture.Apply();
_EncodedColorImageData = ImageConversion.EncodeToJPG(_ColorImageTexture);
_CompressedColorDataSize = _EncodedColorImageData.Length;
}
_StreamingClient.SendDepthAndColorData(_DepthCompressionMethod, _EncodedDepthData,
_KinectSensor.DepthImageWidth, _KinectSensor.DepthImageHeight, _KeyFrame,
_EncodedColorImageData, _ColorImageTexture.width, _ColorImageTexture.height, _FrameCount);
}
void Update()
{
if (_KinectSensor.RawDepthImage != null)
{
// Visualize original depth image
short[] depthImage = _KinectSensor.RawDepthImage;
Buffer.BlockCopy(depthImage, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DepthImageTexture.LoadRawTextureData(_DepthRawData);
_DepthImageTexture.Apply();
bool keyFrame = (_FrameCount++ % _KeyFrameInterval == 0);
_Stopwatch.Reset();
_Stopwatch.Start();
// Temporal RVL compression
_EncodedDepthData = _TrvlEncoder.Encode(depthImage, keyFrame);
_Stopwatch.Stop();
long encodingTimeMillseconds = _Stopwatch.ElapsedMilliseconds;
_Stopwatch.Reset();
_Stopwatch.Start();
// Temporal RVL decompression
_DecodedDepthData = _TrvlDecoder.Decode(_EncodedDepthData, keyFrame);
_Stopwatch.Stop();
long decodingTimeMillseconds = _Stopwatch.ElapsedMilliseconds;
// Visualize decoded depth image
Buffer.BlockCopy(_DecodedDepthData, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DecodedDepthImageTexture.LoadRawTextureData(_DepthRawData);
_DecodedDepthImageTexture.Apply();
// Difference of depth images
for (int i = 0; i < depthImage.Length; i++)
{
_Diff[i] = (short)Math.Abs(depthImage[i] - _DecodedDepthData[i]);
}
// Visualize diff image
Buffer.BlockCopy(_Diff, 0, _DepthRawData, 0, _DepthRawData.Length * sizeof(byte));
_DiffImageTexture.LoadRawTextureData(_DepthRawData);
_DiffImageTexture.Apply();
// Display info
int originalDepthDataSize = depthImage.Length * sizeof(short);
int compressedDepthDataSize = _EncodedDepthData.Length;
float compressionRatio = originalDepthDataSize / compressedDepthDataSize;
_DepthImageSize.text = string.Format("Size: {2:#,0} [bytes] Resolution: {0}x{1}",
_DepthImageTexture.width, _DepthImageTexture.height, originalDepthDataSize);
_CompressedDepthImageSize.text = string.Format("Size: {0:#,0} [bytes] Data compression ratio: {1:F1}",
compressedDepthDataSize, compressionRatio);
_ProcessingTime.text = string.Format("Processing time:\n Encode: {0} [ms]\n Decode: {1} [ms]",
encodingTimeMillseconds, decodingTimeMillseconds);
}
if (_KinectSensor.RawColorImage != null)
{
_ColorImageTexture.LoadRawTextureData(_KinectSensor.RawColorImage);
_ColorImageTexture.Apply();
}
}