public void UpdateFrame(UdpSocket udpSocket, List <Tuple <int, VideoSenderMessageData> > videoMessageList)
{
// If texture is not created, create and assign them to quads.
if (!prepared)
{
// Check whether the native plugin has Direct3D textures that
// can be connected to Unity textures.
if (textureGroup.IsInitialized())
{
// TextureGroup includes Y, U, V, and a depth texture.
azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());
prepared = true;
UnityEngine.Debug.Log("textureGroup intialized");
}
}
{
foreach (var frameMessagePair in videoMessageList)
{
// C# Dictionary throws an error when you add an element with
// a key that is already taken.
if (videoMessages.ContainsKey(frameMessagePair.Item1))
{
continue;
}
videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
}
}
if (videoMessages.Count == 0)
{
return;
}
int?beginFrameId = null;
// If there is a key frame, use the most recent one.
foreach (var frameMessagePair in videoMessages)
{
if (frameMessagePair.Key <= lastVideoFrameId)
{
continue;
}
if (frameMessagePair.Value.keyframe)
{
beginFrameId = frameMessagePair.Key;
}
}
// When there is no key frame, go through all the frames to check
// if there is the one right after the previously rendered one.
if (!beginFrameId.HasValue)
{
if (videoMessages.ContainsKey(lastVideoFrameId + 1))
{
beginFrameId = lastVideoFrameId + 1;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
var decoderStopWatch = Stopwatch.StartNew();
for (int i = beginFrameId.Value; ; ++i)
{
if (!videoMessages.ContainsKey(i))
{
break;
}
var frameMessage = videoMessages[i];
lastVideoFrameId = i;
var colorEncoderFrame = frameMessage.colorEncoderFrame;
var depthEncoderFrame = frameMessage.depthEncoderFrame;
ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(sessionId,
lastVideoFrameId,
(float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds), endPoint);
// Invokes a function to be called in a render thread.
if (prepared)
{
//Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
textureGroup.SetFFmpegFrame(ffmpegFrame);
//Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
textureGroup.SetTrvlFrame(trvlFrame);
PluginHelper.UpdateTextureGroup(textureGroup.GetId());
}
// Remove frame messages before the rendered frame.
var frameMessageKeys = new List <int>();
foreach (int key in videoMessages.Keys)
{
frameMessageKeys.Add(key);
}
foreach (int key in frameMessageKeys)
{
if (key < lastVideoFrameId)
{
videoMessages.Remove(key);
}
}
}
private void UpdateTextureGroup()
{
{
Tuple <int, VideoSenderMessageData> frameMessagePair;
while (videoMessageQueue.TryDequeue(out frameMessagePair))
{
// C# Dictionary throws an error when you add an element with
// a key that is already taken.
if (videoMessages.ContainsKey(frameMessagePair.Item1))
{
continue;
}
videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
}
}
if (videoMessages.Count == 0)
{
return;
}
int?beginFrameId = null;
// If there is a key frame, use the most recent one.
foreach (var frameMessagePair in videoMessages)
{
if (frameMessagePair.Key <= lastVideoFrameId)
{
continue;
}
if (frameMessagePair.Value.keyframe)
{
beginFrameId = frameMessagePair.Key;
}
}
// When there is no key frame, go through all the frames to check
// if there is the one right after the previously rendered one.
if (!beginFrameId.HasValue)
{
if (videoMessages.ContainsKey(lastVideoFrameId + 1))
{
beginFrameId = lastVideoFrameId + 1;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
var decoderStopWatch = Stopwatch.StartNew();
//for (int i = beginIndex.Value; i < frameMessages.Count; ++i)
//{
// var frameMessage = frameMessages[i];
for (int i = beginFrameId.Value; ; ++i)
{
if (!videoMessages.ContainsKey(i))
{
break;
}
var frameMessage = videoMessages[i];
lastVideoFrameId = i;
var colorEncoderFrame = frameMessage.colorEncoderFrame;
var depthEncoderFrame = frameMessage.depthEncoderFrame;
ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
//print($"id: {lastFrameId}, packet collection time: {packetCollectionTime.TotalMilliseconds}, " +
// $"decoder time: {decoderTime.TotalMilliseconds}, frame time: {frameTime.TotalMilliseconds}");
udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(lastVideoFrameId,
(float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds));
// Invokes a function to be called in a render thread.
if (preapared)
{
//Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
textureGroup.SetFFmpegFrame(ffmpegFrame);
//Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
textureGroup.SetTrvlFrame(trvlFrame);
PluginHelper.UpdateTextureGroup();
}
// Remove frame messages before the rendered frame.
var frameMessageKeys = new List <int>();
foreach (int key in videoMessages.Keys)
{
frameMessageKeys.Add(key);
}
foreach (int key in frameMessageKeys)
{
if (key < lastVideoFrameId)
{
videoMessages.Remove(key);
}
}
}