public TrvlFrame Decode(byte[] frame, bool keyframe)
{
IntPtr bytes = Marshal.AllocHGlobal(frame.Length);
Marshal.Copy(frame, 0, bytes, frame.Length);
var trvlFrame = new TrvlFrame(Plugin.trvl_decoder_decode(ptr, bytes, frame.Length, keyframe));
Marshal.FreeHGlobal(bytes);
return(trvlFrame);
}
void Update()
{
// Space key resets the scene to be placed in front of the camera.
if (Input.GetKeyDown(KeyCode.Space))
{
ResetView();
}
// Sends virtual keyboards strokes to the TextMeshes for the IP address and the port.
AbsorbInput();
// If texture is not created, create and assign them to quads.
if (textureGroup == null)
{
// Check whether the native plugin has Direct3D textures that
// can be connected to Unity textures.
if (Plugin.texture_group_get_y_texture_view().ToInt64() != 0)
{
// TextureGroup includes Y, U, V, and a depth texture.
textureGroup = new TextureGroup(Plugin.texture_group_get_width(),
Plugin.texture_group_get_height());
azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.YTexture);
azureKinectScreenMaterial.SetTexture("_UTex", textureGroup.UTexture);
azureKinectScreenMaterial.SetTexture("_VTex", textureGroup.VTexture);
azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.DepthTexture);
print("textureGroup intialized");
}
}
if (receiver == null)
{
return;
}
while (initPacketQueue.TryDequeue(out byte[] packet))
{
int cursor = 0;
//int sessionId = BitConverter.ToInt32(packet, cursor);
cursor += 4;
//var packetType = packet[cursor];
cursor += 1;
var calibration = ManagerHelper.ReadAzureKinectCalibrationFromMessage(packet, cursor);
Plugin.texture_group_set_width(calibration.DepthCamera.Width);
Plugin.texture_group_set_height(calibration.DepthCamera.Height);
PluginHelper.InitTextureGroup();
colorDecoder = new Vp8Decoder();
depthDecoder = new TrvlDecoder(calibration.DepthCamera.Width * calibration.DepthCamera.Height);
azureKinectScreen.Setup(calibration);
}
while (frameMessageQueue.TryDequeue(out FrameMessage frameMessage))
{
frameMessages.Add(frameMessage);
}
frameMessages.Sort((x, y) => x.FrameId.CompareTo(y.FrameId));
if (frameMessages.Count == 0)
{
return;
}
int?beginIndex = null;
// If there is a key frame, use the most recent one.
for (int i = frameMessages.Count - 1; i >= 0; --i)
{
if (frameMessages[i].Keyframe)
{
beginIndex = i;
break;
}
}
// When there is no key frame, go through all the frames if the first
// FrameMessage is the one right after the previously rendered one.
if (!beginIndex.HasValue)
{
if (frameMessages[0].FrameId == lastFrameId + 1)
{
beginIndex = 0;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
TimeSpan packetCollectionTime;
var decoderStopWatch = Stopwatch.StartNew();
for (int i = beginIndex.Value; i < frameMessages.Count; ++i)
{
var frameMessage = frameMessages[i];
lastFrameId = frameMessage.FrameId;
packetCollectionTime = frameMessage.PacketCollectionTime;
var colorEncoderFrame = frameMessage.GetColorEncoderFrame();
var depthEncoderFrame = frameMessage.GetDepthEncoderFrame();
IntPtr colorEncoderFrameBytes = Marshal.AllocHGlobal(colorEncoderFrame.Length);
Marshal.Copy(colorEncoderFrame, 0, colorEncoderFrameBytes, colorEncoderFrame.Length);
ffmpegFrame = colorDecoder.Decode(colorEncoderFrameBytes, colorEncoderFrame.Length);
Marshal.FreeHGlobal(colorEncoderFrameBytes);
IntPtr depthEncoderFrameBytes = Marshal.AllocHGlobal(depthEncoderFrame.Length);
Marshal.Copy(depthEncoderFrame, 0, depthEncoderFrameBytes, depthEncoderFrame.Length);
trvlFrame = depthDecoder.Decode(depthEncoderFrameBytes, frameMessage.Keyframe);
Marshal.FreeHGlobal(depthEncoderFrameBytes);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
print($"id: {lastFrameId}, packet collection time: {packetCollectionTime.TotalMilliseconds}," +
$"decoder time: {decoderTime.TotalMilliseconds}, frame time: {frameTime.TotalMilliseconds}");
receiver.Send(lastFrameId, (float)packetCollectionTime.TotalMilliseconds, (float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds, summaryPacketCount);
summaryPacketCount = 0;
// Invokes a function to be called in a render thread.
if (textureGroup != null)
{
Plugin.texture_group_set_ffmpeg_frame(ffmpegFrame.Ptr);
Plugin.texture_group_set_depth_pixels(trvlFrame.Ptr);
PluginHelper.UpdateTextureGroup();
}
frameMessages = new List <FrameMessage>();
}
public void SetTrvlFrame(TrvlFrame trvlFrame)
{
Plugin.texture_group_set_depth_pixels(ptr, trvlFrame.Ptr);
}
public void UpdateFrame(UdpSocket udpSocket, List <Tuple <int, VideoSenderMessageData> > videoMessageList)
{
// If texture is not created, create and assign them to quads.
if (!prepared)
{
// Check whether the native plugin has Direct3D textures that
// can be connected to Unity textures.
if (textureGroup.IsInitialized())
{
// TextureGroup includes Y, U, V, and a depth texture.
azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());
prepared = true;
UnityEngine.Debug.Log("textureGroup intialized");
}
}
{
foreach (var frameMessagePair in videoMessageList)
{
// C# Dictionary throws an error when you add an element with
// a key that is already taken.
if (videoMessages.ContainsKey(frameMessagePair.Item1))
{
continue;
}
videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
}
}
if (videoMessages.Count == 0)
{
return;
}
int?beginFrameId = null;
// If there is a key frame, use the most recent one.
foreach (var frameMessagePair in videoMessages)
{
if (frameMessagePair.Key <= lastVideoFrameId)
{
continue;
}
if (frameMessagePair.Value.keyframe)
{
beginFrameId = frameMessagePair.Key;
}
}
// When there is no key frame, go through all the frames to check
// if there is the one right after the previously rendered one.
if (!beginFrameId.HasValue)
{
if (videoMessages.ContainsKey(lastVideoFrameId + 1))
{
beginFrameId = lastVideoFrameId + 1;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
var decoderStopWatch = Stopwatch.StartNew();
for (int i = beginFrameId.Value; ; ++i)
{
if (!videoMessages.ContainsKey(i))
{
break;
}
var frameMessage = videoMessages[i];
lastVideoFrameId = i;
var colorEncoderFrame = frameMessage.colorEncoderFrame;
var depthEncoderFrame = frameMessage.depthEncoderFrame;
ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(sessionId,
lastVideoFrameId,
(float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds), endPoint);
// Invokes a function to be called in a render thread.
if (prepared)
{
//Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
textureGroup.SetFFmpegFrame(ffmpegFrame);
//Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
textureGroup.SetTrvlFrame(trvlFrame);
PluginHelper.UpdateTextureGroup(textureGroup.GetId());
}
// Remove frame messages before the rendered frame.
var frameMessageKeys = new List <int>();
foreach (int key in videoMessages.Keys)
{
frameMessageKeys.Add(key);
}
foreach (int key in frameMessageKeys)
{
if (key < lastVideoFrameId)
{
videoMessages.Remove(key);
}
}
}
private void UpdateTextureGroup()
{
{
Tuple <int, VideoSenderMessageData> frameMessagePair;
while (videoMessageQueue.TryDequeue(out frameMessagePair))
{
// C# Dictionary throws an error when you add an element with
// a key that is already taken.
if (videoMessages.ContainsKey(frameMessagePair.Item1))
{
continue;
}
videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
}
}
if (videoMessages.Count == 0)
{
return;
}
int?beginFrameId = null;
// If there is a key frame, use the most recent one.
foreach (var frameMessagePair in videoMessages)
{
if (frameMessagePair.Key <= lastVideoFrameId)
{
continue;
}
if (frameMessagePair.Value.keyframe)
{
beginFrameId = frameMessagePair.Key;
}
}
// When there is no key frame, go through all the frames to check
// if there is the one right after the previously rendered one.
if (!beginFrameId.HasValue)
{
if (videoMessages.ContainsKey(lastVideoFrameId + 1))
{
beginFrameId = lastVideoFrameId + 1;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
var decoderStopWatch = Stopwatch.StartNew();
//for (int i = beginIndex.Value; i < frameMessages.Count; ++i)
//{
// var frameMessage = frameMessages[i];
for (int i = beginFrameId.Value; ; ++i)
{
if (!videoMessages.ContainsKey(i))
{
break;
}
var frameMessage = videoMessages[i];
lastVideoFrameId = i;
var colorEncoderFrame = frameMessage.colorEncoderFrame;
var depthEncoderFrame = frameMessage.depthEncoderFrame;
ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
//print($"id: {lastFrameId}, packet collection time: {packetCollectionTime.TotalMilliseconds}, " +
// $"decoder time: {decoderTime.TotalMilliseconds}, frame time: {frameTime.TotalMilliseconds}");
udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(lastVideoFrameId,
(float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds));
// Invokes a function to be called in a render thread.
if (preapared)
{
//Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
textureGroup.SetFFmpegFrame(ffmpegFrame);
//Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
textureGroup.SetTrvlFrame(trvlFrame);
PluginHelper.UpdateTextureGroup();
}
// Remove frame messages before the rendered frame.
var frameMessageKeys = new List <int>();
foreach (int key in videoMessages.Keys)
{
frameMessageKeys.Add(key);
}
foreach (int key in frameMessageKeys)
{
if (key < lastVideoFrameId)
{
videoMessages.Remove(key);
}
}
}
