public IEnumerator SetupTextureGroup(VideoInitSenderPacketData initPacketData)
{
if (state != PrepareState.Unprepared)
{
throw new Exception("State has to be Unprepared to prepare TextureGroupUpdater.");
}
state = PrepareState.Preparing;
textureGroup.SetWidth(initPacketData.depthWidth);
textureGroup.SetHeight(initPacketData.depthHeight);
PluginHelper.InitTextureGroup(textureGroup.GetId());
depthDecoder = new TrvlDecoder(initPacketData.depthWidth * initPacketData.depthHeight);
state = PrepareState.Prepared;
while (!textureGroup.IsInitialized())
{
yield return(null);
}
// TextureGroup includes Y, U, V, and a depth texture.
azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());
state = PrepareState.Prepared;
}
public TextureGroupUpdater(Material azureKinectScreenMaterial, InitSenderPacketData initPacketData, int sessionId, IPEndPoint endPoint)
{
this.azureKinectScreenMaterial = azureKinectScreenMaterial;
textureGroup = new TextureGroup(Plugin.texture_group_create());
UnityEngine.Debug.Log($"textureGroup id: {textureGroup.GetId()}");
lastVideoFrameId = -1;
prepared = false;
videoMessages = new Dictionary <int, VideoSenderMessageData>();
frameStopWatch = Stopwatch.StartNew();
textureGroup.SetWidth(initPacketData.depthWidth);
textureGroup.SetHeight(initPacketData.depthHeight);
PluginHelper.InitTextureGroup(textureGroup.GetId());
colorDecoder = new Vp8Decoder();
depthDecoder = new TrvlDecoder(initPacketData.depthWidth * initPacketData.depthHeight);
this.sessionId = sessionId;
this.endPoint = endPoint;
}
public void UpdateFrame(UdpSocket udpSocket, List <Tuple <int, VideoSenderMessageData> > videoMessageList)
{
// If texture is not created, create and assign them to quads.
if (!prepared)
{
// Check whether the native plugin has Direct3D textures that
// can be connected to Unity textures.
if (textureGroup.IsInitialized())
{
// TextureGroup includes Y, U, V, and a depth texture.
azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());
prepared = true;
UnityEngine.Debug.Log("textureGroup intialized");
}
}
{
foreach (var frameMessagePair in videoMessageList)
{
// C# Dictionary throws an error when you add an element with
// a key that is already taken.
if (videoMessages.ContainsKey(frameMessagePair.Item1))
{
continue;
}
videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
}
}
if (videoMessages.Count == 0)
{
return;
}
int?beginFrameId = null;
// If there is a key frame, use the most recent one.
foreach (var frameMessagePair in videoMessages)
{
if (frameMessagePair.Key <= lastVideoFrameId)
{
continue;
}
if (frameMessagePair.Value.keyframe)
{
beginFrameId = frameMessagePair.Key;
}
}
// When there is no key frame, go through all the frames to check
// if there is the one right after the previously rendered one.
if (!beginFrameId.HasValue)
{
if (videoMessages.ContainsKey(lastVideoFrameId + 1))
{
beginFrameId = lastVideoFrameId + 1;
}
else
{
// Wait for more frames if there is way to render without glitches.
return;
}
}
// ffmpegFrame and trvlFrame are guaranteed to be non-null
// since the existence of beginIndex's value.
FFmpegFrame ffmpegFrame = null;
TrvlFrame trvlFrame = null;
var decoderStopWatch = Stopwatch.StartNew();
for (int i = beginFrameId.Value; ; ++i)
{
if (!videoMessages.ContainsKey(i))
{
break;
}
var frameMessage = videoMessages[i];
lastVideoFrameId = i;
var colorEncoderFrame = frameMessage.colorEncoderFrame;
var depthEncoderFrame = frameMessage.depthEncoderFrame;
ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
}
decoderStopWatch.Stop();
var decoderTime = decoderStopWatch.Elapsed;
frameStopWatch.Stop();
var frameTime = frameStopWatch.Elapsed;
frameStopWatch = Stopwatch.StartNew();
udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(sessionId,
lastVideoFrameId,
(float)decoderTime.TotalMilliseconds,
(float)frameTime.TotalMilliseconds), endPoint);
// Invokes a function to be called in a render thread.
if (prepared)
{
//Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
textureGroup.SetFFmpegFrame(ffmpegFrame);
//Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
textureGroup.SetTrvlFrame(trvlFrame);
PluginHelper.UpdateTextureGroup(textureGroup.GetId());
}
// Remove frame messages before the rendered frame.
var frameMessageKeys = new List <int>();
foreach (int key in videoMessages.Keys)
{
frameMessageKeys.Add(key);
}
foreach (int key in frameMessageKeys)
{
if (key < lastVideoFrameId)
{
videoMessages.Remove(key);
}
}
}