public void UpdateFrame()
    {
        // If texture is not created, create and assign them to quads.
        if (!preapared)
        {
            // Check whether the native plugin has Direct3D textures that
            // can be connected to Unity textures.
            if (textureGroup.IsInitialized())
            {
                // TextureGroup includes Y, U, V, and a depth texture.
                azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
                azureKinectScreenMaterial.SetTexture("_UTex", textureGroup.GetUTexture());
                azureKinectScreenMaterial.SetTexture("_VTex", textureGroup.GetVTexture());
                azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());

                preapared = true;

                UnityEngine.Debug.Log("textureGroup intialized");
            }
        }

        if (udpSocket == null)
        {
            return;
        }

        UpdateTextureGroup();
    }
예제 #2
0
    public IEnumerator SetupTextureGroup(VideoInitSenderPacketData initPacketData)
    {
        if (state != PrepareState.Unprepared)
        {
            throw new Exception("State has to be Unprepared to prepare TextureGroupUpdater.");
        }

        state = PrepareState.Preparing;

        textureGroup.SetWidth(initPacketData.depthWidth);
        textureGroup.SetHeight(initPacketData.depthHeight);
        PluginHelper.InitTextureGroup(textureGroup.GetId());

        depthDecoder = new TrvlDecoder(initPacketData.depthWidth * initPacketData.depthHeight);

        state = PrepareState.Prepared;

        while (!textureGroup.IsInitialized())
        {
            yield return(null);
        }

        // TextureGroup includes Y, U, V, and a depth texture.
        azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
        azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
        azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());

        state = PrepareState.Prepared;
    }
예제 #3
0
    public void UpdateFrame(UdpSocket udpSocket, List <Tuple <int, VideoSenderMessageData> > videoMessageList)
    {
        // If texture is not created, create and assign them to quads.
        if (!prepared)
        {
            // Check whether the native plugin has Direct3D textures that
            // can be connected to Unity textures.
            if (textureGroup.IsInitialized())
            {
                // TextureGroup includes Y, U, V, and a depth texture.
                azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture());
                azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture());
                azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture());

                prepared = true;

                UnityEngine.Debug.Log("textureGroup intialized");
            }
        }

        {
            foreach (var frameMessagePair in videoMessageList)
            {
                // C# Dictionary throws an error when you add an element with
                // a key that is already taken.
                if (videoMessages.ContainsKey(frameMessagePair.Item1))
                {
                    continue;
                }

                videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2);
            }
        }

        if (videoMessages.Count == 0)
        {
            return;
        }

        int?beginFrameId = null;

        // If there is a key frame, use the most recent one.
        foreach (var frameMessagePair in videoMessages)
        {
            if (frameMessagePair.Key <= lastVideoFrameId)
            {
                continue;
            }

            if (frameMessagePair.Value.keyframe)
            {
                beginFrameId = frameMessagePair.Key;
            }
        }

        // When there is no key frame, go through all the frames to check
        // if there is the one right after the previously rendered one.
        if (!beginFrameId.HasValue)
        {
            if (videoMessages.ContainsKey(lastVideoFrameId + 1))
            {
                beginFrameId = lastVideoFrameId + 1;
            }
            else
            {
                // Wait for more frames if there is way to render without glitches.
                return;
            }
        }

        // ffmpegFrame and trvlFrame are guaranteed to be non-null
        // since the existence of beginIndex's value.
        FFmpegFrame ffmpegFrame = null;
        TrvlFrame   trvlFrame   = null;

        var decoderStopWatch = Stopwatch.StartNew();

        for (int i = beginFrameId.Value; ; ++i)
        {
            if (!videoMessages.ContainsKey(i))
            {
                break;
            }

            var frameMessage = videoMessages[i];
            lastVideoFrameId = i;

            var colorEncoderFrame = frameMessage.colorEncoderFrame;
            var depthEncoderFrame = frameMessage.depthEncoderFrame;

            ffmpegFrame = colorDecoder.Decode(colorEncoderFrame);
            trvlFrame   = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe);
        }

        decoderStopWatch.Stop();
        var decoderTime = decoderStopWatch.Elapsed;

        frameStopWatch.Stop();
        var frameTime = frameStopWatch.Elapsed;

        frameStopWatch = Stopwatch.StartNew();

        udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(sessionId,
                                                                    lastVideoFrameId,
                                                                    (float)decoderTime.TotalMilliseconds,
                                                                    (float)frameTime.TotalMilliseconds), endPoint);

        // Invokes a function to be called in a render thread.
        if (prepared)
        {
            //Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr);
            textureGroup.SetFFmpegFrame(ffmpegFrame);
            //Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr);
            textureGroup.SetTrvlFrame(trvlFrame);
            PluginHelper.UpdateTextureGroup(textureGroup.GetId());
        }

        // Remove frame messages before the rendered frame.
        var frameMessageKeys = new List <int>();

        foreach (int key in videoMessages.Keys)
        {
            frameMessageKeys.Add(key);
        }
        foreach (int key in frameMessageKeys)
        {
            if (key < lastVideoFrameId)
            {
                videoMessages.Remove(key);
            }
        }
    }