public FFmpegFrame Decode(byte[] frame) { IntPtr bytes = Marshal.AllocHGlobal(frame.Length); Marshal.Copy(frame, 0, bytes, frame.Length); var ffmpegFrame = new FFmpegFrame(Plugin.vp8_decoder_decode(ptr, bytes, frame.Length)); Marshal.FreeHGlobal(bytes); return(ffmpegFrame); }
void Update() { // Space key resets the scene to be placed in front of the camera. if (Input.GetKeyDown(KeyCode.Space)) { ResetView(); } // Sends virtual keyboards strokes to the TextMeshes for the IP address and the port. AbsorbInput(); // If texture is not created, create and assign them to quads. if (textureGroup == null) { // Check whether the native plugin has Direct3D textures that // can be connected to Unity textures. if (Plugin.texture_group_get_y_texture_view().ToInt64() != 0) { // TextureGroup includes Y, U, V, and a depth texture. textureGroup = new TextureGroup(Plugin.texture_group_get_width(), Plugin.texture_group_get_height()); azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.YTexture); azureKinectScreenMaterial.SetTexture("_UTex", textureGroup.UTexture); azureKinectScreenMaterial.SetTexture("_VTex", textureGroup.VTexture); azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.DepthTexture); print("textureGroup intialized"); } } if (receiver == null) { return; } while (initPacketQueue.TryDequeue(out byte[] packet)) { int cursor = 0; //int sessionId = BitConverter.ToInt32(packet, cursor); cursor += 4; //var packetType = packet[cursor]; cursor += 1; var calibration = ManagerHelper.ReadAzureKinectCalibrationFromMessage(packet, cursor); Plugin.texture_group_set_width(calibration.DepthCamera.Width); Plugin.texture_group_set_height(calibration.DepthCamera.Height); PluginHelper.InitTextureGroup(); colorDecoder = new Vp8Decoder(); depthDecoder = new TrvlDecoder(calibration.DepthCamera.Width * calibration.DepthCamera.Height); azureKinectScreen.Setup(calibration); } while (frameMessageQueue.TryDequeue(out FrameMessage frameMessage)) { frameMessages.Add(frameMessage); } frameMessages.Sort((x, y) => x.FrameId.CompareTo(y.FrameId)); if (frameMessages.Count == 0) { return; } int?beginIndex = null; // If there is a key frame, use the most recent one. for (int i = frameMessages.Count - 1; i >= 0; --i) { if (frameMessages[i].Keyframe) { beginIndex = i; break; } } // When there is no key frame, go through all the frames if the first // FrameMessage is the one right after the previously rendered one. if (!beginIndex.HasValue) { if (frameMessages[0].FrameId == lastFrameId + 1) { beginIndex = 0; } else { // Wait for more frames if there is way to render without glitches. return; } } // ffmpegFrame and trvlFrame are guaranteed to be non-null // since the existence of beginIndex's value. FFmpegFrame ffmpegFrame = null; TrvlFrame trvlFrame = null; TimeSpan packetCollectionTime; var decoderStopWatch = Stopwatch.StartNew(); for (int i = beginIndex.Value; i < frameMessages.Count; ++i) { var frameMessage = frameMessages[i]; lastFrameId = frameMessage.FrameId; packetCollectionTime = frameMessage.PacketCollectionTime; var colorEncoderFrame = frameMessage.GetColorEncoderFrame(); var depthEncoderFrame = frameMessage.GetDepthEncoderFrame(); IntPtr colorEncoderFrameBytes = Marshal.AllocHGlobal(colorEncoderFrame.Length); Marshal.Copy(colorEncoderFrame, 0, colorEncoderFrameBytes, colorEncoderFrame.Length); ffmpegFrame = colorDecoder.Decode(colorEncoderFrameBytes, colorEncoderFrame.Length); Marshal.FreeHGlobal(colorEncoderFrameBytes); IntPtr depthEncoderFrameBytes = Marshal.AllocHGlobal(depthEncoderFrame.Length); Marshal.Copy(depthEncoderFrame, 0, depthEncoderFrameBytes, depthEncoderFrame.Length); trvlFrame = depthDecoder.Decode(depthEncoderFrameBytes, frameMessage.Keyframe); Marshal.FreeHGlobal(depthEncoderFrameBytes); } decoderStopWatch.Stop(); var decoderTime = decoderStopWatch.Elapsed; frameStopWatch.Stop(); var frameTime = frameStopWatch.Elapsed; frameStopWatch = Stopwatch.StartNew(); print($"id: {lastFrameId}, packet collection time: {packetCollectionTime.TotalMilliseconds}," + $"decoder time: {decoderTime.TotalMilliseconds}, frame time: {frameTime.TotalMilliseconds}"); receiver.Send(lastFrameId, (float)packetCollectionTime.TotalMilliseconds, (float)decoderTime.TotalMilliseconds, (float)frameTime.TotalMilliseconds, summaryPacketCount); summaryPacketCount = 0; // Invokes a function to be called in a render thread. if (textureGroup != null) { Plugin.texture_group_set_ffmpeg_frame(ffmpegFrame.Ptr); Plugin.texture_group_set_depth_pixels(trvlFrame.Ptr); PluginHelper.UpdateTextureGroup(); } frameMessages = new List <FrameMessage>(); }
public void UpdateFrame(UdpSocket udpSocket, List <Tuple <int, VideoSenderMessageData> > videoMessageList) { // If texture is not created, create and assign them to quads. if (!prepared) { // Check whether the native plugin has Direct3D textures that // can be connected to Unity textures. if (textureGroup.IsInitialized()) { // TextureGroup includes Y, U, V, and a depth texture. azureKinectScreenMaterial.SetTexture("_YTex", textureGroup.GetYTexture()); azureKinectScreenMaterial.SetTexture("_UvTex", textureGroup.GetUvTexture()); azureKinectScreenMaterial.SetTexture("_DepthTex", textureGroup.GetDepthTexture()); prepared = true; UnityEngine.Debug.Log("textureGroup intialized"); } } { foreach (var frameMessagePair in videoMessageList) { // C# Dictionary throws an error when you add an element with // a key that is already taken. if (videoMessages.ContainsKey(frameMessagePair.Item1)) { continue; } videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2); } } if (videoMessages.Count == 0) { return; } int?beginFrameId = null; // If there is a key frame, use the most recent one. foreach (var frameMessagePair in videoMessages) { if (frameMessagePair.Key <= lastVideoFrameId) { continue; } if (frameMessagePair.Value.keyframe) { beginFrameId = frameMessagePair.Key; } } // When there is no key frame, go through all the frames to check // if there is the one right after the previously rendered one. if (!beginFrameId.HasValue) { if (videoMessages.ContainsKey(lastVideoFrameId + 1)) { beginFrameId = lastVideoFrameId + 1; } else { // Wait for more frames if there is way to render without glitches. return; } } // ffmpegFrame and trvlFrame are guaranteed to be non-null // since the existence of beginIndex's value. FFmpegFrame ffmpegFrame = null; TrvlFrame trvlFrame = null; var decoderStopWatch = Stopwatch.StartNew(); for (int i = beginFrameId.Value; ; ++i) { if (!videoMessages.ContainsKey(i)) { break; } var frameMessage = videoMessages[i]; lastVideoFrameId = i; var colorEncoderFrame = frameMessage.colorEncoderFrame; var depthEncoderFrame = frameMessage.depthEncoderFrame; ffmpegFrame = colorDecoder.Decode(colorEncoderFrame); trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe); } decoderStopWatch.Stop(); var decoderTime = decoderStopWatch.Elapsed; frameStopWatch.Stop(); var frameTime = frameStopWatch.Elapsed; frameStopWatch = Stopwatch.StartNew(); udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(sessionId, lastVideoFrameId, (float)decoderTime.TotalMilliseconds, (float)frameTime.TotalMilliseconds), endPoint); // Invokes a function to be called in a render thread. if (prepared) { //Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr); textureGroup.SetFFmpegFrame(ffmpegFrame); //Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr); textureGroup.SetTrvlFrame(trvlFrame); PluginHelper.UpdateTextureGroup(textureGroup.GetId()); } // Remove frame messages before the rendered frame. var frameMessageKeys = new List <int>(); foreach (int key in videoMessages.Keys) { frameMessageKeys.Add(key); } foreach (int key in frameMessageKeys) { if (key < lastVideoFrameId) { videoMessages.Remove(key); } } }
public void SetFFmpegFrame(FFmpegFrame ffmpegFrame) { Plugin.texture_group_set_ffmpeg_frame(ptr, ffmpegFrame.Ptr); }
private void UpdateTextureGroup() { { Tuple <int, VideoSenderMessageData> frameMessagePair; while (videoMessageQueue.TryDequeue(out frameMessagePair)) { // C# Dictionary throws an error when you add an element with // a key that is already taken. if (videoMessages.ContainsKey(frameMessagePair.Item1)) { continue; } videoMessages.Add(frameMessagePair.Item1, frameMessagePair.Item2); } } if (videoMessages.Count == 0) { return; } int?beginFrameId = null; // If there is a key frame, use the most recent one. foreach (var frameMessagePair in videoMessages) { if (frameMessagePair.Key <= lastVideoFrameId) { continue; } if (frameMessagePair.Value.keyframe) { beginFrameId = frameMessagePair.Key; } } // When there is no key frame, go through all the frames to check // if there is the one right after the previously rendered one. if (!beginFrameId.HasValue) { if (videoMessages.ContainsKey(lastVideoFrameId + 1)) { beginFrameId = lastVideoFrameId + 1; } else { // Wait for more frames if there is way to render without glitches. return; } } // ffmpegFrame and trvlFrame are guaranteed to be non-null // since the existence of beginIndex's value. FFmpegFrame ffmpegFrame = null; TrvlFrame trvlFrame = null; var decoderStopWatch = Stopwatch.StartNew(); //for (int i = beginIndex.Value; i < frameMessages.Count; ++i) //{ // var frameMessage = frameMessages[i]; for (int i = beginFrameId.Value; ; ++i) { if (!videoMessages.ContainsKey(i)) { break; } var frameMessage = videoMessages[i]; lastVideoFrameId = i; var colorEncoderFrame = frameMessage.colorEncoderFrame; var depthEncoderFrame = frameMessage.depthEncoderFrame; ffmpegFrame = colorDecoder.Decode(colorEncoderFrame); trvlFrame = depthDecoder.Decode(depthEncoderFrame, frameMessage.keyframe); } decoderStopWatch.Stop(); var decoderTime = decoderStopWatch.Elapsed; frameStopWatch.Stop(); var frameTime = frameStopWatch.Elapsed; frameStopWatch = Stopwatch.StartNew(); //print($"id: {lastFrameId}, packet collection time: {packetCollectionTime.TotalMilliseconds}, " + // $"decoder time: {decoderTime.TotalMilliseconds}, frame time: {frameTime.TotalMilliseconds}"); udpSocket.Send(PacketHelper.createReportReceiverPacketBytes(lastVideoFrameId, (float)decoderTime.TotalMilliseconds, (float)frameTime.TotalMilliseconds)); // Invokes a function to be called in a render thread. if (preapared) { //Plugin.texture_group_set_ffmpeg_frame(textureGroup, ffmpegFrame.Ptr); textureGroup.SetFFmpegFrame(ffmpegFrame); //Plugin.texture_group_set_depth_pixels(textureGroup, trvlFrame.Ptr); textureGroup.SetTrvlFrame(trvlFrame); PluginHelper.UpdateTextureGroup(); } // Remove frame messages before the rendered frame. var frameMessageKeys = new List <int>(); foreach (int key in videoMessages.Keys) { frameMessageKeys.Add(key); } foreach (int key in frameMessageKeys) { if (key < lastVideoFrameId) { videoMessages.Remove(key); } } }