private static void VideoRenderer(object parameter) { var state = (SharedState)parameter; var videoTrack = state.VideoTrack; var peerConnection = videoTrack.PeerConnection; var logger = state.Logger; try { DateTime startTime = default; int frameIndex = 0; // Create swap-chain for displaying rendered images on the server // var sw = new Stopwatch(); using (var clock = new PreciseWaitableClock(EventResetMode.AutoReset)) using (var renderer = CreateRenderer(videoTrack, logger)) { while (Thread.CurrentThread.IsAlive && !peerConnection.IsDisposed) { if (peerConnection.SignalingState == SignalingState.Stable) { if (frameIndex == 0) { startTime = clock.GetCurrentTime(); } MouseMessage lastMouseMessage = null; while (state.MouseMessageQueue.TryDequeue(out var mouseMessage)) { lastMouseMessage = mouseMessage; } if (lastMouseMessage != null) { // Render mouse events as quickly as possible. renderer.MousePosition = lastMouseMessage.Kind != MouseEventKind.Up ? lastMouseMessage.Pos : (RawVector2?)null; } var elapsedTime = TimeSpan.FromTicks(frameIndex * TimeSpan.TicksPerSecond / videoTrack.FrameRate); renderer.SendFrame(elapsedTime); // Wait until we can render the next frame var currentTime = clock.GetCurrentTime(); var skippedFrameCount = 0; for (; ;) { var nextFrameTime = startTime.AddTicks(++frameIndex * TimeSpan.TicksPerSecond / videoTrack.FrameRate); if (nextFrameTime >= currentTime) { clock.SetFutureEventTime(nextFrameTime); break; } ++skippedFrameCount; } if (skippedFrameCount > 0) { logger.LogWarning($"Skipped {skippedFrameCount} frames!"); } // Wait for the next frame. clock.WaitHandle.WaitOne(); } else { // Wait until peer connection is stable before sending frames. Thread.Sleep(500); } } } } catch (ThreadInterruptedException) { } catch (Exception ex) { logger.LogError(ex, "Error in RtcRendererServer thread"); } }
private static unsafe void Render() { const int frameWidth = 2560; const int frameHeight = 1440; const int frameRate = 60; // var options = VideoEncoderOptions.OptimizedFor(frameWidth, frameHeight, frameRate); var options = new VideoEncoderOptions { MaxBitsPerSecond = 12_000_000, MinBitsPerSecond = 10_000_000, MaxFramesPerSecond = frameRate }; using (var sender = new ObservablePeerConnection(new PeerConnectionOptions())) using (var receiver = new ObservablePeerConnection(new PeerConnectionOptions { CanReceiveVideo = true })) { using (var vt = new ObservableVideoTrack(sender, options)) { using (var rnd = new BouncingBallRenderer(vt, 10, new BoundingBallOptions { VideoFrameWidth = frameWidth, VideoFrameHeight = frameHeight, VideoFrameQueueSize = 2 })) { receiver.Connect( Observable.Never <DataMessage>(), sender.LocalSessionDescriptionStream, sender.LocalIceCandidateStream); sender.Connect( Observable.Never <DataMessage>(), receiver.LocalSessionDescriptionStream, receiver.LocalIceCandidateStream); sender.CreateOffer(); int remoteVideoFrameReceivedCount = 0; receiver.RemoteVideoFrameReceived += (pc, frame) => { remoteVideoFrameReceivedCount += 1; // Save as JPEG for debugging. SLOW! // TODO: Doesn't work yet, H264 decoding not yet supported, only VP8 //if (frame is VideoFrameYuvAlpha yuvFrame && yuvFrame.Width == yuvFrame.StrideY) //{ // var span = new ReadOnlySpan<byte>(yuvFrame.DataY.ToPointer(), yuvFrame.Width * yuvFrame.Height); // using (var image = Image.LoadPixelData<Y8>(span, yuvFrame.Width, yuvFrame.Height)) // { // image.Save($@"frame_{remoteVideoFrameReceivedCount:D000000}.bmp"); // } //} }; using (var clock = new PreciseWaitableClock(EventResetMode.AutoReset)) { var startTime = clock.GetCurrentTime().AddSeconds(1); var nextTime = startTime; // The remote peer connection is not immediately ready to receive frames, // so we keep sending until it succeeds. // TODO: Figure out what webrtc event can be used for this. while (!Console.KeyAvailable) { clock.SetFutureEventTime(nextTime); clock.WaitHandle.WaitOne(); var elapsedTime = clock.GetCurrentTime() - startTime; rnd.SendFrame(elapsedTime); nextTime = nextTime.AddSeconds(1.0 / frameRate); } } } // The video renderer is now disposed while the video track is still encoding some textures // This should not crash. // We need to wait a while before disposing the video-track and peer-connection to check this. Thread.Sleep(100); } } } }