private static void VideoRenderer(object parameter)
{
var state = (SharedState)parameter;
var videoTrack = state.VideoTrack;
var peerConnection = videoTrack.PeerConnection;
var logger = state.Logger;
try
{
DateTime startTime = default;
int frameIndex = 0;
// Create swap-chain for displaying rendered images on the server
// var sw = new Stopwatch();
using (var clock = new PreciseWaitableClock(EventResetMode.AutoReset))
using (var renderer = CreateRenderer(videoTrack, logger))
{
while (Thread.CurrentThread.IsAlive && !peerConnection.IsDisposed)
{
if (peerConnection.SignalingState == SignalingState.Stable)
{
if (frameIndex == 0)
{
startTime = clock.GetCurrentTime();
}
MouseMessage lastMouseMessage = null;
while (state.MouseMessageQueue.TryDequeue(out var mouseMessage))
{
lastMouseMessage = mouseMessage;
}
if (lastMouseMessage != null)
{
// Render mouse events as quickly as possible.
renderer.MousePosition = lastMouseMessage.Kind != MouseEventKind.Up
? lastMouseMessage.Pos
: (RawVector2?)null;
}
var elapsedTime = TimeSpan.FromTicks(frameIndex * TimeSpan.TicksPerSecond / videoTrack.FrameRate);
renderer.SendFrame(elapsedTime);
// Wait until we can render the next frame
var currentTime = clock.GetCurrentTime();
var skippedFrameCount = 0;
for (; ;)
{
var nextFrameTime = startTime.AddTicks(++frameIndex * TimeSpan.TicksPerSecond / videoTrack.FrameRate);
if (nextFrameTime >= currentTime)
{
clock.SetFutureEventTime(nextFrameTime);
break;
}
++skippedFrameCount;
}
if (skippedFrameCount > 0)
{
logger.LogWarning($"Skipped {skippedFrameCount} frames!");
}
// Wait for the next frame.
clock.WaitHandle.WaitOne();
}
else
{
// Wait until peer connection is stable before sending frames.
Thread.Sleep(500);
}
}
}
}
catch (ThreadInterruptedException)
{
}
catch (Exception ex)
{
logger.LogError(ex, "Error in RtcRendererServer thread");
}
}
private static unsafe void Render()
{
const int frameWidth = 2560;
const int frameHeight = 1440;
const int frameRate = 60;
// var options = VideoEncoderOptions.OptimizedFor(frameWidth, frameHeight, frameRate);
var options = new VideoEncoderOptions
{
MaxBitsPerSecond = 12_000_000,
MinBitsPerSecond = 10_000_000,
MaxFramesPerSecond = frameRate
};
using (var sender = new ObservablePeerConnection(new PeerConnectionOptions()))
using (var receiver = new ObservablePeerConnection(new PeerConnectionOptions {
CanReceiveVideo = true
}))
{
using (var vt = new ObservableVideoTrack(sender, options))
{
using (var rnd = new BouncingBallRenderer(vt, 10, new BoundingBallOptions
{
VideoFrameWidth = frameWidth,
VideoFrameHeight = frameHeight,
VideoFrameQueueSize = 2
}))
{
receiver.Connect(
Observable.Never <DataMessage>(),
sender.LocalSessionDescriptionStream,
sender.LocalIceCandidateStream);
sender.Connect(
Observable.Never <DataMessage>(),
receiver.LocalSessionDescriptionStream,
receiver.LocalIceCandidateStream);
sender.CreateOffer();
int remoteVideoFrameReceivedCount = 0;
receiver.RemoteVideoFrameReceived += (pc, frame) =>
{
remoteVideoFrameReceivedCount += 1;
// Save as JPEG for debugging. SLOW!
// TODO: Doesn't work yet, H264 decoding not yet supported, only VP8
//if (frame is VideoFrameYuvAlpha yuvFrame && yuvFrame.Width == yuvFrame.StrideY)
//{
// var span = new ReadOnlySpan<byte>(yuvFrame.DataY.ToPointer(), yuvFrame.Width * yuvFrame.Height);
// using (var image = Image.LoadPixelData<Y8>(span, yuvFrame.Width, yuvFrame.Height))
// {
// image.Save($@"frame_{remoteVideoFrameReceivedCount:D000000}.bmp");
// }
//}
};
using (var clock = new PreciseWaitableClock(EventResetMode.AutoReset))
{
var startTime = clock.GetCurrentTime().AddSeconds(1);
var nextTime = startTime;
// The remote peer connection is not immediately ready to receive frames,
// so we keep sending until it succeeds.
// TODO: Figure out what webrtc event can be used for this.
while (!Console.KeyAvailable)
{
clock.SetFutureEventTime(nextTime);
clock.WaitHandle.WaitOne();
var elapsedTime = clock.GetCurrentTime() - startTime;
rnd.SendFrame(elapsedTime);
nextTime = nextTime.AddSeconds(1.0 / frameRate);
}
}
}
// The video renderer is now disposed while the video track is still encoding some textures
// This should not crash.
// We need to wait a while before disposing the video-track and peer-connection to check this.
Thread.Sleep(100);
}
}
}
}
