private static IRenderer CreateRenderer(ObservableVideoTrack videoTrack, ILogger logger)
{
PeerConnection.Configure(new GlobalOptions
{
MinimumLogLevel = TraceLevel.Info
});
bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
bool supportsNvEnc = PeerConnection.SupportsHardwareTextureEncoding;
// TODO: Add support for OpenGL, and test it.
// Maybe use https://github.com/mellinoe/veldrid
return(isWindows && supportsNvEnc
? (IRenderer) new BouncingBallRenderer(videoTrack,
8,
new BoundingBallOptions
{
VideoFrameWidth = VideoFrameWidth,
VideoFrameHeight = VideoFrameHeight,
PreviewWindowOptions = new PreviewWindowOptions
{
Width = 1920 / 2
},
TimeRulerOptions = new TimeRulerOptions()
})
: new ImageSharpRenderer(VideoFrameWidth, VideoFrameHeight, videoTrack));
}
public D3D11Renderer(ObservableVideoTrack videoTrack, RendererOptions options)
: base(videoTrack, options)
{
// _factoryDWrite = new DWrite.Factory(DWrite.FactoryType.Shared);
var device2D = new D2D1.Device(DeviceDXGI, new D2D1.CreationProperties
{
DebugLevel = D2D1.DebugLevel.Warning,
ThreadingMode = D2D1.ThreadingMode.MultiThreaded,
Options = D2D1.DeviceContextOptions.None
});
_context2D = new D2D1.DeviceContext(device2D, D2D1.DeviceContextOptions.None);
// Load the background image
using (var factoryWic = new WIC.ImagingFactory2())
using (var decoder = new WIC.JpegBitmapDecoder(factoryWic))
using (var inputStream = new WIC.WICStream(factoryWic, "background-small.jpg", NativeFileAccess.Read))
using (var formatConverter = new WIC.FormatConverter(factoryWic))
using (var bitmapScaler = new WIC.BitmapScaler(factoryWic))
{
decoder.Initialize(inputStream, WIC.DecodeOptions.CacheOnLoad);
formatConverter.Initialize(decoder.GetFrame(0), WIC.PixelFormat.Format32bppPBGRA);
bitmapScaler.Initialize(formatConverter, VideoFrameWidth, VideoFrameHeight,
WIC.BitmapInterpolationMode.Fant);
_backgroundBitmap = D2D1.Bitmap1.FromWicBitmap(_context2D, bitmapScaler);
}
// Create render target
_ballEllipse = new D2D1.Ellipse {
RadiusX = VideoFrameWidth / 20f, RadiusY = VideoFrameWidth / 20f
};
_ballBrush = new D2D1.SolidColorBrush(_context2D, new RawColor4(1f, 1f, 0f, 1f));
}
private static IRenderer CreateRenderer(ObservableVideoTrack videoTrack, ILogger logger)
{
bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
bool supportsNvEnc = PeerConnection.SupportsHardwareTextureEncoding;
// TODO: Add support for OpenGL, and test it.
// Maybe use https://github.com/mellinoe/veldrid
return(isWindows && supportsNvEnc
? (IRenderer) new D3D11Renderer(videoTrack,
new GraphicsD3D11.RendererOptions
{
VideoFrameWidth = VideoFrameWidth,
VideoFrameHeight = VideoFrameHeight,
})
: new ImageSharpRenderer(VideoFrameWidth, VideoFrameHeight, videoTrack));
}
public SharedState(ObservableVideoTrack videoTrack, ILogger logger)
{
VideoTrack = videoTrack;
Logger = logger;
}
public static async Task Run(WebSocket ws, CancellationToken cancellation, ILogger logger)
{
T ToObject <T>(JToken token)
{
var obj = token.ToObject <T>();
//var json = JToken.FromObject(obj);
//Debug.Assert(JToken.DeepEquals(token, json));
return(obj);
}
var renderThread = new Thread(VideoRenderer);
// PeerConnection.Configure(options => options.IsSingleThreaded = true);
using (var pc = new ObservablePeerConnection(new PeerConnectionOptions
{
Name = "WebRTC Server",
IceServers = { "stun:stun.l.google.com:19302" }
}))
using (pc.LocalIceCandidateStream.Subscribe(ice => ws.SendJsonAsync("ice", ice, cancellation)))
using (pc.LocalSessionDescriptionStream.Subscribe(sd => ws.SendJsonAsync("sdp", sd, cancellation)))
using (var videoTrack = new ObservableVideoTrack(pc,
VideoEncoderOptions.OptimizedFor(VideoFrameWidth, VideoFrameHeight, VideoFrameRate, VideoMotion.Medium, VideoMotion.High)))
{
var msgStream = Observable.Never <DataMessage>();
var iceStream = new Subject <IceCandidate>();
var sdpStream = new Subject <SessionDescription>();
var sharedState = new SharedState(videoTrack, logger);
renderThread.Start(sharedState);
pc.Connect(msgStream, sdpStream, iceStream);
pc.AddDataChannel(new DataChannelOptions());
pc.CreateOffer();
var reader = new WebSocketReader(ws, cancellation);
while (reader.CanRead && !cancellation.IsCancellationRequested)
{
var message = await reader.ReadJsonAsync();
if (message == null)
{
break;
}
var payload = message["payload"];
if (payload.Any())
{
switch (message["action"].Value <string>())
{
case "ice":
{
iceStream.OnNext(ToObject <IceCandidate>(payload));
break;
}
case "sdp":
{
sdpStream.OnNext(ToObject <SessionDescription>(payload));
break;
}
case "pos":
{
sharedState.MouseMessageQueue.Enqueue(ToObject <MouseMessage>(payload));
break;
}
}
}
}
logger.LogInformation(ws.CloseStatus.HasValue ? "Websocket was closed by client" : "Application is stopping...");
renderThread.Interrupt();
renderThread.Join();
}
}
private static unsafe void Render()
{
const int frameWidth = 2560;
const int frameHeight = 1440;
const int frameRate = 60;
// var options = VideoEncoderOptions.OptimizedFor(frameWidth, frameHeight, frameRate);
var options = new VideoEncoderOptions
{
MaxBitsPerSecond = 12_000_000,
MinBitsPerSecond = 10_000_000,
MaxFramesPerSecond = frameRate
};
using (var sender = new ObservablePeerConnection(new PeerConnectionOptions()))
using (var receiver = new ObservablePeerConnection(new PeerConnectionOptions {
CanReceiveVideo = true
}))
{
using (var vt = new ObservableVideoTrack(sender, options))
{
using (var rnd = new BouncingBallRenderer(vt, 10, new BoundingBallOptions
{
VideoFrameWidth = frameWidth,
VideoFrameHeight = frameHeight,
VideoFrameQueueSize = 2
}))
{
receiver.Connect(
Observable.Never <DataMessage>(),
sender.LocalSessionDescriptionStream,
sender.LocalIceCandidateStream);
sender.Connect(
Observable.Never <DataMessage>(),
receiver.LocalSessionDescriptionStream,
receiver.LocalIceCandidateStream);
sender.CreateOffer();
int remoteVideoFrameReceivedCount = 0;
receiver.RemoteVideoFrameReceived += (pc, frame) =>
{
remoteVideoFrameReceivedCount += 1;
// Save as JPEG for debugging. SLOW!
// TODO: Doesn't work yet, H264 decoding not yet supported, only VP8
//if (frame is VideoFrameYuvAlpha yuvFrame && yuvFrame.Width == yuvFrame.StrideY)
//{
// var span = new ReadOnlySpan<byte>(yuvFrame.DataY.ToPointer(), yuvFrame.Width * yuvFrame.Height);
// using (var image = Image.LoadPixelData<Y8>(span, yuvFrame.Width, yuvFrame.Height))
// {
// image.Save($@"frame_{remoteVideoFrameReceivedCount:D000000}.bmp");
// }
//}
};
using (var clock = new PreciseWaitableClock(EventResetMode.AutoReset))
{
var startTime = clock.GetCurrentTime().AddSeconds(1);
var nextTime = startTime;
// The remote peer connection is not immediately ready to receive frames,
// so we keep sending until it succeeds.
// TODO: Figure out what webrtc event can be used for this.
while (!Console.KeyAvailable)
{
clock.SetFutureEventTime(nextTime);
clock.WaitHandle.WaitOne();
var elapsedTime = clock.GetCurrentTime() - startTime;
rnd.SendFrame(elapsedTime);
nextTime = nextTime.AddSeconds(1.0 / frameRate);
}
}
}
// The video renderer is now disposed while the video track is still encoding some textures
// This should not crash.
// We need to wait a while before disposing the video-track and peer-connection to check this.
Thread.Sleep(100);
}
}
}
}
public void RendersAndSendsFrameUsingD3D11()
{
bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
bool hasNvEnc = PeerConnection.SupportsHardwareTextureEncoding;
if (isWindows && hasNvEnc)
{
PeerConnection.Configure(new GlobalOptions
{
UseFakeDecoders = true,
LogToDebugOutput = false,
MinimumLogLevel = TraceLevel.Info
});
using (var sender = new ObservablePeerConnection(new PeerConnectionOptions()))
using (var receiver = new ObservablePeerConnection(new PeerConnectionOptions {
CanReceiveVideo = true
}))
using (var vt = new ObservableVideoTrack(sender, VideoEncoderOptions.OptimizedFor(320, 240, 10)))
{
using (var rnd = new VideoRenderer(vt, new RendererOptions {
VideoFrameQueueSize = 2
}))
{
// Wait until sender and receiver are connected,
// signaling is complete,
// and video track is added.
// TODO: When using tasks for this, this test hangs when disposing!
// ReSharper disable once InvokeAsExtensionMethod
//var ready = Observable.Zip(
// receiver.ConnectionStateStream.FirstAsync(s => s == ConnectionState.Connected),
// sender.ConnectionStateStream.FirstAsync(s => s == ConnectionState.Connected),
// receiver.SignalingStateStream.FirstAsync(s => s == SignalingState.Stable),
// sender.SignalingStateStream.FirstAsync(s => s == SignalingState.Stable),
// receiver.RemoteTrackChangeStream.FirstAsync(
// c => !string.IsNullOrEmpty(c.TransceiverMid) &
// c.MediaKind == TrackMediaKind.Video &&
// c.ChangeKind == TrackChangeKind.Changed),
// (a, b, c, d, e) => true);
//// Wait until connected and video track is ready.
//var ev = new AutoResetEvent(false);
//ready.Subscribe(_ => ev.Set());
receiver.Connect(
Observable.Never <DataMessage>(),
sender.LocalSessionDescriptionStream,
sender.LocalIceCandidateStream);
sender.Connect(
Observable.Never <DataMessage>(),
receiver.LocalSessionDescriptionStream,
receiver.LocalIceCandidateStream);
sender.CreateOffer();
int remoteVideoFrameReceivedCount = 0;
receiver.RemoteVideoFrameReceived += (pc, frame) =>
{
remoteVideoFrameReceivedCount += 1;
};
// The remote peer connection is not immediately ready to receive frames,
// so we keep sending until it succeeds.
// TODO: Figure out what webrtc event can be used for this.
while (remoteVideoFrameReceivedCount == 0)
{
using (rnd.TakeNextFrameForSending())
{
}
}
// Continue sending until the video queue is empty
while (rnd.VideoFrameQueueCount > 0)
{
using (rnd.TakeNextFrameForSending())
{
}
}
}
// The video renderer is now disposed while the video track is still encoding some textures
// This should not crash.
// We need to wait a while before disposing the video-track and peer-connection to check this.
Thread.Sleep(100);
}
}
}
