public TransportThreadContext(IoUringOptions options, MemoryPool <byte> memoryPool, int eventFd, ConcurrentQueue <ulong> asyncOperationQueue)
{
Options = options;
MemoryPool = memoryPool;
_eventFd = eventFd;
_asyncOperationQueue = asyncOperationQueue;
}
public TransportThread(IoUringOptions options)
{
int res = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (res == -1)
{
throw new ErrnoException(errno);
}
_eventfd = res;
// Pin buffer for eventfd reads via io_uring
byte[] bytes = new byte[8];
_eventfdBytes = GCHandle.Alloc(bytes, GCHandleType.Pinned);
// Pin iovec used for eventfd reads via io_uring
var eventfdIoVec = new iovec
{
iov_base = (void *)_eventfdBytes.AddrOfPinnedObject(),
iov_len = bytes.Length
};
_eventfdIoVecHandle = GCHandle.Alloc(eventfdIoVec, GCHandleType.Pinned);
_eventfdIoVec = (iovec *)_eventfdIoVecHandle.AddrOfPinnedObject();
var memoryPool = new SlabMemoryPool();
_threadContext = new TransportThreadContext(options, memoryPool, _eventfd, s => Read(s), s => Write(s));
_maxBufferSize = memoryPool.MaxBufferSize;
}
protected IoUringThread(string name, IoUringOptions options, int cpuId)
{
_options = options;
_ring = new Ring(options.RingSize);
_unblockHandle = new RingUnblockHandle(_ring);
_cpuId = cpuId;
int id;
lock (_threadIds)
{
if (!_threadIds.TryGetValue(name, out id))
{
_threadIds[name] = id = -1;
}
_threadIds[name] = id += 1;
}
_thread = new Thread(obj => ((IoUringThread)obj).Loop())
{
IsBackground = true,
Name = $"{name} - {id}"
};
}
public IoUringTransport(IOptions <IoUringOptions> options)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
Limits.SetToMax(Resource.RLIMIT_NOFILE);
List <int> cpus = null;
if (_options.SetThreadAffinity)
{
cpus = CpuInfo.GetPreferredCpuIds(_options.ThreadCount);
}
var threads = new TransportThread[_options.ThreadCount];
int cpuIdx = 0;
for (int i = 0; i < threads.Length; i++)
{
var cpuId = cpus == null ? TransportThread.NoCpuAffinity : cpus[cpuIdx++ % cpus.Count];
var thread = new TransportThread(_options, cpuId);
thread.Run();
threads[i] = thread;
}
TransportThreads = threads;
}
public TransportThreadContext(IoUringOptions options, MemoryPool <byte> memoryPool, int eventFd, Action <int> readAction, Action <int> writeAction)
{
Options = options;
MemoryPool = memoryPool;
_eventFd = eventFd;
_readAction = readAction;
_writeAction = writeAction;
}
public TransportThreadContext(IoUringOptions options, ConcurrentQueue <IoUringConnectionContext> readPollQueue,
ConcurrentQueue <IoUringConnectionContext> writePollQueue, MemoryPool <byte> memoryPool, int eventFd)
{
ReadPollQueue = readPollQueue;
WritePollQueue = writePollQueue;
_eventFd = eventFd;
Options = options;
MemoryPool = memoryPool;
}
public SocketReceiver(LinuxSocket recipient, ChannelWriter<ConnectionContext> acceptQueue, EndPoint endPoint, MemoryPool<byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
_recipient = recipient;
AcceptQueue = acceptQueue;
_endPoint = endPoint;
_memoryPool = memoryPool;
_options = options;
_scheduler = scheduler;
}
private ConnectionListener(EndPoint endpoint, IoUringTransport transport, IoUringOptions options)
{
_transport = transport;
_options = options;
_acceptQueue = Channel.CreateUnbounded <ConnectionContext>(new UnboundedChannelOptions
{
SingleReader = true, // reads happen sequentially
SingleWriter = false,
AllowSynchronousContinuations = options.ApplicationSchedulingMode == PipeScheduler.Inline
});
EndPoint = endpoint;
}
private AcceptSocket(LinuxSocket socket, EndPoint endPoint, ChannelWriter <ConnectionContext> acceptQueue, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
Socket = socket;
EndPoint = endPoint;
AcceptQueue = acceptQueue;
_memoryPool = memoryPool;
_options = options;
_scheduler = scheduler;
_unbindCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
if (endPoint is IPEndPoint) // _addr is null by intention for the remaining EndPoint types
{
_addr = GC.AllocateUninitializedArray <byte>(SizeOf.sockaddr_storage, pinned: true);
_addrLen = GC.AllocateUninitializedArray <byte>(SizeOf.socklen_t, pinned: true);
}
}
public static AcceptSocket Bind(UnixDomainSocketEndPoint unixDomainSocketEndPoint, ChannelWriter <ConnectionContext> acceptQueue, IoUringOptions options)
{
var socketPath = unixDomainSocketEndPoint.ToString();
var s = new LinuxSocket(AF_UNIX, SOCK_STREAM, 0, blocking: false);
File.Delete(socketPath);
s.Bind(unixDomainSocketEndPoint);
s.Listen(options.ListenBacklog);
return(new AcceptSocket(s, unixDomainSocketEndPoint, acceptQueue, null, options, null));
}
public static AcceptSocket Bind(IPEndPoint ipEndPoint, ChannelWriter <ConnectionContext> acceptQueue, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
var domain = ipEndPoint.AddressFamily == AddressFamily.InterNetwork ? AF_INET : AF_INET6;
LinuxSocket s = new LinuxSocket(domain, SOCK_STREAM, IPPROTO_TCP, blocking: false);
s.SetOption(SOL_SOCKET, SO_REUSEADDR, 1);
s.SetOption(SOL_SOCKET, SO_REUSEPORT, 1);
s.Bind(ipEndPoint);
s.Listen(options.ListenBacklog);
return(new AcceptSocket(s, s.GetLocalAddress(), acceptQueue, memoryPool, options, scheduler));
}
public AcceptThread(IoUringOptions options, TransportThread[] transportThreads)
: base("IoUring Accept Thread", options, NoCpuAffinity)
{
_scheduler = new AcceptThreadScheduler(_unblockHandle, _asyncOperationQueue);
_transportThreads = transportThreads;
}
private OutboundConnection(LinuxSocket socket, EndPoint remote, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler, TaskCompletionSource <ConnectionContext> connectCompletion)
: base(socket, null, remote, memoryPool, options, scheduler)
{
unsafe
{
switch (remote)
{
case IPEndPoint ipEndPoint:
{
ipEndPoint.ToSockAddr((sockaddr_storage *)Addr, out var addrLength);
AddrLen = addrLength;
break;
}
case UnixDomainSocketEndPoint unixDomainSocketEndPoint:
{
unixDomainSocketEndPoint.ToSockAddr((sockaddr_un *)Addr);
AddrLen = SizeOf.sockaddr_un;
break;
}
}
}
_connectCompletion = connectCompletion;
// Add IConnectionInherentKeepAliveFeature to the tcp connection impl since Kestrel doesn't implement
// the IConnectionHeartbeatFeature
Features.Set <IConnectionInherentKeepAliveFeature>(this);
}
public static OutboundConnection Create(EndPoint endpoint, TaskCompletionSource <ConnectionContext> connectCompletion, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
LinuxSocket s = default;
switch (endpoint)
{
case IPEndPoint _:
var domain = endpoint.AddressFamily == AddressFamily.InterNetwork ? AF_INET : AF_INET6;
s = new LinuxSocket(domain, SOCK_STREAM, IPPROTO_TCP, blocking: false);
if (options.TcpNoDelay)
{
s.SetOption(SOL_TCP, TCP_NODELAY, 1);
}
break;
case UnixDomainSocketEndPoint _:
s = new LinuxSocket(AF_UNIX, SOCK_STREAM, 0, blocking: false);
break;
case FileHandleEndPoint fileHandleEndPoint:
s = (int)fileHandleEndPoint.FileHandle;
break;
default:
ThrowHelper.ThrowNewNotSupportedException_EndPointNotSupported();
break;
}
return(new OutboundConnection(s, endpoint, memoryPool, options, scheduler, connectCompletion));
}
public ConnectionListenerFactory(IoUringTransport ioUringTransport, IOptions <IoUringOptions> options)
{
_ioUringTransport = ioUringTransport;
_options = options.Value;
}
public TransportThread(IoUringOptions options)
{
_ring = new Ring(RingSize);
SetupEventFd();
_threadContext = new TransportThreadContext(options, _readPollQueue, _writePollQueue, _memoryPool, _eventfd);
}
private ConnectionListener(EndPoint endpoint, IoUringTransport transport, IoUringOptions options)
{
EndPoint = endpoint;
_transport = transport;
_options = options;
}
public static AcceptSocket Bind(FileHandleEndPoint fileHandleEndPoint, ChannelWriter <ConnectionContext> acceptQueue, IoUringOptions options)
{
LinuxSocket s = (int)fileHandleEndPoint.FileHandle;
var endPoint = s.GetLocalAddress();
return(new AcceptSocket(s, endPoint ?? fileHandleEndPoint, acceptQueue, null, options, null));
}
public InboundConnection(LinuxSocket socket, EndPoint local, EndPoint remote, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
: base(socket, local, remote, memoryPool, options, scheduler)
{
}
public static ValueTask <IConnectionListener> Create(EndPoint endpoint, IoUringTransport transport, IoUringOptions options)
{
var listener = new ConnectionListener(endpoint, transport, options);
listener.Bind();
return(new ValueTask <IConnectionListener>(listener));
}
public static async ValueTask <IConnectionListener> BindAsync(EndPoint endpoint, IoUringTransport transport, IoUringOptions options)
{
var listener = new ConnectionListener(endpoint, transport, options);
await listener.BindAsync();
return(listener);
}
protected IoUringConnection(LinuxSocket socket, EndPoint local, EndPoint remote, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
Socket = socket;
LocalEndPoint = local;
RemoteEndPoint = remote;
MemoryPool = memoryPool;
Debug.Assert(MaxBufferSize == MemoryPool.MaxBufferSize);
_scheduler = scheduler;
_connectionClosedTokenSource = new CancellationTokenSource();
ConnectionClosed = _connectionClosedTokenSource.Token;
_waitForConnectionClosedTcs = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var appScheduler = options.ApplicationSchedulingMode;
var inputOptions = new PipeOptions(memoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(memoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = () => HandleFlushedToApp();
_onReadFromApp = () => HandleReadFromApp();
_ioVecBytes = GC.AllocateUninitializedArray <byte>(SizeOf.iovec * (ReadIOVecCount + WriteIOVecCount), pinned: true);
unsafe
{
_iovec = (iovec *)MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_ioVecBytes);
}
}
public TransportThread(IoUringOptions options, int cpuId)
: base("IoUring Transport Thread", options, cpuId)
{
_memoryPool = new SlabMemoryPool();
_scheduler = new TransportThreadScheduler(_unblockHandle, _asyncOperationQueue, _asyncOperationStates);
}