public void Dispose()
{
_epoll?.Dispose();
_pipeEnds.Dispose();
MemoryPool?.Dispose();
if (_aioEventsMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioEventsMemory);
}
if (_aioCbsMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioCbsMemory);
}
if (_aioCbsTableMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioCbsTableMemory);
}
if (_ioVectorTableMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_ioVectorTableMemory);
}
if (_aioContext != IntPtr.Zero)
{
AioInterop.IoDestroy(_aioContext);
}
}
public unsafe void Dispose()
{
AbortQueuedConnectionAsync().GetAwaiter().GetResult();
_epoll?.Dispose();
_pipeEnds.Dispose();
MemoryPool?.Dispose();
if (_aioEventsMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioEventsMemory);
}
if (_aioCbsMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioCbsMemory);
}
if (_aioCbsTableMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_aioCbsTableMemory);
}
if (_ioVectorTableMemory != IntPtr.Zero)
{
Marshal.FreeHGlobal(_ioVectorTableMemory);
}
if (_aioContext.ring != null)
{
AioInterop.IoDestroy(_aioContext);
}
}
public unsafe ThreadContext(TransportThread transportThread)
{
_transportThread = transportThread;
_connectionDispatcher = transportThread.ConnectionDispatcher;
_sockets = new Dictionary <int, TSocket>();
_logger = _transportThread.LoggerFactory.CreateLogger($"{nameof(RedHat)}.{nameof(TransportThread)}.{_transportThread.ThreadId}");
_acceptSockets = new List <TSocket>();
_transportOptions = transportThread.TransportOptions;
_scheduledSendAdding = new List <ScheduledSend>(1024);
_scheduledSendRunning = new List <ScheduledSend>(1024);
_epollState = EPollBlocked;
if (_transportOptions.AioReceive | _transportOptions.AioSend)
{
_aioEventsMemory = AllocMemory(sizeof(io_event) * EventBufferLength);
_aioCbsMemory = AllocMemory(sizeof(iocb) * EventBufferLength);
_aioCbsTableMemory = AllocMemory(IntPtr.Size * EventBufferLength);
_ioVectorTableMemory = AllocMemory(SizeOf.iovec * IoVectorsPerAioSocket * EventBufferLength);
for (int i = 0; i < EventBufferLength; i++)
{
AioCbsTable[i] = &AioCbs[i];
}
if (_transportOptions.AioSend)
{
_aioSendBuffers = new ReadOnlySequence <byte> [EventBufferLength];
}
}
int maxMemoryHandleCount = TSocket.MaxIOVectorReceiveLength;
if (_transportOptions.AioReceive || _transportOptions.AioSend)
{
maxMemoryHandleCount = Math.Max(maxMemoryHandleCount, EventBufferLength);
}
if (_transportOptions.DeferSend)
{
maxMemoryHandleCount = Math.Max(maxMemoryHandleCount, TSocket.MaxIOVectorSendLength);
}
MemoryHandles = new MemoryHandle[maxMemoryHandleCount];
// These members need to be Disposed
_epoll = EPoll.Create();
_epollFd = _epoll.DangerousGetHandle().ToInt32();
MemoryPool = CreateMemoryPool();
_pipeEnds = PipeEnd.CreatePair(blocking: false);
if (_aioEventsMemory != IntPtr.Zero)
{
aio_context_t ctx;
AioInterop.IoSetup(EventBufferLength, &ctx).ThrowOnError();
_aioContext = ctx;
}
}
private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
AioCb * aioCbs = AioCbs;
IOVector *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
if (error == TransportConstants.StopSentinel)
{
error = null;
}
socket.CompleteOutput(error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->Fd = socket.Fd;
aioCbs->Data = i;
aioCbs->OpCode = AioOpCode.PWritev;
aioCbs->Buffer = ioVectors;
aioCbs->Length = ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = (int)aioEvent->Data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
AioCb * aioCb = AioCbs;
IOVector * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);
aioCb->Fd = socket.Fd;
aioCb->Data = PackReceiveState(0, advanced, memoryAllocation.IovLength);
aioCb->OpCode = AioOpCode.PReadv;
aioCb->Buffer = ioVectors;
aioCb->Length = memoryAllocation.IovLength;
aioCb++;
ioVectors += memoryAllocation.IovLength;
receiveMemoryHandleCount += memoryAllocation.IovLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
aioEvent->AioCb->OpCode = AioOpCode.Noop;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
AioCb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].OpCode != AioOpCode.Noop)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
}
public unsafe ThreadContext(TransportThread transportThread)
{
_transportThread = transportThread;
_sockets = new Dictionary <int, TSocket>();
_logger = _transportThread.LoggerFactory.CreateLogger($"{nameof(RedHat)}.{nameof(TransportThread)}.{_transportThread.ThreadId}");
_acceptSockets = new List <TSocket>();
_transportOptions = transportThread.TransportOptions;
_scheduledSendAdding = new List <ScheduledSend>(1024);
_scheduledSendRunning = new List <ScheduledSend>(1024);
_epollState = EPollBlocked;
if (_transportOptions.AioReceive | _transportOptions.AioSend)
{
_aioEventsMemory = AllocMemory(sizeof(io_event) * EventBufferLength);
_aioCbsMemory = AllocMemory(sizeof(iocb) * EventBufferLength);
_aioCbsTableMemory = AllocMemory(IntPtr.Size * EventBufferLength);
_ioVectorTableMemory = AllocMemory(SizeOf.iovec * IoVectorsPerAioSocket * EventBufferLength);
for (int i = 0; i < EventBufferLength; i++)
{
AioCbsTable[i] = &AioCbs[i];
}
if (_transportOptions.AioSend)
{
_aioSendBuffers = new ReadOnlySequence <byte> [EventBufferLength];
}
}
int maxMemoryHandleCount = TSocket.MaxIOVectorReceiveLength;
if (_transportOptions.AioReceive || _transportOptions.AioSend)
{
maxMemoryHandleCount = Math.Max(maxMemoryHandleCount, EventBufferLength);
}
if (_transportOptions.DeferSend)
{
maxMemoryHandleCount = Math.Max(maxMemoryHandleCount, TSocket.MaxIOVectorSendLength);
}
MemoryHandles = new MemoryHandle[maxMemoryHandleCount];
// These members need to be Disposed
_epoll = EPoll.Create();
_epollFd = _epoll.DangerousGetHandle().ToInt32();
MemoryPool = CreateMemoryPool();
_pipeEnds = PipeEnd.CreatePair(blocking: false);
if (_aioEventsMemory != IntPtr.Zero)
{
aio_context_t ctx;
AioInterop.IoSetup(EventBufferLength, &ctx).ThrowOnError();
_aioContext = ctx;
}
// Single reader, single writer queue since all writes happen from the TransportThread and reads happen sequentially
// This channel is unbounded which means there's nothing limiting the number of sockets we're accepting.
// This is similar to having an unbounded number of thread pool work items queued to invoke a ConnectionHandler
// which was the previous pattern, but now it's more explicit.
// TODO: Find a reasonable limit and start applying accept backpressure once the channel reaches that limit.
_acceptQueue = Channel.CreateUnbounded <TSocket>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = true,
AllowSynchronousContinuations = _transportOptions.ApplicationSchedulingMode == PipeScheduler.Inline,
});
}