private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
AioCb * aioCbs = AioCbs;
IOVector *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
socket.CompleteOutput(error == TransportConstants.StopSentinel ? null : error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->Fd = socket.Fd;
aioCbs->Data = i;
aioCbs->OpCode = AioOpCode.PWritev;
aioCbs->Buffer = ioVectors;
aioCbs->Length = ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = (int)aioEvent->Data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
AioCb * aioCb = AioCbs;
IOVector * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
bool checkAvailable = _transportOptions.CheckAvailable;
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
int availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes();
int ioVectorLength = socket.CalcIOVectorLengthForReceive(availableBytes, IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(availableBytes, ioVectors, receiveMemoryHandles, ref ioVectorLength);
aioCb->Fd = socket.Fd;
aioCb->Data = PackReceiveState(0, advanced, ioVectorLength);
aioCb->OpCode = AioOpCode.PReadv;
aioCb->Buffer = ioVectors;
aioCb->Length = ioVectorLength;
aioCb++;
ioVectors += ioVectorLength;
receiveMemoryHandleCount += ioVectorLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(ioVectorLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
aioEvent->AioCb->OpCode = AioOpCode.Noop;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
AioCb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].OpCode != AioOpCode.Noop)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
}