private static unsafe AioEvent *Copy(AioEvent *start, AioEvent *end, AioEvent *dst)
{
uint byteCount = (uint)((byte *)end - (byte *)start);
Unsafe.CopyBlock(dst, start, byteCount);
return((AioEvent *)((byte *)dst + byteCount));
}
public unsafe static PosixResult IoGetEvents(IntPtr ctxp, int nr, AioEvent *events)
{
if (nr <= 0)
{
return(new PosixResult(PosixResult.EINVAL));
}
AioRing *pRing = (AioRing *)ctxp;
if (pRing->Magic == 0xa10a10a1 && pRing->IncompatFeatures == 0)
{
int head = pRing->Head;
int tail = pRing->Tail;
int available = tail - head;
if (available < 0)
{
available += pRing->Nr;
}
if (available >= nr)
{
AioEvent *ringEvents = (AioEvent *)((byte *)pRing + pRing->HeaderLength);
AioEvent *start = ringEvents + head;
AioEvent *end = start + nr;
if (head + nr > pRing->Nr)
{
end -= pRing->Nr;
}
if (end > start)
{
Copy(start, end, events);
}
else
{
AioEvent *eventsEnd = Copy(start, ringEvents + pRing->Nr, events);
Copy(ringEvents, end, eventsEnd);
}
head += nr;
if (head >= pRing->Nr)
{
head -= pRing->Nr;
}
pRing->Head = head;
return(new PosixResult(nr));
}
}
return(IoGetEvents(ctxp, nr, nr, events, -1));
}
private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
AioCb * aioCbs = AioCbs;
IOVector *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
if (error == TransportConstants.StopSentinel)
{
error = null;
}
socket.CompleteOutput(error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->Fd = socket.Fd;
aioCbs->Data = i;
aioCbs->OpCode = AioOpCode.PWritev;
aioCbs->Buffer = ioVectors;
aioCbs->Length = ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = (int)aioEvent->Data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
AioCb * aioCb = AioCbs;
IOVector * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);
aioCb->Fd = socket.Fd;
aioCb->Data = PackReceiveState(0, advanced, memoryAllocation.IovLength);
aioCb->OpCode = AioOpCode.PReadv;
aioCb->Buffer = ioVectors;
aioCb->Length = memoryAllocation.IovLength;
aioCb++;
ioVectors += memoryAllocation.IovLength;
receiveMemoryHandleCount += memoryAllocation.IovLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
aioEvent->AioCb->OpCode = AioOpCode.Noop;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
AioCb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].OpCode != AioOpCode.Noop)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
}
public unsafe static extern PosixResult IoGetEvents(IntPtr ctxp, int minNr, int maxNr, AioEvent *events, int timeoutMs);
