private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
iocb * aioCbs = AioCbs;
iovec *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
if (error == TransportConstants.StopSentinel)
{
error = null;
}
socket.CompleteOutput(error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->aio_fildes = socket.Fd;
aioCbs->aio_data = (ulong)i;
aioCbs->aio_lio_opcode = IOCB_CMD_PWRITEV;
aioCbs->aio_buf = (ulong)ioVectors;
aioCbs->aio_nbytes = (ulong)ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
PosixResult res = AioInterop.IoSubmit(_aioContext, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
io_event *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(_aioContext, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
io_event *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = new PosixResult((int)aioEvent->res);
int socketIndex = (int)aioEvent->data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
public unsafe bool ExecuteOperations()
{
List <Operation>?scheduled = _scheduledOperations;
if (scheduled == null)
{
return(false);
}
_scheduledOperations = _cachedOperationsList;
_cachedOperationsList = null;
if (_results == null || _results.Length < scheduled.Count)
{
_results = new AsyncOperationResult[Math.Max(scheduled.Count, 2 * (_results?.Length ?? 0))];
}
AsyncOperationResult[] results = _results;
int queueLength = scheduled.Count;
int queueOffset = 0;
while (queueOffset < queueLength)
{
int nr = Math.Min((queueLength - queueOffset), IocbLength);
try
{
iocb *aioCbs = AioCbs;
for (int i = queueOffset; i < (queueOffset + nr); i++)
{
Operation op = scheduled[i];
int fd = op.Handle.DangerousGetHandle().ToInt32(); // TODO: make safe
MemoryHandle handle = op.Memory.Pin();
_memoryHandles.Add(handle);
aioCbs->aio_fildes = fd;
aioCbs->aio_data = (ulong)i;
aioCbs->aio_lio_opcode = op.IsReadNotWrite ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE;
aioCbs->aio_buf = (ulong)handle.Pointer;
aioCbs->aio_nbytes = (ulong)op.Memory.Length;
aioCbs++;
}
iocb **iocbpp = AioCbsTable;
int toSubmit = nr;
while (toSubmit > 0)
{
int rv = io_submit(_ctx, toSubmit, iocbpp);
if (rv == -1)
{
PlatformException.Throw();
}
int toReceive = rv;
toSubmit -= rv;
iocbpp += rv;
while (toReceive > 0)
{
do
{
rv = IoGetEvents(_ctx, toReceive, AioEvents);
} while (rv == -1 && errno == EINTR);
if (rv == -1)
{
PlatformException.Throw();
}
io_event *events = AioEvents;
for (int i = 0; i < rv; i++)
{
results[(int)events->data] = new AsyncOperationResult(events->res);
events++;
}
toReceive -= rv;
}
}
}
finally
{
foreach (var handle in _memoryHandles)
{
handle.Dispose();
}
_memoryHandles.Clear();
}
// Callbacks
for (int i = queueOffset; i < (queueOffset + nr); i++)
{
Operation op = scheduled[i];
op.Callback(results[i], op.State, op.Data);
}
queueOffset += nr;
}
scheduled.Clear();
if (_scheduledOperations == null)
{
_scheduledOperations = scheduled;
return(false);
}
else
{
_cachedOperationsList = scheduled;
return(_scheduledOperations.Count > 0);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
ulong PackReceiveState(int received, int advanced, int iovLength) => ((ulong)received << 32) + (ulong)(advanced << 8) + (ulong)(iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(ulong data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
iocb * aioCb = AioCbs;
iovec * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);
aioCb->aio_fildes = socket.Fd;
aioCb->aio_data = PackReceiveState(0, advanced, memoryAllocation.IovLength);
aioCb->aio_lio_opcode = IOCB_CMD_PREADV;
aioCb->aio_buf = (ulong)ioVectors;
aioCb->aio_nbytes = (ulong)memoryAllocation.IovLength;
aioCb++;
ioVectors += memoryAllocation.IovLength;
receiveMemoryHandleCount += memoryAllocation.IovLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
PosixResult res = AioInterop.IoSubmit(_aioContext, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
io_event *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(_aioContext, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
io_event *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = new PosixResult((int)aioEvent->res);
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (iovec *)((iocb *)aioEvent->obj)->aio_buf, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
((iocb *)aioEvent->obj)->aio_lio_opcode = IOCB_CMD_NOOP;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
((iocb *)aioEvent->obj)->aio_data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
iocb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].aio_lio_opcode != IOCB_CMD_NOOP)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
}