private unsafe void AioSend(List <ScheduledSend> sendQueue) { while (sendQueue.Count > 0) { int sendCount = 0; int completedCount = 0; AioCb * aioCbs = AioCbs; IOVector *ioVectors = IoVectorTable; ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers; Span <MemoryHandle> memoryHandles = MemoryHandles; int memoryHandleCount = 0; for (int i = 0; i < sendQueue.Count; i++) { TSocket socket = sendQueue[i].Socket; ReadOnlySequence <byte> buffer; Exception error = socket.GetReadResult(out buffer); if (error != null) { socket.CompleteOutput(error == TransportConstants.StopSentinel ? null : error); completedCount++; } else { int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket); socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles); memoryHandles = memoryHandles.Slice(ioVectorLength); memoryHandleCount += ioVectorLength; aioCbs->Fd = socket.Fd; aioCbs->Data = i; aioCbs->OpCode = AioOpCode.PWritev; aioCbs->Buffer = ioVectors; aioCbs->Length = ioVectorLength; aioCbs++; sendBuffers[sendCount] = buffer; sendCount++; if (sendCount == EventBufferLength) { break; } ioVectors += ioVectorLength; } } if (sendCount > 0) { IntPtr ctxp = _aioContext; PosixResult res = AioInterop.IoSubmit(ctxp, sendCount, AioCbsTable); memoryHandles = MemoryHandles; for (int i = 0; i < memoryHandleCount; i++) { memoryHandles[i].Dispose(); } if (res != sendCount) { throw new NotSupportedException("Unexpected IoSubmit Send retval " + res); } AioEvent *aioEvents = AioEvents; res = AioInterop.IoGetEvents(ctxp, sendCount, aioEvents); if (res != sendCount) { throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res); } AioEvent *aioEvent = aioEvents; for (int i = 0; i < sendCount; i++) { PosixResult result = aioEvent->Result; int socketIndex = (int)aioEvent->Data; TSocket socket = sendQueue[socketIndex].Socket; ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events sendBuffers[i] = default; socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false); aioEvent++; } } sendQueue.RemoveRange(0, sendCount + completedCount); } }
private unsafe void AioReceive(List <TSocket> readableSockets) { long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength); (int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff)); int readableSocketCount = readableSockets.Count; AioCb * aioCb = AioCbs; IOVector * ioVectors = IoVectorTable; PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount]; bool checkAvailable = _transportOptions.CheckAvailable; Span <MemoryHandle> receiveMemoryHandles = MemoryHandles; int receiveMemoryHandleCount = 0; for (int i = 0; i < readableSocketCount; i++) { TSocket socket = readableSockets[i]; int availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes(); int ioVectorLength = socket.CalcIOVectorLengthForReceive(availableBytes, IoVectorsPerAioSocket); int advanced = socket.FillReceiveIOVector(availableBytes, ioVectors, receiveMemoryHandles, ref ioVectorLength); aioCb->Fd = socket.Fd; aioCb->Data = PackReceiveState(0, advanced, ioVectorLength); aioCb->OpCode = AioOpCode.PReadv; aioCb->Buffer = ioVectors; aioCb->Length = ioVectorLength; aioCb++; ioVectors += ioVectorLength; receiveMemoryHandleCount += ioVectorLength; receiveMemoryHandles = receiveMemoryHandles.Slice(ioVectorLength); } int eAgainCount = 0; while (readableSocketCount > 0) { IntPtr ctxp = _aioContext; PosixResult res = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoSubmit retval " + res); } AioEvent *aioEvents = AioEvents; res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoGetEvents retval " + res); } int socketsRemaining = readableSocketCount; bool allEAgain = true; AioEvent *aioEvent = aioEvents; for (int i = 0; i < readableSocketCount; i++) { PosixResult result = aioEvent->Result; int socketIndex = i; // assumes in-order events TSocket socket = readableSockets[socketIndex]; (int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data); (bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength); if (done) { receiveResults[socketIndex] = retval; socketsRemaining--; aioEvent->AioCb->OpCode = AioOpCode.Noop; allEAgain = false; } else if (retval != PosixResult.EAGAIN) { aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength); allEAgain = false; } aioEvent++; } if (socketsRemaining > 0) { if (allEAgain) { eAgainCount++; if (eAgainCount == TransportConstants.MaxEAgainCount) { throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes."); } } else { aioCb = AioCbs; AioCb *aioCbWriteAt = aioCb; // The kernel doesn't handle Noop, we need to remove them from the aioCbs for (int i = 0; i < readableSocketCount; i++) { if (aioCb[i].OpCode != AioOpCode.Noop) { if (aioCbWriteAt != aioCb) { *aioCbWriteAt = *aioCb; } aioCbWriteAt++; } aioCb++; } readableSocketCount = socketsRemaining; eAgainCount = 0; } } else { readableSocketCount = 0; } } for (int i = 0; i < readableSockets.Count; i++) { readableSockets[i].OnReceiveFromSocket(receiveResults[i]); } readableSockets.Clear(); receiveMemoryHandles = MemoryHandles; for (int i = 0; i < receiveMemoryHandleCount; i++) { receiveMemoryHandles[i].Dispose(); } }