private unsafe void AioSend(List <ScheduledSend> sendQueue)
            {
                while (sendQueue.Count > 0)
                {
                    int    sendCount      = 0;
                    int    completedCount = 0;
                    iocb * aioCbs         = AioCbs;
                    iovec *ioVectors      = IoVectorTable;
                    ReadOnlySequence <byte>[] sendBuffers   = _aioSendBuffers;
                    Span <MemoryHandle>       memoryHandles = MemoryHandles;
                    int memoryHandleCount = 0;
                    for (int i = 0; i < sendQueue.Count; i++)
                    {
                        TSocket socket = sendQueue[i].Socket;
                        ReadOnlySequence <byte> buffer;
                        Exception error = socket.GetReadResult(out buffer);
                        if (error != null)
                        {
                            if (error == TransportConstants.StopSentinel)
                            {
                                error = null;
                            }
                            socket.CompleteOutput(error);
                            completedCount++;
                        }
                        else
                        {
                            int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
                            socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
                            memoryHandles      = memoryHandles.Slice(ioVectorLength);
                            memoryHandleCount += ioVectorLength;

                            aioCbs->aio_fildes     = socket.Fd;
                            aioCbs->aio_data       = (ulong)i;
                            aioCbs->aio_lio_opcode = IOCB_CMD_PWRITEV;
                            aioCbs->aio_buf        = (ulong)ioVectors;
                            aioCbs->aio_nbytes     = (ulong)ioVectorLength;
                            aioCbs++;

                            sendBuffers[sendCount] = buffer;
                            sendCount++;
                            if (sendCount == EventBufferLength)
                            {
                                break;
                            }

                            ioVectors += ioVectorLength;
                        }
                    }
                    if (sendCount > 0)
                    {
                        PosixResult res = AioInterop.IoSubmit(_aioContext, sendCount, AioCbsTable);

                        memoryHandles = MemoryHandles;
                        for (int i = 0; i < memoryHandleCount; i++)
                        {
                            memoryHandles[i].Dispose();
                        }

                        if (res != sendCount)
                        {
                            throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
                        }

                        io_event *aioEvents = AioEvents;
                        res = AioInterop.IoGetEvents(_aioContext, sendCount, aioEvents);
                        if (res != sendCount)
                        {
                            throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
                        }

                        io_event *aioEvent = aioEvents;
                        for (int i = 0; i < sendCount; i++)
                        {
                            PosixResult             result      = new PosixResult((int)aioEvent->res);
                            int                     socketIndex = (int)aioEvent->data;
                            TSocket                 socket      = sendQueue[socketIndex].Socket;
                            ReadOnlySequence <byte> buffer      = sendBuffers[i]; // assumes in-order events
                            sendBuffers[i] = default;
                            socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
                            aioEvent++;
                        }
                    }

                    sendQueue.RemoveRange(0, sendCount + completedCount);
                }
            }
Beispiel #2
0
            public unsafe bool ExecuteOperations()
            {
                List <Operation>?scheduled = _scheduledOperations;

                if (scheduled == null)
                {
                    return(false);
                }
                _scheduledOperations  = _cachedOperationsList;
                _cachedOperationsList = null;

                if (_results == null || _results.Length < scheduled.Count)
                {
                    _results = new AsyncOperationResult[Math.Max(scheduled.Count, 2 * (_results?.Length ?? 0))];
                }
                AsyncOperationResult[] results = _results;

                int queueLength = scheduled.Count;
                int queueOffset = 0;

                while (queueOffset < queueLength)
                {
                    int nr = Math.Min((queueLength - queueOffset), IocbLength);
                    try
                    {
                        iocb *aioCbs = AioCbs;
                        for (int i = queueOffset; i < (queueOffset + nr); i++)
                        {
                            Operation op = scheduled[i];
                            int       fd = op.Handle.DangerousGetHandle().ToInt32(); // TODO: make safe

                            MemoryHandle handle = op.Memory.Pin();
                            _memoryHandles.Add(handle);

                            aioCbs->aio_fildes     = fd;
                            aioCbs->aio_data       = (ulong)i;
                            aioCbs->aio_lio_opcode = op.IsReadNotWrite ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE;
                            aioCbs->aio_buf        = (ulong)handle.Pointer;
                            aioCbs->aio_nbytes     = (ulong)op.Memory.Length;
                            aioCbs++;
                        }
                        iocb **iocbpp   = AioCbsTable;
                        int    toSubmit = nr;
                        while (toSubmit > 0)
                        {
                            int rv = io_submit(_ctx, toSubmit, iocbpp);
                            if (rv == -1)
                            {
                                PlatformException.Throw();
                            }
                            int toReceive = rv;
                            toSubmit -= rv;
                            iocbpp   += rv;
                            while (toReceive > 0)
                            {
                                do
                                {
                                    rv = IoGetEvents(_ctx, toReceive, AioEvents);
                                } while (rv == -1 && errno == EINTR);
                                if (rv == -1)
                                {
                                    PlatformException.Throw();
                                }
                                io_event *events = AioEvents;
                                for (int i = 0; i < rv; i++)
                                {
                                    results[(int)events->data] = new AsyncOperationResult(events->res);
                                    events++;
                                }
                                toReceive -= rv;
                            }
                        }
                    }
                    finally
                    {
                        foreach (var handle in _memoryHandles)
                        {
                            handle.Dispose();
                        }
                        _memoryHandles.Clear();
                    }

                    // Callbacks
                    for (int i = queueOffset; i < (queueOffset + nr); i++)
                    {
                        Operation op = scheduled[i];
                        op.Callback(results[i], op.State, op.Data);
                    }

                    queueOffset += nr;
                }

                scheduled.Clear();
                if (_scheduledOperations == null)
                {
                    _scheduledOperations = scheduled;
                    return(false);
                }
                else
                {
                    _cachedOperationsList = scheduled;
                    return(_scheduledOperations.Count > 0);
                }
            }
            private unsafe void AioReceive(List <TSocket> readableSockets)
            {
                ulong PackReceiveState(int received, int advanced, int iovLength) => ((ulong)received << 32) + (ulong)(advanced << 8) + (ulong)(iovLength);
                (int received, int advanced, int iovLength) UnpackReceiveState(ulong data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));

                int                 readableSocketCount = readableSockets.Count;
                iocb *              aioCb                    = AioCbs;
                iovec *             ioVectors                = IoVectorTable;
                PosixResult *       receiveResults           = stackalloc PosixResult[readableSocketCount];
                Span <MemoryHandle> receiveMemoryHandles     = MemoryHandles;
                int                 receiveMemoryHandleCount = 0;

                for (int i = 0; i < readableSocketCount; i++)
                {
                    TSocket socket           = readableSockets[i];
                    var     memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
                    int     advanced         = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);

                    aioCb->aio_fildes     = socket.Fd;
                    aioCb->aio_data       = PackReceiveState(0, advanced, memoryAllocation.IovLength);
                    aioCb->aio_lio_opcode = IOCB_CMD_PREADV;
                    aioCb->aio_buf        = (ulong)ioVectors;
                    aioCb->aio_nbytes     = (ulong)memoryAllocation.IovLength;
                    aioCb++;

                    ioVectors += memoryAllocation.IovLength;
                    receiveMemoryHandleCount += memoryAllocation.IovLength;
                    receiveMemoryHandles      = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
                }
                int eAgainCount = 0;

                while (readableSocketCount > 0)
                {
                    PosixResult res = AioInterop.IoSubmit(_aioContext, readableSocketCount, AioCbsTable);
                    if (res != readableSocketCount)
                    {
                        throw new NotSupportedException("Unexpected IoSubmit retval " + res);
                    }

                    io_event *aioEvents = AioEvents;
                    res = AioInterop.IoGetEvents(_aioContext, readableSocketCount, aioEvents);
                    if (res != readableSocketCount)
                    {
                        throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
                    }
                    int       socketsRemaining = readableSocketCount;
                    bool      allEAgain        = true;
                    io_event *aioEvent         = aioEvents;
                    for (int i = 0; i < readableSocketCount; i++)
                    {
                        PosixResult result      = new PosixResult((int)aioEvent->res);
                        int         socketIndex = i; // assumes in-order events
                        TSocket     socket      = readableSockets[socketIndex];
                        (int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->data);
                        (bool done, PosixResult retval)             = socket.InterpretReceiveResult(result, ref received, advanced, (iovec *)((iocb *)aioEvent->obj)->aio_buf, iovLength);
                        if (done)
                        {
                            receiveResults[socketIndex] = retval;
                            socketsRemaining--;
                            ((iocb *)aioEvent->obj)->aio_lio_opcode = IOCB_CMD_NOOP;
                            allEAgain = false;
                        }
                        else if (retval != PosixResult.EAGAIN)
                        {
                            ((iocb *)aioEvent->obj)->aio_data = PackReceiveState(received, advanced, iovLength);
                            allEAgain = false;
                        }
                        aioEvent++;
                    }
                    if (socketsRemaining > 0)
                    {
                        if (allEAgain)
                        {
                            eAgainCount++;
                            if (eAgainCount == TransportConstants.MaxEAgainCount)
                            {
                                throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
                            }
                        }
                        else
                        {
                            aioCb = AioCbs;
                            iocb *aioCbWriteAt = aioCb;
                            // The kernel doesn't handle Noop, we need to remove them from the aioCbs
                            for (int i = 0; i < readableSocketCount; i++)
                            {
                                if (aioCb[i].aio_lio_opcode != IOCB_CMD_NOOP)
                                {
                                    if (aioCbWriteAt != aioCb)
                                    {
                                        *aioCbWriteAt = *aioCb;
                                    }
                                    aioCbWriteAt++;
                                }
                                aioCb++;
                            }
                            readableSocketCount = socketsRemaining;
                            eAgainCount         = 0;
                        }
                    }
                    else
                    {
                        readableSocketCount = 0;
                    }
                }
                receiveMemoryHandles = MemoryHandles;
                for (int i = 0; i < receiveMemoryHandleCount; i++)
                {
                    receiveMemoryHandles[i].Dispose();
                }
                for (int i = 0; i < readableSockets.Count; i++)
                {
                    readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
                }
                readableSockets.Clear();
            }