Esempio n. 1
0
            private unsafe void AioReceive(List <TSocket> readableSockets)
            {
                long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength);
                (int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));

                int                 readableSocketCount = readableSockets.Count;
                AioCb *             aioCb                    = AioCbs;
                IOVector *          ioVectors                = IoVectorTable;
                PosixResult *       receiveResults           = stackalloc PosixResult[readableSocketCount];
                bool                checkAvailable           = _transportOptions.CheckAvailable;
                Span <MemoryHandle> receiveMemoryHandles     = MemoryHandles;
                int                 receiveMemoryHandleCount = 0;

                for (int i = 0; i < readableSocketCount; i++)
                {
                    TSocket socket         = readableSockets[i];
                    int     availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes();
                    int     ioVectorLength = socket.CalcIOVectorLengthForReceive(availableBytes, IoVectorsPerAioSocket);
                    int     advanced       = socket.FillReceiveIOVector(availableBytes, ioVectors, receiveMemoryHandles, ref ioVectorLength);

                    aioCb->Fd     = socket.Fd;
                    aioCb->Data   = PackReceiveState(0, advanced, ioVectorLength);
                    aioCb->OpCode = AioOpCode.PReadv;
                    aioCb->Buffer = ioVectors;
                    aioCb->Length = ioVectorLength;
                    aioCb++;

                    ioVectors += ioVectorLength;
                    receiveMemoryHandleCount += ioVectorLength;
                    receiveMemoryHandles      = receiveMemoryHandles.Slice(ioVectorLength);
                }
                int eAgainCount = 0;

                while (readableSocketCount > 0)
                {
                    IntPtr      ctxp = _aioContext;
                    PosixResult res  = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable);
                    if (res != readableSocketCount)
                    {
                        throw new NotSupportedException("Unexpected IoSubmit retval " + res);
                    }

                    AioEvent *aioEvents = AioEvents;
                    res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents);
                    if (res != readableSocketCount)
                    {
                        throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
                    }
                    int       socketsRemaining = readableSocketCount;
                    bool      allEAgain        = true;
                    AioEvent *aioEvent         = aioEvents;
                    for (int i = 0; i < readableSocketCount; i++)
                    {
                        PosixResult result      = aioEvent->Result;
                        int         socketIndex = i; // assumes in-order events
                        TSocket     socket      = readableSockets[socketIndex];
                        (int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data);
                        (bool done, PosixResult retval)             = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength);
                        if (done)
                        {
                            receiveResults[socketIndex] = retval;
                            socketsRemaining--;
                            aioEvent->AioCb->OpCode = AioOpCode.Noop;
                            allEAgain = false;
                        }
                        else if (retval != PosixResult.EAGAIN)
                        {
                            aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength);
                            allEAgain             = false;
                        }
                        aioEvent++;
                    }
                    if (socketsRemaining > 0)
                    {
                        if (allEAgain)
                        {
                            eAgainCount++;
                            if (eAgainCount == TransportConstants.MaxEAgainCount)
                            {
                                throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
                            }
                        }
                        else
                        {
                            aioCb = AioCbs;
                            AioCb *aioCbWriteAt = aioCb;
                            // The kernel doesn't handle Noop, we need to remove them from the aioCbs
                            for (int i = 0; i < readableSocketCount; i++)
                            {
                                if (aioCb[i].OpCode != AioOpCode.Noop)
                                {
                                    if (aioCbWriteAt != aioCb)
                                    {
                                        *aioCbWriteAt = *aioCb;
                                    }
                                    aioCbWriteAt++;
                                }
                                aioCb++;
                            }
                            readableSocketCount = socketsRemaining;
                            eAgainCount         = 0;
                        }
                    }
                    else
                    {
                        readableSocketCount = 0;
                    }
                }
                for (int i = 0; i < readableSockets.Count; i++)
                {
                    readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
                }
                readableSockets.Clear();
                receiveMemoryHandles = MemoryHandles;
                for (int i = 0; i < receiveMemoryHandleCount; i++)
                {
                    receiveMemoryHandles[i].Dispose();
                }
            }
Esempio n. 2
0
            public unsafe void Run()
            {
                try
                {
                    Start();
                    CompleteStateChange(TransportThreadState.Started);
                }
                catch (Exception e)
                {
                    CompleteStateChange(TransportThreadState.Stopped, e);
                    return;
                }

                try
                {
                    int notPacked           = !EPoll.PackedEvents ? 1 : 0;
                    var buffer              = stackalloc int[EventBufferLength * (3 + notPacked)];
                    int statReadEvents      = 0;
                    int statWriteEvents     = 0;
                    int statAcceptEvents    = 0;
                    int statAccepts         = 0;
                    int statZeroCopySuccess = 0;
                    int statZeroCopyCopied  = 0;

                    var  acceptableSockets      = new List <TSocket>(1);
                    var  readableSockets        = new List <TSocket>(EventBufferLength);
                    var  writableSockets        = new List <TSocket>(EventBufferLength);
                    var  reregisterEventSockets = new List <TSocket>(EventBufferLength);
                    var  zeroCopyCompletions    = new List <TSocket>(EventBufferLength);
                    bool pipeReadable           = false;


                    bool running = true;
                    do
                    {
                        int numEvents = EPollInterop.EPollWait(_epollFd, buffer, EventBufferLength, timeout: EPoll.TimeoutInfinite).Value;

                        // actions can be scheduled without unblocking epoll
                        SetEpollNotBlocked();

                        // check events
                        // we don't handle them immediately:
                        // - this ensures we don't mismatch a closed socket with a new socket that have the same fd
                        //     ~ To have the same fd, the previous fd must be closed, which means it is removed from the epoll
                        //     ~ and won't show up in our next call to epoll.Wait.
                        //     ~ The old fd may be present in the buffer still, but lookup won't give a match, since it is removed
                        //     ~ from the dictionary before it is closed. If we were accepting already, a new socket could match.
                        // - this also improves cache/cpu locality of the lookup
                        int *ptr = buffer;
                        lock (_sockets)
                        {
                            for (int i = 0; i < numEvents; i++)
                            {
                                //   Packed             Non-Packed
                                //   ------             ------
                                // 0:Events       ==    Events
                                // 1:Int1 = Key         [Padding]
                                // 2:Int2 = Key   ==    Int1 = Key
                                // 3:~~~~~~~~~~         Int2 = Key
                                //                      ~~~~~~~~~~
                                EPollEvents events = (EPollEvents)ptr[0];
                                int         key    = ptr[2];
                                ptr += 3 + notPacked;
                                TSocket tsocket;
                                if (_sockets.TryGetValue(key, out tsocket))
                                {
                                    var type = tsocket.Type;
                                    if (type == SocketFlags.TypeClient)
                                    {
                                        lock (tsocket.Gate)
                                        {
                                            var pendingEventState = tsocket.PendingEventState;

                                            // zero copy
                                            if ((pendingEventState & EPollEvents.Error & events) != EPollEvents.None)
                                            {
                                                var copyResult = SocketInterop.CompleteZeroCopy(tsocket.Fd);
                                                if (copyResult != PosixResult.EAGAIN)
                                                {
                                                    events            &= ~EPollEvents.Error;
                                                    pendingEventState &= ~EPollEvents.Error;
                                                    zeroCopyCompletions.Add(tsocket);
                                                    if (copyResult == SocketInterop.ZeroCopyCopied)
                                                    {
                                                        tsocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy;
                                                        statZeroCopyCopied++;
                                                    }
                                                    else if (copyResult == SocketInterop.ZeroCopySuccess)
                                                    {
                                                        statZeroCopySuccess++;
                                                    }
                                                    else
                                                    {
                                                        Environment.FailFast($"Error occurred while trying to complete zero copy: {copyResult}");
                                                    }
                                                }
                                            }

                                            // treat Error as Readable, Writable
                                            if ((events & EPollEvents.Error) != EPollEvents.None)
                                            {
                                                events |= EPollEvents.Readable | EPollEvents.Writable;
                                            }

                                            events &= pendingEventState & (EPollEvents.Readable | EPollEvents.Writable);
                                            // readable
                                            if ((events & EPollEvents.Readable) != EPollEvents.None)
                                            {
                                                readableSockets.Add(tsocket);
                                                pendingEventState &= ~EPollEvents.Readable;
                                            }
                                            // writable
                                            if ((events & EPollEvents.Writable) != EPollEvents.None)
                                            {
                                                writableSockets.Add(tsocket);
                                                pendingEventState &= ~EPollEvents.Writable;
                                            }

                                            // reregister
                                            tsocket.PendingEventState = pendingEventState;
                                            if ((pendingEventState & (EPollEvents.Readable | EPollEvents.Writable)) != EPollEvents.None)
                                            {
                                                tsocket.PendingEventState |= TSocket.EventControlPending;
                                                reregisterEventSockets.Add(tsocket);
                                            }
                                        }
                                    }
                                    else
                                    {
                                        statAcceptEvents++;
                                        acceptableSockets.Add(tsocket);
                                    }
                                }
                                else if (key == PipeKey)
                                {
                                    pipeReadable = true;
                                }
                            }
                        }

                        // zero copy
                        for (int i = 0; i < zeroCopyCompletions.Count; i++)
                        {
                            zeroCopyCompletions[i].OnZeroCopyCompleted();
                        }
                        zeroCopyCompletions.Clear();

                        // handle accepts
                        statAcceptEvents += acceptableSockets.Count;
                        for (int i = 0; i < acceptableSockets.Count; i++)
                        {
                            statAccepts += HandleAccept(acceptableSockets[i]);
                        }
                        acceptableSockets.Clear();

                        // handle writes
                        statWriteEvents += writableSockets.Count;
                        for (int i = 0; i < writableSockets.Count; i++)
                        {
                            writableSockets[i].OnWritable(stopped: false);
                        }
                        writableSockets.Clear();

                        // handle reads
                        statReadEvents += readableSockets.Count;
                        if (!_transportOptions.AioReceive)
                        {
                            bool checkAvailable = _transportOptions.CheckAvailable;
                            Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
                            for (int i = 0; i < readableSockets.Count; i++)
                            {
                                TSocket socket         = readableSockets[i];
                                int     availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes();
                                var     receiveResult  = socket.Receive(availableBytes, receiveMemoryHandles);
                                socket.OnReceiveFromSocket(receiveResult);
                            }
                            readableSockets.Clear();
                        }
                        else if (readableSockets.Count > 0)
                        {
                            AioReceive(readableSockets);
                        }

                        // reregister for events
                        for (int i = 0; i < reregisterEventSockets.Count; i++)
                        {
                            var tsocket = reregisterEventSockets[i];
                            lock (tsocket.Gate)
                            {
                                var pendingEventState = tsocket.PendingEventState & ~TSocket.EventControlPending;
                                tsocket.PendingEventState = pendingEventState;
                                UpdateEPollControl(tsocket, pendingEventState, registered: true);
                            }
                        }
                        reregisterEventSockets.Clear();

                        // handle pipe
                        if (pipeReadable)
                        {
                            PosixResult result;
                            do
                            {
                                result = _pipeEnds.ReadEnd.TryReadByte();
                                if (result.Value == PipeStopSockets)
                                {
                                    StopSockets();
                                }
                                else if (result.Value == PipeStopThread)
                                {
                                    running = false;
                                }
                                else if (result.Value == PipeCloseAccept)
                                {
                                    CloseAccept();
                                }
                            } while (result);
                            pipeReadable = false;
                        }

                        // scheduled work
                        // note: this may write a byte to the pipe
                        DoScheduledWork(_transportOptions.AioSend);
                    } while (running);

                    _logger.LogDebug($"Stats A/AE:{statAccepts}/{statAcceptEvents} RE:{statReadEvents} WE:{statWriteEvents} ZCS/ZCC:{statZeroCopySuccess}/{statZeroCopyCopied}");

                    CompleteStateChange(TransportThreadState.Stopped);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.Message);
                    Console.WriteLine(ex.StackTrace);
                    Environment.FailFast("TransportThread", ex);
                }
            }