public unsafe PosixResult TryReceive(ArraySegment <byte> buffer) { ValidateSegment(buffer); fixed(byte *buf = buffer.Array) { IOVector ioVector = new IOVector() { Base = buf + buffer.Offset, Count = (void *)buffer.Count }; return(SocketInterop.Receive(this, &ioVector, 1)); } }
public unsafe PosixResult TryGetPeerIPAddress(out IPEndPoint ep) { IPSocketAddress socketAddress; var rv = SocketInterop.GetPeerName(this, (byte *)&socketAddress, sizeof(IPSocketAddress)); if (rv.IsSuccess) { ep = socketAddress.ToIPEndPoint(); } else { ep = null; } return(rv); }
public unsafe PosixResult TryGetLocalIPAddress(out IPEndPointStruct ep, IPAddress reuseAddress = null) { IPSocketAddress socketAddress; var rv = SocketInterop.GetSockName(this, (byte *)&socketAddress, sizeof(IPSocketAddress)); if (rv.IsSuccess) { ep = socketAddress.ToIPEndPoint(reuseAddress); } else { ep = default(IPEndPointStruct); } return(rv); }
private static unsafe PosixResult TrySend(int fd, ref ReadableBuffer buffer) { int ioVectorLength = 0; foreach (var memory in buffer) { if (memory.Length == 0) { continue; } ioVectorLength++; if (ioVectorLength == MaxIOVectorSendLength) { // No more room in the IOVector break; } } if (ioVectorLength == 0) { return(new PosixResult(0)); } var ioVectors = stackalloc IOVector[ioVectorLength]; int i = 0; foreach (var memory in buffer) { if (memory.Length == 0) { continue; } void *pointer; memory.TryGetPointer(out pointer); ioVectors[i].Base = pointer; ioVectors[i].Count = (void *)memory.Length; i++; if (i == ioVectorLength) { // No more room in the IOVector break; } } return(SocketInterop.Send(fd, ioVectors, ioVectorLength)); }
public PosixResult TryDuplicate(out Socket dup) { return(SocketInterop.Duplicate(this, out dup)); }
public unsafe PosixResult TrySetSocketOption(SocketOptionLevel optionLevel, SocketOptionName optionName, int value) { return(SocketInterop.SetSockOpt(this, optionLevel, optionName, (byte *)&value, 4)); }
public unsafe PosixResult TrySend(IOVector *ioVectors, int ioVectorLen) { return(SocketInterop.Send(this, ioVectors, ioVectorLen)); }
public PosixResult TryShutdown(SocketShutdown shutdown) { return(SocketInterop.Shutdown(this, shutdown)); }
public PosixResult TryListen(int backlog) { return(SocketInterop.Listen(this, backlog)); }
public unsafe void Run() { try { Start(); CompleteStateChange(TransportThreadState.Started); } catch (Exception e) { CompleteStateChange(TransportThreadState.Stopped, e); return; } try { int notPacked = !EPoll.PackedEvents ? 1 : 0; var buffer = stackalloc int[EventBufferLength * (3 + notPacked)]; int statReadEvents = 0; int statWriteEvents = 0; int statAcceptEvents = 0; int statAccepts = 0; int statZeroCopySuccess = 0; int statZeroCopyCopied = 0; var acceptableSockets = new List <TSocket>(1); var readableSockets = new List <TSocket>(EventBufferLength); var writableSockets = new List <TSocket>(EventBufferLength); var reregisterEventSockets = new List <TSocket>(EventBufferLength); var zeroCopyCompletions = new List <TSocket>(EventBufferLength); bool pipeReadable = false; bool running = true; do { int numEvents = EPollInterop.EPollWait(_epollFd, buffer, EventBufferLength, timeout: EPoll.TimeoutInfinite).Value; // actions can be scheduled without unblocking epoll SetEpollNotBlocked(); // check events // we don't handle them immediately: // - this ensures we don't mismatch a closed socket with a new socket that have the same fd // ~ To have the same fd, the previous fd must be closed, which means it is removed from the epoll // ~ and won't show up in our next call to epoll.Wait. // ~ The old fd may be present in the buffer still, but lookup won't give a match, since it is removed // ~ from the dictionary before it is closed. If we were accepting already, a new socket could match. // - this also improves cache/cpu locality of the lookup int *ptr = buffer; lock (_sockets) { for (int i = 0; i < numEvents; i++) { // Packed Non-Packed // ------ ------ // 0:Events == Events // 1:Int1 = Key [Padding] // 2:Int2 = Key == Int1 = Key // 3:~~~~~~~~~~ Int2 = Key // ~~~~~~~~~~ EPollEvents events = (EPollEvents)ptr[0]; int key = ptr[2]; ptr += 3 + notPacked; TSocket tsocket; if (_sockets.TryGetValue(key, out tsocket)) { var type = tsocket.Type; if (type == SocketFlags.TypeClient) { lock (tsocket.Gate) { var pendingEventState = tsocket.PendingEventState; // zero copy if ((pendingEventState & EPollEvents.Error & events) != EPollEvents.None) { var copyResult = SocketInterop.CompleteZeroCopy(tsocket.Fd); if (copyResult != PosixResult.EAGAIN) { events &= ~EPollEvents.Error; pendingEventState &= ~EPollEvents.Error; zeroCopyCompletions.Add(tsocket); if (copyResult == SocketInterop.ZeroCopyCopied) { tsocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy; statZeroCopyCopied++; } else if (copyResult == SocketInterop.ZeroCopySuccess) { statZeroCopySuccess++; } else { Environment.FailFast($"Error occurred while trying to complete zero copy: {copyResult}"); } } } // treat Error as Readable, Writable if ((events & EPollEvents.Error) != EPollEvents.None) { events |= EPollEvents.Readable | EPollEvents.Writable; } events &= pendingEventState & (EPollEvents.Readable | EPollEvents.Writable); // readable if ((events & EPollEvents.Readable) != EPollEvents.None) { readableSockets.Add(tsocket); pendingEventState &= ~EPollEvents.Readable; } // writable if ((events & EPollEvents.Writable) != EPollEvents.None) { writableSockets.Add(tsocket); pendingEventState &= ~EPollEvents.Writable; } // reregister tsocket.PendingEventState = pendingEventState; if ((pendingEventState & (EPollEvents.Readable | EPollEvents.Writable)) != EPollEvents.None) { tsocket.PendingEventState |= TSocket.EventControlPending; reregisterEventSockets.Add(tsocket); } } } else { statAcceptEvents++; acceptableSockets.Add(tsocket); } } else if (key == PipeKey) { pipeReadable = true; } } } // zero copy for (int i = 0; i < zeroCopyCompletions.Count; i++) { zeroCopyCompletions[i].OnZeroCopyCompleted(); } zeroCopyCompletions.Clear(); // handle accepts statAcceptEvents += acceptableSockets.Count; for (int i = 0; i < acceptableSockets.Count; i++) { statAccepts += HandleAccept(acceptableSockets[i]); } acceptableSockets.Clear(); // handle writes statWriteEvents += writableSockets.Count; for (int i = 0; i < writableSockets.Count; i++) { writableSockets[i].OnWritable(stopped: false); } writableSockets.Clear(); // handle reads statReadEvents += readableSockets.Count; if (!_transportOptions.AioReceive) { bool checkAvailable = _transportOptions.CheckAvailable; Span <MemoryHandle> receiveMemoryHandles = MemoryHandles; for (int i = 0; i < readableSockets.Count; i++) { TSocket socket = readableSockets[i]; int availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes(); var receiveResult = socket.Receive(availableBytes, receiveMemoryHandles); socket.OnReceiveFromSocket(receiveResult); } readableSockets.Clear(); } else if (readableSockets.Count > 0) { AioReceive(readableSockets); } // reregister for events for (int i = 0; i < reregisterEventSockets.Count; i++) { var tsocket = reregisterEventSockets[i]; lock (tsocket.Gate) { var pendingEventState = tsocket.PendingEventState & ~TSocket.EventControlPending; tsocket.PendingEventState = pendingEventState; UpdateEPollControl(tsocket, pendingEventState, registered: true); } } reregisterEventSockets.Clear(); // handle pipe if (pipeReadable) { PosixResult result; do { result = _pipeEnds.ReadEnd.TryReadByte(); if (result.Value == PipeStopSockets) { StopSockets(); } else if (result.Value == PipeStopThread) { running = false; } else if (result.Value == PipeCloseAccept) { CloseAccept(); } } while (result); pipeReadable = false; } // scheduled work // note: this may write a byte to the pipe DoScheduledWork(_transportOptions.AioSend); } while (running); _logger.LogDebug($"Stats A/AE:{statAccepts}/{statAcceptEvents} RE:{statReadEvents} WE:{statWriteEvents} ZCS/ZCC:{statZeroCopySuccess}/{statZeroCopyCopied}"); CompleteStateChange(TransportThreadState.Stopped); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); Environment.FailFast("TransportThread", ex); } }
private TSocket CreateAcceptSocket(IPEndPoint endPoint, SocketFlags flags) { int acceptSocketFd = -1; int port = endPoint.Port; try { bool ipv4 = endPoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork; SocketInterop.Socket(ipv4 ? AddressFamily.InterNetwork : AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp, blocking: false, out acceptSocketFd).ThrowOnError(); TSocket acceptSocket = new TSocket(this, acceptSocketFd, flags); if (!ipv4) { // Kestrel does mapped ipv4 by default. acceptSocket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 0); } if (_transportOptions.ReceiveOnIncomingCpu) { if (_transportThread.CpuId != -1) { if (!acceptSocket.TrySetSocketOption(SocketOptionLevel.Socket, SocketOptionName.IncomingCpu, _transportThread.CpuId)) { _logger.LogWarning($"Cannot enable nameof{SocketOptionName.IncomingCpu} for {endPoint}"); } } } // Linux: allow bind during linger time acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1); // Linux: allow concurrent binds and let the kernel do load-balancing acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReusePort, 1); if ((flags & SocketFlags.DeferAccept) != 0) { // Linux: wait up to 1 sec for data to arrive before accepting socket acceptSocket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.DeferAccept, 1); } acceptSocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy; if (_transportOptions.ZeroCopy && _transportOptions.ZeroCopyThreshold != LinuxTransportOptions.NoZeroCopy) { if (acceptSocket.TrySetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ZeroCopy, 1)) { acceptSocket.ZeroCopyThreshold = _transportOptions.ZeroCopyThreshold; } } acceptSocket.Bind(endPoint); if (port == 0) { // When testing we want the OS to select a free port port = acceptSocket.GetLocalIPAddress().Port; } acceptSocket.Listen(ListenBacklog); endPoint.Port = port; return(acceptSocket); } catch { if (acceptSocketFd != -1) { IOInterop.Close(acceptSocketFd); } throw; } }
public unsafe PosixResult TryAcceptAndSendHandleTo(int toSocket) { return(SocketInterop.AcceptAndSendHandleTo(this, toSocket)); }
public unsafe PosixResult TryReceiveSocket(out Socket socket, bool blocking) { return(SocketInterop.ReceiveSocket(this, out socket, blocking)); }
public PosixResult TryGetAvailableBytes() { return(SocketInterop.GetAvailableBytes(this)); }
public unsafe PosixResult TryAccept(out Socket clientSocket, bool blocking) { return(SocketInterop.Accept(this, null, 0, blocking, out clientSocket)); }
public unsafe PosixResult TryBind(IPEndPoint endpoint) { IPSocketAddress socketAddress = new IPSocketAddress(endpoint); return(SocketInterop.Bind(this, (byte *)&socketAddress, sizeof(IPSocketAddress))); }
public unsafe PosixResult TryReceive(IOVector *ioVectors, int ioVectorLen) { return(SocketInterop.Receive(this, ioVectors, ioVectorLen)); }
private static unsafe Exception Receive(int fd, int availableBytes, ref WritableBuffer wb) { int ioVectorLength = availableBytes <= wb.Buffer.Length ? 1 : Math.Min(1 + (availableBytes - wb.Buffer.Length + MaxPooledBlockLength - 1) / MaxPooledBlockLength, MaxIOVectorReceiveLength); var ioVectors = stackalloc IOVector[ioVectorLength]; var allocated = 0; var advanced = 0; int ioVectorsUsed = 0; for (; ioVectorsUsed < ioVectorLength; ioVectorsUsed++) { wb.Ensure(1); var memory = wb.Buffer; var length = memory.Length; void *pointer; wb.Buffer.TryGetPointer(out pointer); ioVectors[ioVectorsUsed].Base = pointer; ioVectors[ioVectorsUsed].Count = (void *)length; allocated += length; if (allocated >= availableBytes) { // Every Memory (except the last one) must be filled completely. ioVectorsUsed++; break; } wb.Advance(length); advanced += length; } var expectedMin = Math.Min(availableBytes, allocated); // Ideally we get availableBytes in a single receive // but we are happy if we get at least a part of it // and we are willing to take {MaxEAgainCount} EAGAINs. // Less data could be returned due to these reasons: // * TCP URG // * packet was not placed in receive queue (race with FIONREAD) // * ? const int MaxEAgainCount = 10; var eAgainCount = 0; var received = 0; do { var result = SocketInterop.Receive(fd, ioVectors, ioVectorsUsed); if (result.IsSuccess) { received += result.Value; if (received >= expectedMin) { // We made it! wb.Advance(received - advanced); return(null); } eAgainCount = 0; // Update ioVectors to match bytes read var skip = result.Value; for (int i = 0; (i < ioVectorsUsed) && (skip > 0); i++) { var length = (int)ioVectors[i].Count; var skipped = Math.Min(skip, length); ioVectors[i].Count = (void *)(length - skipped); ioVectors[i].Base = (byte *)ioVectors[i].Base + skipped; skip -= skipped; } } else if (result == PosixResult.EAGAIN || result == PosixResult.EWOULDBLOCK) { eAgainCount++; if (eAgainCount == MaxEAgainCount) { return(new NotSupportedException("Too many EAGAIN, unable to receive available bytes.")); } } else if (result == PosixResult.ECONNRESET) { return(new ConnectionResetException(result.ErrorDescription(), result.AsException())); } else { return(result.AsException()); } } while (true); }
public unsafe PosixResult TryConnect(string unixPath) { UnixSocketAddress socketAddress = new UnixSocketAddress(unixPath); return(SocketInterop.Connect(this, (byte *)&socketAddress, sizeof(UnixSocketAddress))); }