public unsafe PosixResult TryReceive(ArraySegment <byte> buffer)
{
ValidateSegment(buffer);
fixed(byte *buf = buffer.Array)
{
IOVector ioVector = new IOVector()
{
Base = buf + buffer.Offset, Count = (void *)buffer.Count
};
return(SocketInterop.Receive(this, &ioVector, 1));
}
}
public unsafe PosixResult TryGetPeerIPAddress(out IPEndPoint ep)
{
IPSocketAddress socketAddress;
var rv = SocketInterop.GetPeerName(this, (byte *)&socketAddress, sizeof(IPSocketAddress));
if (rv.IsSuccess)
{
ep = socketAddress.ToIPEndPoint();
}
else
{
ep = null;
}
return(rv);
}
public unsafe PosixResult TryGetLocalIPAddress(out IPEndPointStruct ep, IPAddress reuseAddress = null)
{
IPSocketAddress socketAddress;
var rv = SocketInterop.GetSockName(this, (byte *)&socketAddress, sizeof(IPSocketAddress));
if (rv.IsSuccess)
{
ep = socketAddress.ToIPEndPoint(reuseAddress);
}
else
{
ep = default(IPEndPointStruct);
}
return(rv);
}
private static unsafe PosixResult TrySend(int fd, ref ReadableBuffer buffer)
{
int ioVectorLength = 0;
foreach (var memory in buffer)
{
if (memory.Length == 0)
{
continue;
}
ioVectorLength++;
if (ioVectorLength == MaxIOVectorSendLength)
{
// No more room in the IOVector
break;
}
}
if (ioVectorLength == 0)
{
return(new PosixResult(0));
}
var ioVectors = stackalloc IOVector[ioVectorLength];
int i = 0;
foreach (var memory in buffer)
{
if (memory.Length == 0)
{
continue;
}
void *pointer;
memory.TryGetPointer(out pointer);
ioVectors[i].Base = pointer;
ioVectors[i].Count = (void *)memory.Length;
i++;
if (i == ioVectorLength)
{
// No more room in the IOVector
break;
}
}
return(SocketInterop.Send(fd, ioVectors, ioVectorLength));
}
public PosixResult TryDuplicate(out Socket dup)
{
return(SocketInterop.Duplicate(this, out dup));
}
public unsafe PosixResult TrySetSocketOption(SocketOptionLevel optionLevel, SocketOptionName optionName, int value)
{
return(SocketInterop.SetSockOpt(this, optionLevel, optionName, (byte *)&value, 4));
}
public unsafe PosixResult TrySend(IOVector *ioVectors, int ioVectorLen)
{
return(SocketInterop.Send(this, ioVectors, ioVectorLen));
}
public PosixResult TryShutdown(SocketShutdown shutdown)
{
return(SocketInterop.Shutdown(this, shutdown));
}
public PosixResult TryListen(int backlog)
{
return(SocketInterop.Listen(this, backlog));
}
public unsafe void Run()
{
try
{
Start();
CompleteStateChange(TransportThreadState.Started);
}
catch (Exception e)
{
CompleteStateChange(TransportThreadState.Stopped, e);
return;
}
try
{
int notPacked = !EPoll.PackedEvents ? 1 : 0;
var buffer = stackalloc int[EventBufferLength * (3 + notPacked)];
int statReadEvents = 0;
int statWriteEvents = 0;
int statAcceptEvents = 0;
int statAccepts = 0;
int statZeroCopySuccess = 0;
int statZeroCopyCopied = 0;
var acceptableSockets = new List <TSocket>(1);
var readableSockets = new List <TSocket>(EventBufferLength);
var writableSockets = new List <TSocket>(EventBufferLength);
var reregisterEventSockets = new List <TSocket>(EventBufferLength);
var zeroCopyCompletions = new List <TSocket>(EventBufferLength);
bool pipeReadable = false;
bool running = true;
do
{
int numEvents = EPollInterop.EPollWait(_epollFd, buffer, EventBufferLength, timeout: EPoll.TimeoutInfinite).Value;
// actions can be scheduled without unblocking epoll
SetEpollNotBlocked();
// check events
// we don't handle them immediately:
// - this ensures we don't mismatch a closed socket with a new socket that have the same fd
// ~ To have the same fd, the previous fd must be closed, which means it is removed from the epoll
// ~ and won't show up in our next call to epoll.Wait.
// ~ The old fd may be present in the buffer still, but lookup won't give a match, since it is removed
// ~ from the dictionary before it is closed. If we were accepting already, a new socket could match.
// - this also improves cache/cpu locality of the lookup
int *ptr = buffer;
lock (_sockets)
{
for (int i = 0; i < numEvents; i++)
{
// Packed Non-Packed
// ------ ------
// 0:Events == Events
// 1:Int1 = Key [Padding]
// 2:Int2 = Key == Int1 = Key
// 3:~~~~~~~~~~ Int2 = Key
// ~~~~~~~~~~
EPollEvents events = (EPollEvents)ptr[0];
int key = ptr[2];
ptr += 3 + notPacked;
TSocket tsocket;
if (_sockets.TryGetValue(key, out tsocket))
{
var type = tsocket.Type;
if (type == SocketFlags.TypeClient)
{
lock (tsocket.Gate)
{
var pendingEventState = tsocket.PendingEventState;
// zero copy
if ((pendingEventState & EPollEvents.Error & events) != EPollEvents.None)
{
var copyResult = SocketInterop.CompleteZeroCopy(tsocket.Fd);
if (copyResult != PosixResult.EAGAIN)
{
events &= ~EPollEvents.Error;
pendingEventState &= ~EPollEvents.Error;
zeroCopyCompletions.Add(tsocket);
if (copyResult == SocketInterop.ZeroCopyCopied)
{
tsocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy;
statZeroCopyCopied++;
}
else if (copyResult == SocketInterop.ZeroCopySuccess)
{
statZeroCopySuccess++;
}
else
{
Environment.FailFast($"Error occurred while trying to complete zero copy: {copyResult}");
}
}
}
// treat Error as Readable, Writable
if ((events & EPollEvents.Error) != EPollEvents.None)
{
events |= EPollEvents.Readable | EPollEvents.Writable;
}
events &= pendingEventState & (EPollEvents.Readable | EPollEvents.Writable);
// readable
if ((events & EPollEvents.Readable) != EPollEvents.None)
{
readableSockets.Add(tsocket);
pendingEventState &= ~EPollEvents.Readable;
}
// writable
if ((events & EPollEvents.Writable) != EPollEvents.None)
{
writableSockets.Add(tsocket);
pendingEventState &= ~EPollEvents.Writable;
}
// reregister
tsocket.PendingEventState = pendingEventState;
if ((pendingEventState & (EPollEvents.Readable | EPollEvents.Writable)) != EPollEvents.None)
{
tsocket.PendingEventState |= TSocket.EventControlPending;
reregisterEventSockets.Add(tsocket);
}
}
}
else
{
statAcceptEvents++;
acceptableSockets.Add(tsocket);
}
}
else if (key == PipeKey)
{
pipeReadable = true;
}
}
}
// zero copy
for (int i = 0; i < zeroCopyCompletions.Count; i++)
{
zeroCopyCompletions[i].OnZeroCopyCompleted();
}
zeroCopyCompletions.Clear();
// handle accepts
statAcceptEvents += acceptableSockets.Count;
for (int i = 0; i < acceptableSockets.Count; i++)
{
statAccepts += HandleAccept(acceptableSockets[i]);
}
acceptableSockets.Clear();
// handle writes
statWriteEvents += writableSockets.Count;
for (int i = 0; i < writableSockets.Count; i++)
{
writableSockets[i].OnWritable(stopped: false);
}
writableSockets.Clear();
// handle reads
statReadEvents += readableSockets.Count;
if (!_transportOptions.AioReceive)
{
bool checkAvailable = _transportOptions.CheckAvailable;
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < readableSockets.Count; i++)
{
TSocket socket = readableSockets[i];
int availableBytes = !checkAvailable ? 0 : socket.GetAvailableBytes();
var receiveResult = socket.Receive(availableBytes, receiveMemoryHandles);
socket.OnReceiveFromSocket(receiveResult);
}
readableSockets.Clear();
}
else if (readableSockets.Count > 0)
{
AioReceive(readableSockets);
}
// reregister for events
for (int i = 0; i < reregisterEventSockets.Count; i++)
{
var tsocket = reregisterEventSockets[i];
lock (tsocket.Gate)
{
var pendingEventState = tsocket.PendingEventState & ~TSocket.EventControlPending;
tsocket.PendingEventState = pendingEventState;
UpdateEPollControl(tsocket, pendingEventState, registered: true);
}
}
reregisterEventSockets.Clear();
// handle pipe
if (pipeReadable)
{
PosixResult result;
do
{
result = _pipeEnds.ReadEnd.TryReadByte();
if (result.Value == PipeStopSockets)
{
StopSockets();
}
else if (result.Value == PipeStopThread)
{
running = false;
}
else if (result.Value == PipeCloseAccept)
{
CloseAccept();
}
} while (result);
pipeReadable = false;
}
// scheduled work
// note: this may write a byte to the pipe
DoScheduledWork(_transportOptions.AioSend);
} while (running);
_logger.LogDebug($"Stats A/AE:{statAccepts}/{statAcceptEvents} RE:{statReadEvents} WE:{statWriteEvents} ZCS/ZCC:{statZeroCopySuccess}/{statZeroCopyCopied}");
CompleteStateChange(TransportThreadState.Stopped);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
Environment.FailFast("TransportThread", ex);
}
}
private TSocket CreateAcceptSocket(IPEndPoint endPoint, SocketFlags flags)
{
int acceptSocketFd = -1;
int port = endPoint.Port;
try
{
bool ipv4 = endPoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork;
SocketInterop.Socket(ipv4 ? AddressFamily.InterNetwork : AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp, blocking: false,
out acceptSocketFd).ThrowOnError();
TSocket acceptSocket = new TSocket(this, acceptSocketFd, flags);
if (!ipv4)
{
// Kestrel does mapped ipv4 by default.
acceptSocket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 0);
}
if (_transportOptions.ReceiveOnIncomingCpu)
{
if (_transportThread.CpuId != -1)
{
if (!acceptSocket.TrySetSocketOption(SocketOptionLevel.Socket, SocketOptionName.IncomingCpu, _transportThread.CpuId))
{
_logger.LogWarning($"Cannot enable nameof{SocketOptionName.IncomingCpu} for {endPoint}");
}
}
}
// Linux: allow bind during linger time
acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);
// Linux: allow concurrent binds and let the kernel do load-balancing
acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReusePort, 1);
if ((flags & SocketFlags.DeferAccept) != 0)
{
// Linux: wait up to 1 sec for data to arrive before accepting socket
acceptSocket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.DeferAccept, 1);
}
acceptSocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy;
if (_transportOptions.ZeroCopy && _transportOptions.ZeroCopyThreshold != LinuxTransportOptions.NoZeroCopy)
{
if (acceptSocket.TrySetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ZeroCopy, 1))
{
acceptSocket.ZeroCopyThreshold = _transportOptions.ZeroCopyThreshold;
}
}
acceptSocket.Bind(endPoint);
if (port == 0)
{
// When testing we want the OS to select a free port
port = acceptSocket.GetLocalIPAddress().Port;
}
acceptSocket.Listen(ListenBacklog);
endPoint.Port = port;
return(acceptSocket);
}
catch
{
if (acceptSocketFd != -1)
{
IOInterop.Close(acceptSocketFd);
}
throw;
}
}
public unsafe PosixResult TryAcceptAndSendHandleTo(int toSocket)
{
return(SocketInterop.AcceptAndSendHandleTo(this, toSocket));
}
public unsafe PosixResult TryReceiveSocket(out Socket socket, bool blocking)
{
return(SocketInterop.ReceiveSocket(this, out socket, blocking));
}
public PosixResult TryGetAvailableBytes()
{
return(SocketInterop.GetAvailableBytes(this));
}
public unsafe PosixResult TryAccept(out Socket clientSocket, bool blocking)
{
return(SocketInterop.Accept(this, null, 0, blocking, out clientSocket));
}
public unsafe PosixResult TryBind(IPEndPoint endpoint)
{
IPSocketAddress socketAddress = new IPSocketAddress(endpoint);
return(SocketInterop.Bind(this, (byte *)&socketAddress, sizeof(IPSocketAddress)));
}
public unsafe PosixResult TryReceive(IOVector *ioVectors, int ioVectorLen)
{
return(SocketInterop.Receive(this, ioVectors, ioVectorLen));
}
private static unsafe Exception Receive(int fd, int availableBytes, ref WritableBuffer wb)
{
int ioVectorLength = availableBytes <= wb.Buffer.Length ? 1 :
Math.Min(1 + (availableBytes - wb.Buffer.Length + MaxPooledBlockLength - 1) / MaxPooledBlockLength, MaxIOVectorReceiveLength);
var ioVectors = stackalloc IOVector[ioVectorLength];
var allocated = 0;
var advanced = 0;
int ioVectorsUsed = 0;
for (; ioVectorsUsed < ioVectorLength; ioVectorsUsed++)
{
wb.Ensure(1);
var memory = wb.Buffer;
var length = memory.Length;
void *pointer;
wb.Buffer.TryGetPointer(out pointer);
ioVectors[ioVectorsUsed].Base = pointer;
ioVectors[ioVectorsUsed].Count = (void *)length;
allocated += length;
if (allocated >= availableBytes)
{
// Every Memory (except the last one) must be filled completely.
ioVectorsUsed++;
break;
}
wb.Advance(length);
advanced += length;
}
var expectedMin = Math.Min(availableBytes, allocated);
// Ideally we get availableBytes in a single receive
// but we are happy if we get at least a part of it
// and we are willing to take {MaxEAgainCount} EAGAINs.
// Less data could be returned due to these reasons:
// * TCP URG
// * packet was not placed in receive queue (race with FIONREAD)
// * ?
const int MaxEAgainCount = 10;
var eAgainCount = 0;
var received = 0;
do
{
var result = SocketInterop.Receive(fd, ioVectors, ioVectorsUsed);
if (result.IsSuccess)
{
received += result.Value;
if (received >= expectedMin)
{
// We made it!
wb.Advance(received - advanced);
return(null);
}
eAgainCount = 0;
// Update ioVectors to match bytes read
var skip = result.Value;
for (int i = 0; (i < ioVectorsUsed) && (skip > 0); i++)
{
var length = (int)ioVectors[i].Count;
var skipped = Math.Min(skip, length);
ioVectors[i].Count = (void *)(length - skipped);
ioVectors[i].Base = (byte *)ioVectors[i].Base + skipped;
skip -= skipped;
}
}
else if (result == PosixResult.EAGAIN || result == PosixResult.EWOULDBLOCK)
{
eAgainCount++;
if (eAgainCount == MaxEAgainCount)
{
return(new NotSupportedException("Too many EAGAIN, unable to receive available bytes."));
}
}
else if (result == PosixResult.ECONNRESET)
{
return(new ConnectionResetException(result.ErrorDescription(), result.AsException()));
}
else
{
return(result.AsException());
}
} while (true);
}
public unsafe PosixResult TryConnect(string unixPath)
{
UnixSocketAddress socketAddress = new UnixSocketAddress(unixPath);
return(SocketInterop.Connect(this, (byte *)&socketAddress, sizeof(UnixSocketAddress)));
}