private void AcceptOn(TSocket tsocket)
{
try
{
_acceptSockets.Add(tsocket);
lock (_sockets)
{
_sockets.Add(tsocket.Fd, tsocket);
}
EPollInterop.EPollControl(_epollFd,
EPollOperation.Add,
tsocket.Fd,
EPollEvents.Readable,
EPollData(tsocket.Fd));
}
catch
{
tsocket.Close();
_acceptSockets.Remove(tsocket);
lock (_sockets)
{
_sockets.Remove(tsocket.Fd);
}
throw;
}
}
private void Start()
{
// register pipe
EPollInterop.EPollControl(_epollFd,
EPollOperation.Add,
_pipeEnds.ReadEnd.DangerousGetHandle().ToInt32(),
EPollEvents.Readable,
EPollData(PipeKey));
// create accept socket
{
TSocket acceptSocket;
SocketFlags flags = SocketFlags.None;
if (_transportOptions.DeferSend)
{
flags |= SocketFlags.DeferSend;
}
;
if (_transportThread.AcceptThread != null)
{
flags |= SocketFlags.TypePassFd;
int acceptSocketFd = _transportThread.AcceptThread.CreateReceiveSocket();
acceptSocket = new TSocket(this, acceptSocketFd, flags);
acceptSocket.ZeroCopyThreshold = LinuxTransportOptions.NoZeroCopy;
}
else
{
flags |= SocketFlags.TypeAccept;
acceptSocket = CreateAcceptSocket(_transportThread.EndPoint, flags);
}
// accept connections
AcceptOn(acceptSocket);
}
}
// must be called under tsocket.Gate
public void UpdateEPollControl(TSocket tsocket, EPollEvents flags, bool registered)
{
flags &= EPollEvents.Readable | EPollEvents.Writable | EPollEvents.Error;
EPollInterop.EPollControl(_epollFd,
registered ? EPollOperation.Modify : EPollOperation.Add,
tsocket.Fd,
flags | EPollEvents.OneShot,
EPollData(tsocket.Fd));
}
private static void RegisterForReadable(TSocket tsocket)
{
bool register = tsocket.SetRegistered();
EPollInterop.EPollControl(tsocket.ThreadContext.EPollFd,
register ? EPollOperation.Add : EPollOperation.Modify,
tsocket.Fd,
EPollEvents.Readable | EPollEvents.OneShot,
EPollData(tsocket.Fd));
}
private static void RegisterForWritable(TSocket tsocket)
{
bool registered = tsocket.DupSocket != null;
// To avoid having to synchronize the event mask with the Readable
// we dup the socket.
// In the EPollData we set the highest bit to indicate this is the
// poll for writable.
if (!registered)
{
tsocket.DupSocket = tsocket.Socket.Duplicate();
}
EPollInterop.EPollControl(tsocket.ThreadContext.EPollFd,
registered ? EPollOperation.Modify : EPollOperation.Add,
tsocket.DupSocket.DangerousGetHandle().ToInt32(),
EPollEvents.Writable | EPollEvents.OneShot,
EPollData(tsocket.Fd | DupKeyMask));
}
private unsafe void PollThread(object obj)
{
ThreadContext threadContext = null;
Exception error = null;
try
{
// .NET doesn't support setting thread affinity on Start
// We could change it before starting the thread
// so it gets inherited, but we don't know how many threads
// the runtime may start.
if (_cpuId != -1)
{
Scheduler.SetCurrentThreadAffinity(_cpuId);
}
// objects are allocated on the PollThread heap
int pipeKey;
threadContext = new ThreadContext(this, _transportOptions, _connectionHandler, CreateLogger());
threadContext.Initialize();
{
// register pipe
pipeKey = threadContext.PipeEnds.ReadEnd.DangerousGetHandle().ToInt32();
EPollInterop.EPollControl(threadContext.EPollFd,
EPollOperation.Add,
threadContext.PipeEnds.ReadEnd.DangerousGetHandle().ToInt32(),
EPollEvents.Readable,
EPollData(pipeKey));
// accept connections
AcceptOn(_endPoint, _cpuId, _transportOptions, threadContext);
_threadContext = threadContext;
}
int epollFd = threadContext.EPollFd;
var readEnd = threadContext.PipeEnds.ReadEnd;
int notPacked = !EPoll.PackedEvents ? 1 : 0;
var buffer = stackalloc int[EventBufferLength * (3 + notPacked)];
int statReadEvents = 0;
int statWriteEvents = 0;
int statAcceptEvents = 0;
int statAccepts = 0;
var sockets = threadContext.Sockets;
var acceptableSockets = new List <TSocket>(1);
var readableSockets = new List <TSocket>(EventBufferLength);
var writableSockets = new List <TSocket>(EventBufferLength);
bool pipeReadable = false;
CompleteStateChange(State.Started);
bool running = true;
do
{
int numEvents = EPollInterop.EPollWait(epollFd, buffer, EventBufferLength, timeout: EPoll.TimeoutInfinite).Value;
// actions can be scheduled without unblocking epoll
threadContext.SetEpollNotBlocked();
// check events
// we don't handle them immediately:
// - this ensures we don't mismatch a closed socket with a new socket that have the same fd
// ~ To have the same fd, the previous fd must be closed, which means it is removed from the epoll
// ~ and won't show up in our next call to epoll.Wait.
// ~ The old fd may be present in the buffer still, but lookup won't give a match, since it is removed
// ~ from the dictionary before it is closed. If we were accepting already, a new socket could match.
// - this also improves cache/cpu locality of the lookup
int *ptr = buffer;
lock (sockets)
{
for (int i = 0; i < numEvents; i++)
{
// Packed Non-Packed
// ------ ------
// 0:Events == Events
// 1:Int1 = Key [Padding]
// 2:Int2 = Key == Int1 = Key
// 3:~~~~~~~~~~ Int2 = Key
// ~~~~~~~~~~
int key = ptr[2];
ptr += 3 + notPacked;
TSocket tsocket;
if (sockets.TryGetValue(key & ~DupKeyMask, out tsocket))
{
var type = tsocket.Flags & SocketFlags.TypeMask;
if (type == SocketFlags.TypeClient)
{
bool read = (key & DupKeyMask) == 0;
if (read)
{
readableSockets.Add(tsocket);
}
else
{
writableSockets.Add(tsocket);
}
}
else
{
statAcceptEvents++;
acceptableSockets.Add(tsocket);
}
}
else if (key == pipeKey)
{
pipeReadable = true;
}
}
}
// handle accepts
statAcceptEvents += acceptableSockets.Count;
for (int i = 0; i < acceptableSockets.Count; i++)
{
statAccepts += HandleAccept(acceptableSockets[i], threadContext);
}
acceptableSockets.Clear();
// handle writes
statWriteEvents += writableSockets.Count;
for (int i = 0; i < writableSockets.Count; i++)
{
writableSockets[i].CompleteWritable();
}
writableSockets.Clear();
// handle reads
statReadEvents += readableSockets.Count;
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].CompleteReadable();
}
readableSockets.Clear();
// handle pipe
if (pipeReadable)
{
PosixResult result;
do
{
result = readEnd.TryReadByte();
if (result.Value == PipeStopSockets)
{
StopSockets(threadContext.Sockets);
}
else if (result.Value == PipeStopThread)
{
running = false;
}
} while (result);
pipeReadable = false;
}
// scheduled work
threadContext.DoScheduledWork();
} while (running);
threadContext.Logger.LogInformation($"Thread {_threadId}: Stats A/AE:{statAccepts}/{statAcceptEvents} RE:{statReadEvents} WE:{statWriteEvents}");
}
catch (Exception ex)
{
error = ex;
}
finally
{
// We are not using SafeHandles for epoll to increase performance.
// running == false when there are no more Sockets
// so we are sure there are no more epoll users.
threadContext?.Dispose();
CompleteStateChange(State.Stopped, error);
}
}
private static void AcceptOn(IPEndPoint endPoint, int cpuId, LinuxTransportOptions transportOptions, ThreadContext threadContext)
{
Socket acceptSocket = null;
int fd = 0;
int port = endPoint.Port;
SocketFlags flags = SocketFlags.TypeAccept;
try
{
bool ipv4 = endPoint.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork;
acceptSocket = Socket.Create(ipv4 ? AddressFamily.InterNetwork : AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp, blocking: false);
fd = acceptSocket.DangerousGetHandle().ToInt32();
if (!ipv4)
{
// Don't do mapped ipv4
acceptSocket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 1);
}
if (transportOptions.ReceiveOnIncomingCpu)
{
if (cpuId != -1)
{
if (!acceptSocket.TrySetSocketOption(SocketOptionLevel.Socket, SocketOptionName.IncomingCpu, cpuId))
{
threadContext.Logger.LogWarning($"Cannot enable nameof{SocketOptionName.IncomingCpu} for {endPoint}");
}
}
}
// Linux: allow bind during linger time
acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);
// Linux: allow concurrent binds and let the kernel do load-balancing
acceptSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReusePort, 1);
if (transportOptions.DeferAccept)
{
// Linux: wait up to 1 sec for data to arrive before accepting socket
acceptSocket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.DeferAccept, 1);
flags |= SocketFlags.DeferAccept;
}
acceptSocket.Bind(endPoint);
if (port == 0)
{
// When testing we want the OS to select a free port
port = acceptSocket.GetLocalIPAddress().Port;
}
acceptSocket.Listen(ListenBacklog);
}
catch
{
acceptSocket?.Dispose();
throw;
}
TSocket tsocket = null;
var sockets = threadContext.Sockets;
try
{
tsocket = new TSocket(threadContext)
{
Flags = flags,
Fd = fd,
Socket = acceptSocket
};
threadContext.AcceptSockets.Add(tsocket);
lock (sockets)
{
sockets.Add(tsocket.Fd, tsocket);
}
EPollInterop.EPollControl(threadContext.EPollFd,
EPollOperation.Add,
fd,
EPollEvents.Readable,
EPollData(fd));
}
catch
{
acceptSocket.Dispose();
threadContext.AcceptSockets.Remove(tsocket);
lock (sockets)
{
sockets.Remove(fd);
}
throw;
}
endPoint.Port = port;
}
public PosixResult TryControl(EPollOperation operation, SafeHandle fd, EPollEvents events, EPollData data)
{
return(EPollInterop.EPollControl(this, operation, fd, events, data.Long));
}