public unsafe RioThreadPool(RegisteredIO rio, IntPtr socket, CancellationToken token)
{
_socket = socket;
_rio = rio;
_token = token;
// Count non-HT cores only
var procCount = CpuInfo.PhysicalCoreCount;
// RSS only supports up to 16 cores
_maxThreads = procCount > 16 ? 16 : procCount;
_rioThreads = new RioThread[_maxThreads];
for (var i = 0; i < _rioThreads.Length; i++)
{
IntPtr completionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, IntPtr.Zero, 0, 0);
if (completionPort == IntPtr.Zero)
{
var error = GetLastError();
RioImports.WSACleanup();
throw new Exception($"ERROR: CreateIoCompletionPort returned {error}");
}
var completionMethod = new NotificationCompletion
{
Type = NotificationCompletionType.IocpCompletion,
Iocp = new NotificationCompletionIocp
{
IocpHandle = completionPort,
QueueCorrelation = (ulong) i,
Overlapped = (NativeOverlapped*) (-1) // nativeOverlapped
}
};
IntPtr completionQueue = _rio.RioCreateCompletionQueue(RioTcpServer.MaxOutsandingCompletionsPerThread,
completionMethod);
if (completionQueue == IntPtr.Zero)
{
var error = RioImports.WSAGetLastError();
RioImports.WSACleanup();
throw new Exception($"ERROR: RioCreateCompletionQueue returned {error}");
}
var thread = new RioThread(i, _token, completionPort, completionQueue, rio);
_rioThreads[i] = thread;
}
for (var i = 0; i < _rioThreads.Length; i++)
{
var thread = _rioThreads[i];
thread.Start();
}
}
internal RioTcpConnection(IntPtr socket, long connectionId, IntPtr requestQueue, RioThread rioThread, RegisteredIO rio)
{
_socket = socket;
_connectionId = connectionId;
_rio = rio;
_rioThread = rioThread;
_input = rioThread.PipelineFactory.Create();
_output = rioThread.PipelineFactory.Create();
_requestQueue = requestQueue;
rioThread.AddConnection(connectionId, this);
ProcessReceives();
_sendTask = ProcessSends();
}
public RioThread(int id, CancellationToken token, IntPtr completionPort, IntPtr completionQueue, RegisteredIO rio)
{
_id = id;
_rio = rio;
_token = token;
if (CpuInfo.LogicalProcessorCount > CpuInfo.PhysicalCoreCount)
{
_completionThread = new Thread(RunLogicalCompletions)
{
Name = $"RIO Completion Thread {id:00}",
IsBackground = true
};
_notifyBatches = new Queue<NotifyBatch>(16);
_notify = new object();
_notifyThread = new Thread(RunNotifies)
{
Name = $"RIO Notify Thread {id:00}",
IsBackground = true
};
_processedBatches = new Queue<NotifyBatch>(16);
}
else
{
_completionThread = new Thread(RunPhysicalCompletions)
{
Name = $"RIO Completion Thread {id:00}",
IsBackground = true
};
}
_completionPort = completionPort;
_completionQueue = completionQueue;
}
public unsafe static RegisteredIO Initalize(IntPtr socket)
{
UInt32 dwBytes = 0;
RioExtensionFunctionTable rio = new RioExtensionFunctionTable();
Guid rioFunctionsTableId = new Guid("8509e081-96dd-4005-b165-9e2ee8c79e3f");
int True = -1;
int result = setsockopt(socket, IPPROTO_TCP, TcpNodelay, (char*)&True, 4);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: setsockopt TCP_NODELAY returned {error}");
}
result = WSAIoctlGeneral(socket, SioLoopbackFastPath,
&True, 4, null, 0,
out dwBytes, IntPtr.Zero, IntPtr.Zero);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: WSAIoctl SIO_LOOPBACK_FAST_PATH returned {error}");
}
result = WSAIoctl(socket, SioGetMultipleExtensionFunctionPointer,
ref rioFunctionsTableId, 16, ref rio,
sizeof(RioExtensionFunctionTable),
out dwBytes, IntPtr.Zero, IntPtr.Zero);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: RIOInitalize returned {error}");
}
var rioFunctions = new RegisteredIO();
rioFunctions.RioRegisterBuffer = Marshal.GetDelegateForFunctionPointer<RioRegisterBuffer>(rio.RIORegisterBuffer);
rioFunctions.RioCreateCompletionQueue = Marshal.GetDelegateForFunctionPointer<RioCreateCompletionQueue>(rio.RIOCreateCompletionQueue);
rioFunctions.RioCreateRequestQueue = Marshal.GetDelegateForFunctionPointer<RioCreateRequestQueue>(rio.RIOCreateRequestQueue);
rioFunctions.Notify = Marshal.GetDelegateForFunctionPointer<RioNotify>(rio.RIONotify);
rioFunctions.DequeueCompletion = Marshal.GetDelegateForFunctionPointer<RioDequeueCompletion>(rio.RIODequeueCompletion);
rioFunctions.RioReceive = Marshal.GetDelegateForFunctionPointer<RioReceive>(rio.RIOReceive);
rioFunctions.Send = Marshal.GetDelegateForFunctionPointer<RioSend>(rio.RIOSend);
rioFunctions.CloseCompletionQueue = Marshal.GetDelegateForFunctionPointer<RioCloseCompletionQueue>(rio.RIOCloseCompletionQueue);
rioFunctions.DeregisterBuffer = Marshal.GetDelegateForFunctionPointer<RioDeregisterBuffer>(rio.RIODeregisterBuffer);
rioFunctions.ResizeCompletionQueue = Marshal.GetDelegateForFunctionPointer<RioResizeCompletionQueue>(rio.RIOResizeCompletionQueue);
rioFunctions.ResizeRequestQueue = Marshal.GetDelegateForFunctionPointer<RioResizeRequestQueue>(rio.RIOResizeRequestQueue);
return rioFunctions;
}
public unsafe static RegisteredIO Initalize(IntPtr socket)
{
UInt32 dwBytes = 0;
RioExtensionFunctionTable rio = new RioExtensionFunctionTable();
Guid rioFunctionsTableId = new Guid("8509e081-96dd-4005-b165-9e2ee8c79e3f");
int True = -1;
int result = setsockopt(socket, IPPROTO_TCP, TcpNodelay, (char *)&True, 4);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: setsockopt TCP_NODELAY returned {error}");
}
result = WSAIoctlGeneral(socket, SioLoopbackFastPath,
&True, 4, null, 0,
out dwBytes, IntPtr.Zero, IntPtr.Zero);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: WSAIoctl SIO_LOOPBACK_FAST_PATH returned {error}");
}
result = WSAIoctl(socket, SioGetMultipleExtensionFunctionPointer,
ref rioFunctionsTableId, 16, ref rio,
sizeof(RioExtensionFunctionTable),
out dwBytes, IntPtr.Zero, IntPtr.Zero);
if (result != 0)
{
var error = WSAGetLastError();
WSACleanup();
throw new Exception($"ERROR: RIOInitalize returned {error}");
}
var rioFunctions = new RegisteredIO();
rioFunctions.RioRegisterBuffer = Marshal.GetDelegateForFunctionPointer <RioRegisterBuffer>(rio.RIORegisterBuffer);
rioFunctions.RioCreateCompletionQueue = Marshal.GetDelegateForFunctionPointer <RioCreateCompletionQueue>(rio.RIOCreateCompletionQueue);
rioFunctions.RioCreateRequestQueue = Marshal.GetDelegateForFunctionPointer <RioCreateRequestQueue>(rio.RIOCreateRequestQueue);
rioFunctions.Notify = Marshal.GetDelegateForFunctionPointer <RioNotify>(rio.RIONotify);
rioFunctions.DequeueCompletion = Marshal.GetDelegateForFunctionPointer <RioDequeueCompletion>(rio.RIODequeueCompletion);
rioFunctions.RioReceive = Marshal.GetDelegateForFunctionPointer <RioReceive>(rio.RIOReceive);
rioFunctions.Send = Marshal.GetDelegateForFunctionPointer <RioSend>(rio.RIOSend);
rioFunctions.CloseCompletionQueue = Marshal.GetDelegateForFunctionPointer <RioCloseCompletionQueue>(rio.RIOCloseCompletionQueue);
rioFunctions.DeregisterBuffer = Marshal.GetDelegateForFunctionPointer <RioDeregisterBuffer>(rio.RIODeregisterBuffer);
rioFunctions.ResizeCompletionQueue = Marshal.GetDelegateForFunctionPointer <RioResizeCompletionQueue>(rio.RIOResizeCompletionQueue);
rioFunctions.ResizeRequestQueue = Marshal.GetDelegateForFunctionPointer <RioResizeRequestQueue>(rio.RIOResizeRequestQueue);
return(rioFunctions);
}