public unsafe void Send(long sequence, RIO_BUF *bufferSegmentDescriptor, bool flush)
{
var lockTaken = false;
var spinlock = new SpinLock();
try
{
spinlock.Enter(ref lockTaken);
var rioSendFlags = flush ? RIO_SEND_FLAGS.NONE : RIO_SEND_FLAGS.DEFER;
rioSendFlags |= RIO_SEND_FLAGS.DONT_NOTIFY;
if (!WinSock.Extensions.Send(_handle, bufferSegmentDescriptor, 1, rioSendFlags, sequence))
{
WinSock.ThrowLastWsaError();
}
}
finally
{
if (lockTaken)
{
spinlock.Exit();
}
}
}
private unsafe void FlushReceives()
{
if (!WinSock.Extensions.Receive(_handle, null, 0, RIO_RECEIVE_FLAGS.COMMIT_ONLY, 0))
{
WinSock.ThrowLastWsaError();
}
}
private void Listen()
{
if (WinSock.listen(_listeningSocket, WinSock.Consts.SOMAXCONN) != 0)
{
WinSock.ThrowLastWsaError();
}
}
private void ListenAndRunAcceptLoop(ManualResetEventSlim listeningSignal)
{
Thread.CurrentThread.Name = "Server Listening Worker";
if (!Bind(_listeningSocket, _listeningPort))
{
return;
}
ListeningPort = GetSocketPort();
Listen();
listeningSignal.Set();
while (_isListening)
{
var acceptSocket = WinSock.accept(_listeningSocket, IntPtr.Zero, IntPtr.Zero);
if (acceptSocket == (IntPtr)WinSock.Consts.INVALID_SOCKET)
{
var lastErrorCode = WinSock.WSAGetLastError();
if (lastErrorCode == WinSock.Consts.WSAEINTR)
{
break;
}
WinSock.ThrowLastWsaError();
continue;
}
InitClientSession(acceptSocket);
}
}
private unsafe void FlushSends()
{
if (!WinSock.Extensions.Send(_handle, null, 0, RIO_SEND_FLAGS.COMMIT_ONLY, 0))
{
WinSock.ThrowLastWsaError();
}
}
internal int TryGetCompletionResults(CancellationToken cancellationToken, RIO_RESULT *results, int maxCompletionResults)
{
var completionQueue = QueueHandle;
var spinwait = new SpinWait();
while (!cancellationToken.IsCancellationRequested)
{
var resultCount = WinSock.Extensions.DequeueCompletion(completionQueue, results, (uint)maxCompletionResults);
if (resultCount == 0)
{
spinwait.SpinOnce();
continue;
}
if (resultCount == WinSock.Consts.RIO_CORRUPT_CQ)
{
WinSock.ThrowLastWsaError();
break;
}
return((int)resultCount);
}
return(0);
}
public unsafe void Receive(RioBufferSegment *bufferSegment, int bufferSegmentId, bool flush)
{
var lockTaken = false;
var spinlock = new SpinLock();
try
{
spinlock.Enter(ref lockTaken);
var rioReceiveFlags = flush ? RIO_RECEIVE_FLAGS.NONE : RIO_RECEIVE_FLAGS.DEFER;
rioReceiveFlags |= RIO_RECEIVE_FLAGS.DONT_NOTIFY;
if (!WinSock.Extensions.Receive(_handle, bufferSegment->GetRioBufferDescriptor(), 1, rioReceiveFlags, bufferSegmentId))
{
WinSock.ThrowLastWsaError();
}
}
finally
{
if (lockTaken)
{
spinlock.Exit();
}
}
}
public unsafe void Send(long sequence, RIO_BUF *bufferSegmentDescriptor, bool flush)
{
var rioSendFlags = flush ? RIO_SEND_FLAGS.NONE : RIO_SEND_FLAGS.DEFER;
if (!WinSock.Extensions.Send(_handle, bufferSegmentDescriptor, 1, rioSendFlags, sequence))
{
WinSock.ThrowLastWsaError();
}
}
public unsafe void Receive(RioBufferSegment *bufferSegment, int bufferSegmentId, bool flush)
{
var rioSendFlags = flush ? RIO_RECEIVE_FLAGS.NONE : RIO_RECEIVE_FLAGS.DEFER;
if (!WinSock.Extensions.Receive(_handle, bufferSegment->GetRioBufferDescriptor(), 1, rioSendFlags, bufferSegmentId))
{
WinSock.ThrowLastWsaError();
}
}
public RioCompletionQueue(int size)
{
// we want to handle the polling manually, so we don't have to pass any RIO_NOTIFICATION_COMPLETION
QueueHandle = WinSock.Extensions.CreateCompletionQueue((uint)size);
if (QueueHandle.IsInvalid)
{
WinSock.ThrowLastWsaError();
}
}
public unsafe void Send(RioBuffer buffer)
{
var requestContextKey = new RioRequestContextKey(buffer.Id, RioRequestType.Send);
if (!WinSock.Extensions.Send(_handle, buffer.BufferDescriptor, 1, RIO_SEND_FLAGS.NONE, requestContextKey.ToRioRequestCorrelationId()))
{
WinSock.ThrowLastWsaError();
}
}
public UnmanagedRioBuffer(int bufferCount, int bufferLength)
{
// Example with:
// - Length = 2 entries
// - BufferLength = 1024 bytes
// - Entry of type T having
// - A 1 byte field
// - A 4 bytes field
// - A 4 bytes field
//
// |-- Padding --|- Entry reserved space --------------------------------------------------------|- Entry reserved space -------------------------------------------------------|-- Padding --|
// | 128 bytes | | | 128 bytes |
// | |- Entry #1 -----------------------------------------------| |- Entry #2 -----------------------------------------------| | |
// | | | | | | |
// | | |- RIO_BUF ------------------------------------|- Data - ... -------| |- RIO_BUF ------------------------------------|- Data - ... -------| |
// | | | | | | | | |
//[0] [128] [137] [145] [149] [153] [1177] [1186] [1194] [1198] [1202] | |
// | +-----------+----------------------+-----------+-----------+-------- ... -------+-----------+----------------------+-----------+-----------+-------- ... -------+ |
// | | Fields | BufferId | Offset | Length | Buffer segment | Fields | BufferId | Offset | Length | Buffer segment | |
// | | | IntPtr | int | int | bytes | | IntPtr | int | int | bytes | |
// | | 9 bytes | 8 bytes | 4 | 4 | 1024 | 9 bytes | 8 bytes | 4 | 4 | 1024 | |
// | +-----------+----------------------+-----------+-----------+-------- ... -------+-----------+----------------------+-----------+-----------+-------- ... -------+ |
// | | | 153 | 1024 | | 1202 | 1024 |
// | +---------------------^ +---------------------^
Length = bufferCount;
var entryLength = sizeof(T);
EntryReservedSpaceSize = entryLength + bufferLength;
// Padding is added at the start and the end of the buffer to prevent false sharing when using this UnmanagedRioBuffer
// as the underlying ring buffer memory of an unmanaged disruptor
var totalBufferLength = (uint)(_padding + bufferCount * EntryReservedSpaceSize + _padding);
const int allocationType = Kernel32.Consts.MEM_COMMIT | Kernel32.Consts.MEM_RESERVE;
_buffer = Kernel32.VirtualAlloc(IntPtr.Zero, totalBufferLength, allocationType, Kernel32.Consts.PAGE_READWRITE);
_bufferId = WinSock.Extensions.RegisterBuffer(_buffer, totalBufferLength);
if (_bufferId == WinSock.Consts.RIO_INVALID_BUFFERID)
{
WinSock.ThrowLastWsaError();
}
var currentEntry = FirstEntry = (T *)((byte *)_buffer.ToPointer() + _padding);
for (var i = 0; i < bufferCount; i++)
{
var bufferDescriptor = currentEntry->GetRioBufferDescriptor();
bufferDescriptor->BufferId = _bufferId;
bufferDescriptor->Offset = _padding + i * EntryReservedSpaceSize + sizeof(T);
bufferDescriptor->Length = bufferLength;
currentEntry = (T *)((byte *)(currentEntry + 1) + bufferLength);
}
}
public static RioRequestQueue Create(int correlationId, IntPtr socket, IRioCompletionQueue sendingCompletionQueue, uint maxOutstandingSends, IRioCompletionQueue receivingCompletionQueue, uint maxOutstandingReceives)
{
var requestQueue = WinSock.Extensions.CreateRequestQueue(socket, maxOutstandingReceives, 1, maxOutstandingSends, 1, receivingCompletionQueue.QueueHandle, sendingCompletionQueue.QueueHandle, correlationId);
if (requestQueue == IntPtr.Zero)
{
WinSock.ThrowLastWsaError();
}
return(new RioRequestQueue(requestQueue));
}
private unsafe int GetSocketPort()
{
SockaddrIn sockaddr;
var sockaddrSize = sizeof(SockaddrIn);
if (WinSock.getsockname(_listeningSocket, (byte *)&sockaddr, ref sockaddrSize) != 0)
{
WinSock.ThrowLastWsaError();
}
return(WinSock.ntohs(sockaddr.sin_port));
}
internal int TryGetCompletionResults(RIO_RESULT *results, int maxCompletionResults)
{
var completionQueue = QueueHandle;
var resultCount = WinSock.Extensions.DequeueCompletion(completionQueue, results, (uint)maxCompletionResults);
if (resultCount == WinSock.Consts.RIO_CORRUPT_CQ)
{
WinSock.ThrowLastWsaError();
}
return((int)resultCount);
}
private unsafe static void Connect(IntPtr socket, IPEndPoint ipEndPoint)
{
var endPointAddressBytes = ipEndPoint.Address.GetAddressBytes();
var inAddress = new InAddr(endPointAddressBytes);
var sa = new SockaddrIn
{
sin_family = AddressFamilies.AF_INET,
sin_port = WinSock.htons((ushort)ipEndPoint.Port),
sin_addr = inAddress
};
var errorCode = WinSock.connect(socket, ref sa, sizeof(SockaddrIn));
if (errorCode == WinSock.Consts.SOCKET_ERROR)
{
WinSock.ThrowLastWsaError();
}
}
private RioBufferManager(int bufferCount, int bufferLength)
{
if (bufferLength <= 0)
{
throw new ArgumentOutOfRangeException(nameof(bufferLength));
}
if (bufferCount <= 0)
{
throw new ArgumentOutOfRangeException(nameof(bufferCount));
}
var underlyingBufferLength = (uint)(bufferLength * bufferCount);
_underlyingBuffer = Kernel32.VirtualAlloc(IntPtr.Zero, underlyingBufferLength, Kernel32.Consts.MEM_COMMIT | Kernel32.Consts.MEM_RESERVE, Kernel32.Consts.PAGE_READWRITE);
_bufferDescriptors = Kernel32.VirtualAlloc(IntPtr.Zero, (uint)(Marshal.SizeOf <RIO_BUF>() * bufferCount), Kernel32.Consts.MEM_COMMIT | Kernel32.Consts.MEM_RESERVE, Kernel32.Consts.PAGE_READWRITE);
_bufferId = WinSock.Extensions.RegisterBuffer(_underlyingBuffer, underlyingBufferLength);
if (_bufferId == WinSock.Consts.RIO_INVALID_BUFFERID)
{
WinSock.ThrowLastWsaError();
}
_buffers = new RioBuffer[bufferCount];
var dataPointer = (byte *)_underlyingBuffer.ToPointer();
var segmentPointer = (RIO_BUF *)_bufferDescriptors.ToPointer();
for (var segmentIndex = 0; segmentIndex < bufferCount; segmentIndex++)
{
segmentPointer->BufferId = _bufferId;
segmentPointer->Offset = segmentIndex * bufferLength;
segmentPointer->Length = bufferLength;
var segment = new RioBuffer(segmentIndex, dataPointer, segmentPointer, bufferLength);
_buffers[segmentIndex] = segment;
_freeBufferIds.Push(segment.Id);
segmentPointer++;
dataPointer += bufferLength;
}
}
private static unsafe IntPtr CreateListeningSocket()
{
var socketFlags = SocketFlags.WSA_FLAG_REGISTERED_IO | SocketFlags.WSA_FLAG_OVERLAPPED;
var listeningSocket = WinSock.WSASocket(AddressFamilies.AF_INET, SocketType.SOCK_STREAM, Protocol.IPPROTO_TCP, IntPtr.Zero, 0, socketFlags);
if (listeningSocket == (IntPtr)WinSock.Consts.INVALID_SOCKET)
{
WinSock.ThrowLastWsaError();
return(IntPtr.Zero);
}
var tcpNoDelay = -1;
WinSock.setsockopt(listeningSocket, WinSock.Consts.IPPROTO_TCP, WinSock.Consts.TCP_NODELAY, (char *)&tcpNoDelay, sizeof(int));
var reuseAddr = 1;
WinSock.setsockopt(listeningSocket, WinSock.Consts.SOL_SOCKET, WinSock.Consts.SO_REUSEADDR, (char *)&reuseAddr, sizeof(int));
return(listeningSocket);
}
private void ListenAndRunAcceptLoop(ManualResetEventSlim listeningSignal)
{
Thread.CurrentThread.Name = "Server Listening Worker";
if (!Bind(_listeningSocket, _configuration.ListeningPort))
{
return;
}
ListeningPort = GetSocketPort();
Listen();
listeningSignal.Set();
while (_isListening)
{
var acceptSocket = WinSock.accept(_listeningSocket, IntPtr.Zero, IntPtr.Zero);
if (acceptSocket == (IntPtr)WinSock.Consts.INVALID_SOCKET)
{
var lastErrorCode = WinSock.WSAGetLastError();
if (lastErrorCode == WinSock.Consts.WSAEINTR)
{
break;
}
WinSock.ThrowLastWsaError();
continue;
}
var clientSession = _sessionManager.Acquire();
clientSession.Closed += OnClientSessionClosed;
clientSession.MessageReceived += OnClientSessionMessageReceived;
clientSession.Open(acceptSocket);
clientSession.InitiateReceiving();
ClientConnected(clientSession.Id);
}
}
public unsafe void FlushSends()
{
var lockTaken = false;
var spinlock = new SpinLock();
try
{
spinlock.Enter(ref lockTaken);
if (!WinSock.Extensions.Send(_handle, null, 0, RIO_SEND_FLAGS.COMMIT_ONLY, 0))
{
WinSock.ThrowLastWsaError();
}
}
finally
{
if (lockTaken)
{
spinlock.Exit();
}
}
}
private void Listen()
{
Thread.CurrentThread.Name = "Server Listening Worker";
if (!Bind(_listeningSocket, _configuration.ListeningPort))
{
return;
}
if (WinSock.listen(_listeningSocket, WinSock.Consts.SOMAXCONN) == WinSock.Consts.SOCKET_ERROR)
{
return;
}
while (_isListening)
{
var acceptSocket = WinSock.accept(_listeningSocket, IntPtr.Zero, IntPtr.Zero);
if (acceptSocket == (IntPtr)WinSock.Consts.INVALID_SOCKET)
{
var lastErrorCode = WinSock.WSAGetLastError();
if (lastErrorCode == WinSock.Consts.WSAEINTR)
{
break;
}
WinSock.ThrowLastWsaError();
continue;
}
var clientSession = _sessionManager.Acquire();
clientSession.Closed += OnClientSessionClosed;
clientSession.MessageReceived += OnClientSessionMessageReceived;
clientSession.Open(acceptSocket);
clientSession.InitiateReceiving();
ClientConnected(clientSession.Id);
}
}
private static unsafe bool Bind(IntPtr listeningSocket, int listeningPort)
{
var endPointAddressBytes = IPAddress.Any.GetAddressBytes();
var inAddress = new InAddr(endPointAddressBytes);
var sa = new SockaddrIn
{
sin_family = AddressFamilies.AF_INET,
sin_port = WinSock.htons((ushort)listeningPort),
sin_addr = inAddress
};
var errorCode = WinSock.bind(listeningSocket, ref sa, sizeof(SockaddrIn));
if (errorCode == WinSock.Consts.SOCKET_ERROR)
{
WinSock.ThrowLastWsaError();
return(false);
}
return(true);
}
