/// <summary>
/// Send data over RDMA
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public NtStatus SendData(byte[] data)
{
RdmaSegment sge = new RdmaSegment();
sge.Length = (uint)data.LongLength;
sge.MemoryHandler = 0;
// register memory
rdmaAdapter.RegisterMemory(sge.Length, out sge.MemoryHandler);
// Write data to memory
RdmaEndpoint.WriteToMemory(sge.MemoryHandler, data);
// send
UInt64 resultId;
NtStatus status = (NtStatus)rdmaEndpoint.Send(new RdmaSegment[] { sge }, out resultId);
if (status != NtStatus.STATUS_SUCCESS)
{
return(status);
}
// send request has been submitted
requestCount++;
rdmaNotificationSemaphore.Release();
// get the notification
SmbdRequestResult item = GetRequestResult(new TimeSpan(0, 0, 5), RequestType.Send);
this.LogEvent(string.Format("Send data with result id: 0x{0:X}. And get notification with id: 0x{1:X}, status: {2}",
resultId,
item.ResultId,
item.ResultInfo.Status
));
// deregister memory
rdmaAdapter.DeregisterMemory(sge.MemoryHandler);
return((NtStatus)item.ResultInfo.Status);
}
/// <summary>
/// Receive data over RDMA
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public NtStatus ReceiveData(TimeSpan timeout, out byte[] data)
{
SmbdRequestResult item = GetRequestResult(timeout, RequestType.Receive);
NtStatus status = (NtStatus)item.ResultInfo.Status;
if (status != NtStatus.STATUS_SUCCESS)
{
data = null;
return(status);
}
this.LogEvent(string.Format("Receive {0} bytes from entry 0x{1:X}",
item.ResultInfo.BytesTransferred,
item.EntryIndex));
data = new byte[item.ResultInfo.BytesTransferred];
status = (NtStatus)RdmaEndpoint.ReadFromMemory(
this.receiveEntries[item.EntryIndex].Segment.MemoryHandler,
data);
// reset
this.receiveEntries[receiveIndex].IsOccupied = false;
return(status);
}
/// <summary>
/// Notify callback
/// </summary>
private void NotifyCallback(Object stateInfo)
{
// notify main thread, NotifyCallback is started.
threadStartSemaphore.Release();
LogEvent("NotifyCallback is started...");
while (true)
{
// Wait for submitting request or disconnection
rdmaNotificationSemaphore.WaitOne();
rdmaNotificationSemaphore.Release();
if (requestCount <= 0)
{ // no request in the list, the semaphore is released from disconnection
//threadStopSemaphore.Release();
LogEvent("NotifyCallback has stopped.");
return;
}
NtStatus status = (NtStatus)rdmaCompletionQueue.Notify();
LogEvent(string.Format("NotifyCallback: get notification with status {0}.", status));
if (status != NtStatus.STATUS_SUCCESS)
{
continue;
}
while (true)
{
UInt64 resultId;
RdmaNetworkDirectResult ndResult;
UInt64 size = rdmaCompletionQueue.GetResult(out resultId, out ndResult);
if (size == 0)
{
break;
}
// WaitOne for reduce the request count
rdmaNotificationSemaphore.WaitOne();
requestCount--;
RequestType type = RequestType.None;
int segmentIndex = -1;
#region Receive and invalid memory window
lock (locker)
{
foreach (SmbdRequest request in receiveRequestList)
{
if (request.ResultId == resultId)
{
type = request.Type;
segmentIndex = request.EntryIndex;
// get the result
SmbdRequestResult requestResultItem = new SmbdRequestResult();
requestResultItem.EntryIndex = request.EntryIndex;
requestResultItem.ResultInfo = ndResult;
requestResultItem.ResultId = request.ResultId;
switch (request.Type)
{
case RequestType.Receive:
receiveRequestResult.Enqueue(requestResultItem);
ReceivePostedCount--;
break;
case RequestType.Invalid:
for (int i = 0; i < memoryWindowList.Count; ++i)
{
if (memoryWindowList[i].InvalidResultId == request.ResultId)
{
memoryWindowList[i].IsValid = false;
}
}
break;
}
receiveRequestList.Remove(request);
break;
}
}
}
#endregion
if (type == RequestType.None)
{
otherRequestResult.Enqueue(
new SmbdRequestResult()
{
ResultId = resultId,
ResultInfo = ndResult
});
}
// log
this.LogEvent(
string.Format(
"1 operation {0} has been finished with result {1} and result Id: {2:X};" +
" Bytes of data transferred is {3}; Segment Index is {4}; Count of work items is {5}",
type,
(NtStatus)ndResult.Status,
resultId,
ndResult.BytesTransferred,
segmentIndex,
this.receiveRequestList.Count));
}
}
}
/// <summary>
/// Register memory windows
/// </summary>
/// <param name="size">Size of memory to register</param>
/// <param name="flag"></param>
/// <param name="reversed">if it is true, little-endian and big-endian will be reversed in bufferDescriptor</param>
/// <param name="bufferDescriptor">Buffer Descriptor point to memory windows</param>
/// <returns></returns>
public NtStatus RegisterMemoryWindow(
uint size,
RdmaOperationReadWriteFlag flag,
bool reversed,
out RdmaBufferDescriptorV1 bufferDescriptor)
{
bufferDescriptor = new RdmaBufferDescriptorV1();
// regiser the buffer
SmbdMemoryWindow memoryWindow = new SmbdMemoryWindow();
memoryWindow.IsValid = true;
NtStatus status = (NtStatus)rdmaAdapter.RegisterMemory(size, out memoryWindow.MemoryHandlerId);
if (status != NtStatus.STATUS_SUCCESS)
{
return(status);
}
// create memory window
SmbdRequest invalidRequest = new SmbdRequest();
invalidRequest.Type = RequestType.Invalid;
status = (NtStatus)rdmaAdapter.CreateMemoryWindow(invalidRequest.ResultId,
out memoryWindow.RdmaMW);
memoryWindow.InvalidResultId = invalidRequest.ResultId;
if (status != NtStatus.STATUS_SUCCESS)
{
return(status);
}
// invalid notification request has been submitted
requestCount++;
rdmaNotificationSemaphore.Release();
// bind
UInt64 resultId;
status = (NtStatus)rdmaEndpoint.Bind(
memoryWindow.MemoryHandlerId,
memoryWindow.RdmaMW,
flag,
reversed,
out memoryWindow.BufferDescriptor, out resultId);
if (status != NtStatus.STATUS_SUCCESS)
{
return(status);
}
lock (locker)
{
this.receiveRequestList.Add(invalidRequest);
}
SmbdRequestResult requestResult = GetRequestResult(new TimeSpan(0, 0, 5), RequestType.Bind);
this.LogEvent(string.Format("Bind memory window with result id: {0}. And get notification with id: {1}, status: {2}",
resultId,
requestResult.ResultId,
requestResult.ResultInfo.Status
));
status = (NtStatus)requestResult.ResultInfo.Status;
if (status != NtStatus.STATUS_SUCCESS)
{
return(status);
}
this.memoryWindowList.Add(memoryWindow);
bufferDescriptor = memoryWindow.BufferDescriptor;
return(status);
}
/// <summary>
/// Notify callback
/// </summary>
private void NotifyCallback(Object stateInfo)
{
// notify main thread, NotifyCallback is started.
threadStartSemaphore.Release();
LogEvent("NotifyCallback is started...");
while (true)
{
// Wait for submitting request or disconnection
rdmaNotificationSemaphore.WaitOne();
rdmaNotificationSemaphore.Release();
if (requestCount <= 0)
{ // no request in the list, the semaphore is released from disconnection
//threadStopSemaphore.Release();
LogEvent("NotifyCallback has stopped.");
return;
}
NtStatus status = (NtStatus)rdmaCompletionQueue.Notify();
LogEvent(string.Format("NotifyCallback: get notification with status {0}.", status));
if (status != NtStatus.STATUS_SUCCESS)
{
continue;
}
while (true)
{
UInt64 resultId;
RdmaNetworkDirectResult ndResult;
UInt64 size = rdmaCompletionQueue.GetResult(out resultId, out ndResult);
if (size == 0)
{
break;
}
// WaitOne for reduce the request count
rdmaNotificationSemaphore.WaitOne();
requestCount--;
RequestType type = RequestType.None;
int segmentIndex = -1;
#region Receive and invalid memory window
lock (locker)
{
foreach (SmbdRequest request in receiveRequestList)
{
if (request.ResultId == resultId)
{
type = request.Type;
segmentIndex = request.EntryIndex;
// get the result
SmbdRequestResult requestResultItem = new SmbdRequestResult();
requestResultItem.EntryIndex = request.EntryIndex;
requestResultItem.ResultInfo = ndResult;
requestResultItem.ResultId = request.ResultId;
switch (request.Type)
{
case RequestType.Receive:
receiveRequestResult.Enqueue(requestResultItem);
ReceivePostedCount--;
break;
case RequestType.Invalid:
for (int i = 0; i < memoryWindowList.Count; ++i)
{
if (memoryWindowList[i].InvalidResultId == request.ResultId)
{
memoryWindowList[i].IsValid = false;
}
}
break;
}
receiveRequestList.Remove(request);
break;
}
}
}
#endregion
if (type == RequestType.None)
{
otherRequestResult.Enqueue(
new SmbdRequestResult()
{
ResultId = resultId,
ResultInfo = ndResult
});
}
// log
this.LogEvent(
string.Format(
"1 operation {0} has been finished with result {1} and result Id: {2:X};" +
" Bytes of data transferred is {3}; Segment Index is {4}; Count of work items is {5}",
type,
(NtStatus)ndResult.Status,
resultId,
ndResult.BytesTransferred,
segmentIndex,
this.receiveRequestList.Count));
}
}
}
