private void PerProcessRecvThread(int srcProcessID)
{
#if RECV_AFFINITY
PinnedThread pin = new PinnedThread(srcProcessID % 8);
#endif
Tracing.Trace("@RecvThread[{0:00}]", srcProcessID);
Logging.Info("Initializing per-process recv thread for {0}", srcProcessID);
this.startCommunicatingEvent.WaitOne();
Logging.Info("Starting per-process recv thread for {0}", srcProcessID);
Socket socket;
if (this.Controller.Configuration.DuplexSockets)
{
if (srcProcessID < this.localProcessID)
socket = this.connections[srcProcessID].RecvSocket;
else
socket = this.connections[srcProcessID].SendSocket;
}
else
{
socket = this.connections[srcProcessID].RecvSocket;
}
if (!this.Controller.Configuration.Nagling)
{
socket.NoDelay = true;
}
long numRecvs = 0;
int nextConnectionSequenceNumber = 0;
long recvBytesIn = 0;
long recvBytesOut = 0;
while (true)
{
SocketError errorCode;
ArraySegment<byte> recvSegment = this.connections[srcProcessID].RecvBufferSheaf.GetFreeSegment();
// Keep track of size of buffers passed to recv
recvBytesIn += recvSegment.Count;
int bytesRecvd = socket.Receive(recvSegment.Array, recvSegment.Offset, recvSegment.Count, SocketFlags.None, out errorCode);
// If the remote host shuts down the Socket connection with the Shutdown method,
// and all available data has been received, the Receive method will complete
// immediately and return zero bytes.
if (bytesRecvd == 0)
return;
recvBytesOut += bytesRecvd;
numRecvs++;
//Logging.Progress("Received {0} bytes from {1}", bytesRecvd, srcProcessID);
if (errorCode != SocketError.Success)
{
Tracing.Trace("*Socket Error {0}", errorCode);
this.HandleSocketError(srcProcessID, errorCode);
}
this.connections[srcProcessID].RecvBufferSheaf.OnBytesProduced(bytesRecvd);
foreach (SerializedMessage message in this.connections[srcProcessID].RecvBufferSheaf.ConsumeMessages(this.HeaderSerializer))
{
message.ConnectionSequenceNumber = nextConnectionSequenceNumber++;
this.connections[srcProcessID].recvStatistics[(int)RuntimeStatistic.RxNetMessages] += 1;
this.connections[srcProcessID].recvStatistics[(int)RuntimeStatistic.RxNetBytes] += message.Header.Length;
switch (message.Type)
{
case SerializedMessageType.Startup:
Logging.Progress("Received startup message from {0}", srcProcessID);
this.OnRecvBarrierMessageAndBlock(message.Header.ChannelID); // we put the barrier id in here
break;
case SerializedMessageType.Failure:
Logging.Error("Received graph failure message from {0}", srcProcessID);
this.Controller.GetInternalComputation(message.Header.ChannelID).Cancel(new Exception(string.Format("Received graph failure message from {0}", srcProcessID)));
break;
case SerializedMessageType.Shutdown:
Logging.Progress("Received shutdown message from {0}", srcProcessID);
Logging.Info("PerProcessRecvThread[{0}]: numRecvs {1} avgBytesIn {2} avgBytesOut {3}", srcProcessID, numRecvs, recvBytesIn / numRecvs, recvBytesOut / numRecvs);
this.shutdownRecvCountdown.Signal();
return;
case SerializedMessageType.Checkpoint:
// Pause the thread until we are informed that we can continue.
Logging.Progress("Got checkpoint message from process {0}", srcProcessID);
this.connections[srcProcessID].ReceivedCheckpointMessages++;
this.connections[srcProcessID].LastCheckpointSequenceNumber = message.ConnectionSequenceNumber;
this.connections[srcProcessID].CheckpointPauseEvent.Set();
Logging.Progress("Pausing recieve thread for process {0} because of {1}", srcProcessID, message.Type);
this.connections[srcProcessID].CheckpointResumeEvent.WaitOne();
Logging.Progress("Resuming receive thread for process {0} after checkpoint", srcProcessID);
break;
case SerializedMessageType.Data:
bool success = this.AttemptDelivery(message, srcProcessID);
Debug.Assert(success);
break;
default:
Logging.Progress("Received BAD msg type {0} from process {1}! ", message.Type, srcProcessID);
Debug.Assert(false);
break;
}
}
}
#if RECV_AFFINITY
pin.Dispose();
#endif
}
private void PerProcessSendThread(int destProcessID)
{
#if SEND_AFFINITY
//PinnedThread pin = new PinnedThread(0xC0UL);
PinnedThread pin = new PinnedThread(destProcessID % 8);
#endif
Tracing.Trace("@SendThread[{0:00}]", destProcessID);
// Connect to the destination socket.
while (true)
{
Logging.Info("Connect({0}, ..., {1})", this.connections[destProcessID].EndPoint, destProcessID);
this.connections[destProcessID].SendSocket = new Socket(this.connections[destProcessID].EndPoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
if (!this.Controller.Configuration.Nagling)
{
this.connections[destProcessID].SendSocket.NoDelay = true;
}
try
{
this.connections[destProcessID].SendSocket.Connect(this.connections[destProcessID].EndPoint);
break;
}
catch (SocketException se)
{
if (se.SocketErrorCode == SocketError.TimedOut || se.SocketErrorCode == SocketError.ConnectionRefused)
{
// Remote process hasn't started yet, so retry in a second.
this.connections[destProcessID].SendSocket.Dispose();
Thread.Sleep(1000); // FIXME: Better to use a timer if we do lots of these?
}
else
{
Logging.Fatal("Fatal error connecting to {0} {1}", this.connections[destProcessID].EndPoint, se.SocketErrorCode);
Logging.Fatal(se.Message);
System.Environment.Exit(-1);
}
}
}
SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes((int)NaiadProtocolOpcode.PeerConnect));
SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes(this.id));
SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes(this.localProcessID));
this.connections[destProcessID].Status = ConnectionStatus.Idle;
this.sendConnectionCountdown.Signal(1);
this.startCommunicatingEvent.WaitOne();
Socket socket;
if (this.Controller.Configuration.DuplexSockets)
{
if (destProcessID > this.localProcessID)
socket = this.connections[destProcessID].SendSocket;
else
socket = this.connections[destProcessID].RecvSocket;
}
else
{
socket = this.connections[destProcessID].SendSocket;
}
if (!this.Controller.Configuration.Nagling)
{
socket.NoDelay = true;
}
if (this.Controller.Configuration.KeepAlives)
{
socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
Microsoft.Research.Naiad.Utilities.Win32.SetKeepaliveOptions(socket.Handle);
}
long wakeupCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
bool shuttingDown = false;
while (true)
{
BufferSegment seg;
int length = 0;
while (this.connections[destProcessID].HighPrioritySegmentQueue.TryDequeue(out seg))
{
Debug.Assert(seg.Length > 0);
length += seg.Length;
shuttingDown = (seg.Type == SerializedMessageType.Shutdown);
SocketError errorCode = SendAllBytes(socket, seg.ToArraySegment());
if (errorCode != SocketError.Success)
{
Tracing.Trace("*Socket Error {0}", errorCode);
this.HandleSocketError(destProcessID, errorCode);
}
this.connections[destProcessID].ProgressSegmentsSent += 1;
this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxHighPriorityMessages] += 1;
this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxHighPriorityBytes] += seg.Length;
seg.Dispose();
}
while (this.connections[destProcessID].SegmentQueue.TryDequeue(out seg))
{
Debug.Assert(seg.Length > 0);
length += seg.Length;
shuttingDown = (seg.Type == SerializedMessageType.Shutdown);
SocketError errorCode = SendAllBytes(socket, seg.ToArraySegment());
if (errorCode != SocketError.Success)
{
Tracing.Trace("*Socket Error {0}", errorCode);
this.HandleSocketError(destProcessID, errorCode);
}
this.connections[destProcessID].DataSegmentsSent += 1;
this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxNormalPriorityMessages] += 1;
this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxNormalPriorityBytes] += seg.Length;
seg.Dispose();
}
if (shuttingDown)
break;
if (length == 0)
{
if (this.useBroadcastWakeup)
{
this.wakeUpEvent.Await(this.connections[destProcessID].SendEvent, wakeupCount + 1);
wakeupCount = this.wakeUpEvent.Read();
}
else
{
this.connections[destProcessID].SendEvent.WaitOne();
}
continue;
}
//this.connections[destProcessID].Status = shuttingDown ? ConnectionStatus.ShuttingDown : ConnectionStatus.Sending;
this.connections[destProcessID].BytesSent += length; // Progress + Data
//Logging.Progress("Sent {0} bytes to {1} (of {2})", bytesSent, destProcessID, length);
}
this.shutdownSendCountdown.Signal();
#if SEND_AFFINITY
pin.Dispose();
#endif
}