Пример #1
0
        private void PerProcessRecvThread(int srcProcessID)
        {
#if RECV_AFFINITY
            PinnedThread pin = new PinnedThread(srcProcessID % 8);
#endif
            Tracing.Trace("@RecvThread[{0:00}]", srcProcessID);
            Logging.Info("Initializing per-process recv thread for {0}", srcProcessID);

            this.startCommunicatingEvent.WaitOne();

            Logging.Info("Starting per-process recv thread for {0}", srcProcessID);

            Socket socket;

            if (this.Controller.Configuration.DuplexSockets)
            {
                if (srcProcessID < this.localProcessID)
                    socket = this.connections[srcProcessID].RecvSocket;
                else
                    socket = this.connections[srcProcessID].SendSocket;
            }
            else
            {
                socket = this.connections[srcProcessID].RecvSocket;
            }

            if (!this.Controller.Configuration.Nagling)
            {
                socket.NoDelay = true;
            }

            long numRecvs = 0;


            int nextConnectionSequenceNumber = 0;

            long recvBytesIn = 0;
            long recvBytesOut = 0;

            while (true)
            {
                SocketError errorCode;


                ArraySegment<byte> recvSegment = this.connections[srcProcessID].RecvBufferSheaf.GetFreeSegment();

                // Keep track of size of buffers passed to recv
                
                recvBytesIn += recvSegment.Count;

                int bytesRecvd = socket.Receive(recvSegment.Array, recvSegment.Offset, recvSegment.Count, SocketFlags.None, out errorCode);
                
                // If the remote host shuts down the Socket connection with the Shutdown method,
                // and all available data has been received, the Receive method will complete 
                // immediately and return zero bytes.
                if (bytesRecvd == 0)
                    return;

                recvBytesOut += bytesRecvd;
                numRecvs++;

                //Logging.Progress("Received {0} bytes from {1}", bytesRecvd, srcProcessID);
                if (errorCode != SocketError.Success)
                {
                    Tracing.Trace("*Socket Error {0}", errorCode);

                    this.HandleSocketError(srcProcessID, errorCode);
                }
                this.connections[srcProcessID].RecvBufferSheaf.OnBytesProduced(bytesRecvd);

                foreach (SerializedMessage message in this.connections[srcProcessID].RecvBufferSheaf.ConsumeMessages(this.HeaderSerializer))
                {
                    message.ConnectionSequenceNumber = nextConnectionSequenceNumber++;

                    this.connections[srcProcessID].recvStatistics[(int)RuntimeStatistic.RxNetMessages] += 1;
                    this.connections[srcProcessID].recvStatistics[(int)RuntimeStatistic.RxNetBytes] += message.Header.Length;

                    switch (message.Type)
                    {
                        case SerializedMessageType.Startup:
                            Logging.Progress("Received startup message from {0}", srcProcessID);
                            this.OnRecvBarrierMessageAndBlock(message.Header.ChannelID);    // we put the barrier id in here
                            break;
                        case SerializedMessageType.Failure:
                            Logging.Error("Received graph failure message from {0}", srcProcessID);
                            this.Controller.GetInternalComputation(message.Header.ChannelID).Cancel(new Exception(string.Format("Received graph failure message from {0}", srcProcessID)));
                            break;
                        case SerializedMessageType.Shutdown:
                            Logging.Progress("Received shutdown message from {0}", srcProcessID);
                            Logging.Info("PerProcessRecvThread[{0}]: numRecvs {1} avgBytesIn {2} avgBytesOut {3}", srcProcessID, numRecvs, recvBytesIn / numRecvs, recvBytesOut / numRecvs);
                            this.shutdownRecvCountdown.Signal();
                            return;
                        case SerializedMessageType.Checkpoint:
                            // Pause the thread until we are informed that we can continue.
                            Logging.Progress("Got checkpoint message from process {0}", srcProcessID);
                            this.connections[srcProcessID].ReceivedCheckpointMessages++;
                            this.connections[srcProcessID].LastCheckpointSequenceNumber = message.ConnectionSequenceNumber;
                            this.connections[srcProcessID].CheckpointPauseEvent.Set();

                            Logging.Progress("Pausing recieve thread for process {0} because of {1}", srcProcessID, message.Type);
                            this.connections[srcProcessID].CheckpointResumeEvent.WaitOne();
                            Logging.Progress("Resuming receive thread for process {0} after checkpoint", srcProcessID);

                            break;
                        case SerializedMessageType.Data:
                            bool success = this.AttemptDelivery(message, srcProcessID);
                            Debug.Assert(success);
                            break;
                        default:
                            Logging.Progress("Received BAD msg type {0} from process {1}! ", message.Type, srcProcessID);
                            Debug.Assert(false);
                            break;
                    }
                }
            }
#if RECV_AFFINITY
            pin.Dispose();
#endif
        }
Пример #2
0
        private void PerProcessSendThread(int destProcessID)
        {
#if SEND_AFFINITY
            //PinnedThread pin = new PinnedThread(0xC0UL);
            PinnedThread pin = new PinnedThread(destProcessID % 8);
#endif
            Tracing.Trace("@SendThread[{0:00}]", destProcessID);
            // Connect to the destination socket.
            while (true) 
            {
                Logging.Info("Connect({0}, ..., {1})", this.connections[destProcessID].EndPoint, destProcessID);

                this.connections[destProcessID].SendSocket = new Socket(this.connections[destProcessID].EndPoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
                if (!this.Controller.Configuration.Nagling)
                {
                    this.connections[destProcessID].SendSocket.NoDelay = true;
                }
                
                try
                {
                    this.connections[destProcessID].SendSocket.Connect(this.connections[destProcessID].EndPoint);
                    break;
                }
                catch (SocketException se)
                {
                    if (se.SocketErrorCode == SocketError.TimedOut || se.SocketErrorCode == SocketError.ConnectionRefused)
                    {
                        // Remote process hasn't started yet, so retry in a second.
                        this.connections[destProcessID].SendSocket.Dispose();
                        Thread.Sleep(1000); // FIXME: Better to use a timer if we do lots of these?
                    }
                    else
                    {
                        Logging.Fatal("Fatal error connecting to {0} {1}", this.connections[destProcessID].EndPoint, se.SocketErrorCode);
                        Logging.Fatal(se.Message);
                        System.Environment.Exit(-1);
                    }
                }
            }

            SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes((int)NaiadProtocolOpcode.PeerConnect));
            SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes(this.id));
            SendAllBytes(this.connections[destProcessID].SendSocket, BitConverter.GetBytes(this.localProcessID));

            this.connections[destProcessID].Status = ConnectionStatus.Idle;

            this.sendConnectionCountdown.Signal(1);

            this.startCommunicatingEvent.WaitOne();
            Socket socket;

            if (this.Controller.Configuration.DuplexSockets)
            {
                if (destProcessID > this.localProcessID)
                    socket = this.connections[destProcessID].SendSocket;
                else
                    socket = this.connections[destProcessID].RecvSocket;
            }
            else
            {
                socket = this.connections[destProcessID].SendSocket;
            }

            if (!this.Controller.Configuration.Nagling)
            {
                socket.NoDelay = true;
            }
            
            if (this.Controller.Configuration.KeepAlives)
            {
                socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
                Microsoft.Research.Naiad.Utilities.Win32.SetKeepaliveOptions(socket.Handle);
            } 

            long wakeupCount = 0;

            Stopwatch sw = new Stopwatch();
            sw.Start();
            

            bool shuttingDown = false;
            while (true)
            {
                BufferSegment seg;
                int length = 0;
                
                while (this.connections[destProcessID].HighPrioritySegmentQueue.TryDequeue(out seg))
                {
                    Debug.Assert(seg.Length > 0); 
                    
                    length += seg.Length;
                    shuttingDown = (seg.Type == SerializedMessageType.Shutdown);

                    SocketError errorCode = SendAllBytes(socket, seg.ToArraySegment());
                    if (errorCode != SocketError.Success)
                    {
                        Tracing.Trace("*Socket Error {0}", errorCode);
                        this.HandleSocketError(destProcessID, errorCode);
                    }

                    this.connections[destProcessID].ProgressSegmentsSent += 1;
                    this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxHighPriorityMessages] += 1;
                    this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxHighPriorityBytes] += seg.Length;
                    
                    seg.Dispose();
                }

                while (this.connections[destProcessID].SegmentQueue.TryDequeue(out seg))
                {
                    Debug.Assert(seg.Length > 0);

                    length += seg.Length;
                    shuttingDown = (seg.Type == SerializedMessageType.Shutdown);

                    SocketError errorCode = SendAllBytes(socket, seg.ToArraySegment());
                    if (errorCode != SocketError.Success)
                    {
                        Tracing.Trace("*Socket Error {0}", errorCode);
                        this.HandleSocketError(destProcessID, errorCode);
                    }

                    this.connections[destProcessID].DataSegmentsSent += 1;
                    this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxNormalPriorityMessages] += 1;
                    this.connections[destProcessID].sendStatistics[(int)RuntimeStatistic.TxNormalPriorityBytes] += seg.Length;

                    seg.Dispose();
                }

                if (shuttingDown)
                    break;
                if (length == 0)
                {
                    if (this.useBroadcastWakeup)
                    {
                        this.wakeUpEvent.Await(this.connections[destProcessID].SendEvent, wakeupCount + 1);
                        wakeupCount = this.wakeUpEvent.Read();
                    }
                    else
                    {
                        this.connections[destProcessID].SendEvent.WaitOne();
                    }
                    continue;
                }

                //this.connections[destProcessID].Status = shuttingDown ? ConnectionStatus.ShuttingDown : ConnectionStatus.Sending;

                this.connections[destProcessID].BytesSent += length; // Progress + Data
                
                //Logging.Progress("Sent {0} bytes to {1} (of {2})", bytesSent, destProcessID, length);
            }

            this.shutdownSendCountdown.Signal();
#if SEND_AFFINITY
            pin.Dispose();
#endif
        }