Esempio n. 1
0
        /**
         * Flush the message queue - note this does not assure that it's empty, it just
         * makes sure that all messages that are in the queue at the time of this method
         * call are processed. This call is blocking.
         */
        public bool FlushMessageQueue(Connection RequestingConnection, bool WithDisconnect)
        {
            Log(EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format("[FlushMessageQueue] Draining message queue for {0:X8}", RequestingConnection.Handle));

            // This special call will act as a "writer" since we only want to
            // allow one of these to act at a time. By doing this, we ensure
            // that all "readers" that use this lock are finished.
            SendMessageLock.AcquireWriterLock(Timeout.Infinite);
            Log(EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format("[FlushMessageQueue] Lock acquired for {0:X8}", RequestingConnection.Handle));

            // Flush the message queue with a signal message, optionally with a disconnection event
            AgentSignalMessage SignalMessage;

            if (WithDisconnect)
            {
                SignalMessage = new DisconnectionSignalMessage(RequestingConnection);
            }
            else
            {
                SignalMessage = new AgentSignalMessage();
            }
            Int32 SignalMessageSent = SendMessageInternal(RequestingConnection, SignalMessage);

            // Release our "writer" lock
            SendMessageLock.ReleaseWriterLock();
            Log(EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format("[FlushMessageQueue] Lock released for {0:X8}", RequestingConnection.Handle));

            if (SignalMessageSent == Constants.SUCCESS)
            {
                SignalMessage.ResetEvent.WaitOne(Timeout.Infinite);
                Log(EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format("[FlushMessageQueue] Drain complete for {0:X8}", RequestingConnection.Handle));
                return(true);
            }
            else
            {
                Log(EVerbosityLevel.ExtraVerbose, ELogColour.Green, "[FlushMessageQueue] Drain failed because of an error sending the signal message");
                return(false);
            }
        }
Esempio n. 2
0
        /**
         * Flush the message queue - note this does not assure that it's empty, it just
         * makes sure that all messages that are in the queue at the time of this method
         * call are processed. This call is blocking.
         */
        public bool FlushMessageQueue( Connection RequestingConnection, bool WithDisconnect )
        {
            Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Draining message queue for {0:X8}", RequestingConnection.Handle ) );

            // This special call will act as a "writer" since we only want to
            // allow one of these to act at a time. By doing this, we ensure
            // that all "readers" that use this lock are finished.
            SendMessageLock.AcquireWriterLock( Timeout.Infinite );
            Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock acquired for {0:X8}", RequestingConnection.Handle ) );

            // Flush the message queue with a signal message, optionally with a disconnection event
            AgentSignalMessage SignalMessage;
            if( WithDisconnect )
            {
                SignalMessage = new DisconnectionSignalMessage( RequestingConnection );
            }
            else
            {
                SignalMessage = new AgentSignalMessage();
            }
            Int32 SignalMessageSent = SendMessageInternal( RequestingConnection, SignalMessage );

            // Release our "writer" lock
            SendMessageLock.ReleaseWriterLock();
            Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock released for {0:X8}", RequestingConnection.Handle ) );

            if( SignalMessageSent == Constants.SUCCESS )
            {
                SignalMessage.ResetEvent.WaitOne( Timeout.Infinite );
                Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Drain complete for {0:X8}", RequestingConnection.Handle ) );
                return true;
            }
            else
            {
                Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, "[FlushMessageQueue] Drain failed because of an error sending the signal message" );
                return false;
            }
        }
Esempio n. 3
0
        /**
         * Takes messages off the internal message queue and handles them by either
         * sending responses, forwarding the message on, or processing it internally
         */
        private void ProcessMessagesThreadProc()
        {
            // A response message queue used to send messages back to the one which sent it
            Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();

            while (AgentHasShutDown == false)
            {
                StartTiming("ProcessMessage-Internal", true);

                lock ( MessageQueueLock )
                {
                    // Swap the SM and PM message queue to keep things moving at full speed
                    Queue <AgentMessage> Temp = MessageQueuePM;
                    MessageQueuePM = MessageQueueSM;
                    MessageQueueSM = Temp;
                }

                // Process all messages currently in the queue
                while (MessageQueuePM.Count > 0)
                {
                    Debug.Assert(ResponseMessageQueue.Count == 0);

                    // Get and process the next message
                    AgentMessage NextMessage = MessageQueuePM.Dequeue();
                    Debug.Assert(NextMessage != null);

                    bool bMessageHandled = false;
                    switch (NextMessage.Type)
                    {
                    case EMessageType.SIGNAL:
                    {
                        if (NextMessage is DisconnectionSignalMessage)
                        {
                            // Mark the connection as inactive
                            DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
                            Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
                            DisconnectMessage.ConnectionToDisconnect.CurrentState     = ConnectionState.DISCONNECTED;
                            DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
                        }

                        // Signal the message and move on
                        AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
                        SignalMessage.ResetEvent.Set();
                        bMessageHandled = true;
                    }
                    break;

                    case EMessageType.TIMING:
                    {
                        Connection FromConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
                        {
                            Connection ToConnection;
                            if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
                                (ToConnection is LocalConnection))
                            {
                                // Handle message
                                AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
                                AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
                                bMessageHandled = true;
                            }
                        }
                    }
                    break;

                    case EMessageType.TASK_REQUEST:
                    {
                        // Look up the requesting connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection RequestingConnection;
                        if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
                        {
                            // Look up the specified Job
                            AgentJob JobToAskForTasks = RequestingConnection.Job;
                            if (JobToAskForTasks != null)
                            {
                                // If we get a valid response back, add it to the queue
                                AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
                                if (Response != null)
                                {
                                    ResponseMessageQueue.Enqueue(Response);

                                    // Specifications and releases are always handled here, but
                                    // reservations are special in that we will send a reservation
                                    // back to local connections but we'll need to make sure the
                                    // message continues on to remote connections.
                                    if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
                                        (Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
                                        ((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
                                         (JobToAskForTasks.Owner is LocalConnection)))
                                    {
                                        bMessageHandled = true;
                                    }
                                }
                            }
                            else
                            {
                                // Unable to find the Job, just send back a release message
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
                                //ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
                                //															ETaskRequestResponseType.RELEASE ) );
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;

                    case EMessageType.TASK_STATE:
                    {
                        // Look up the sending connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection SendingConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
                            (SendingConnection.Job != null))
                        {
                            // Look up the specified Job
                            AgentJob UpdatedJob;
                            if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
                            {
                                AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
                                UpdatedJob.UpdateTaskState(UpdatedTaskState);

                                if (UpdatedJob.Owner is LocalConnection)
                                {
                                    // If the Task state change is of a type potentially interesting to
                                    // the Instigator, return it
                                    switch (UpdatedTaskState.TaskState)
                                    {
                                    case EJobTaskState.TASK_STATE_INVALID:
                                    case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
                                    case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
                                        // For these message types, allow the message to continue on
                                        break;

                                    default:
                                        // Nothing to do otherwise, mark the message as handled
                                        bMessageHandled = true;
                                        break;
                                    }
                                }
                                else
                                {
                                    // Always send messages on for remote connections
                                }
                            }
                            else
                            {
                                // Unable to find the Job, swallow the request
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;
                    }

                    // If the message was not handled completely, send it on
                    if (bMessageHandled == false)
                    {
                        // Look up who the message is being sent to and make sure they're
                        // still active and if not, ignore the message
                        Connection Recipient;
                        Debug.Assert(NextMessage.To != Constants.INVALID);
                        if (Connections.TryGetValue(NextMessage.To,out Recipient))
                        {
                            if (Recipient is LocalConnection)
                            {
                                // If the recipient is local, place it in the proper queue
                                // and signal that a message is ready
                                LocalConnection LocalRecipient = Recipient as LocalConnection;
                                lock (LocalRecipient.MessageQueue)
                                {
                                    LocalRecipient.MessageQueue.Enqueue(NextMessage);

                                    string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
                                                                         NextMessage.To,
                                                                         NextMessage.From,
                                                                         NextMessage.Type,
                                                                         LocalRecipient.MessageQueue.Count);

                                    Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);

                                    LocalRecipient.MessageAvailableSignal();
                                }
                            }
                            else
                            {
                                Debug.Assert(Recipient is RemoteConnection);

                                // If the recipient is remote, send the message via SendMessage
                                // unless the message is a Task being sent back, which is sent
                                // via the dedicated Task API
                                RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
                                if (NextMessage is AgentTaskSpecification)
                                {
                                    // All new Tasks are sent via the dedicated Task API
                                    AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;

                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"]       = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Specification"] = TaskSpecification;
                                    Hashtable RemoteOutParameters = null;

                                    Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
                                    if (Error >= 0)
                                    {
                                        // Perhaps we should be sending an accept message back?
                                    }
                                    else
                                    {
                                        AgentTaskState UpdateMessage;
                                        if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
                                        {
                                            // Special case of the connection dropping while we're adding the
                                            // task, say it's been killed to requeue
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_KILLED);
                                        }
                                        else
                                        {
                                            // All other error cases will be rejections
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_REJECTED);
                                        }
                                        AgentJob Job;
                                        if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
                                        {
                                            Job.UpdateTaskState(UpdateMessage);
                                        }
                                    }
                                }
                                else
                                {
                                    // All standard messages are sent via the SendMessage API
                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Message"] = NextMessage;
                                    Hashtable RemoteOutParameters = null;

                                    RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
                                }

                                string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
                                                                     NextMessage.To,
                                                                     NextMessage.From,
                                                                     NextMessage.Type);

                                Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
                            }
                        }
                        else
                        {
                            Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
                        }
                    }

                    // If there are any responses to the message, send them
                    if (ResponseMessageQueue.Count > 0)
                    {
                        foreach (AgentMessage NextResponse in ResponseMessageQueue)
                        {
                            // For each one of the messages, set the routing fields properly
                            NextResponse.To   = NextMessage.From;
                            NextResponse.From = NextMessage.To;

                            // And then queue the message back up immediately
                            MessageQueuePM.Enqueue(NextResponse);
                        }
                        ResponseMessageQueue.Clear();
                    }
                }

                StopTiming();

                // Wait for a message to become available and once unlocked, swap the queues
                // and check for messages to process. Set a timeout, so we'll wake up every
                // now and then to check for a quit signal at least
                MessageQueueReady.WaitOne(500);
            }
        }