Example #1
0
        /**
         * Takes messages off the internal message queue and handles them by either
         * sending responses, forwarding the message on, or processing it internally
         */
        private void ProcessMessagesThreadProc()
        {
            // A response message queue used to send messages back to the one which sent it
            Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();

            while (AgentHasShutDown == false)
            {
                StartTiming("ProcessMessage-Internal", true);

                lock ( MessageQueueLock )
                {
                    // Swap the SM and PM message queue to keep things moving at full speed
                    Queue <AgentMessage> Temp = MessageQueuePM;
                    MessageQueuePM = MessageQueueSM;
                    MessageQueueSM = Temp;
                }

                // Process all messages currently in the queue
                while (MessageQueuePM.Count > 0)
                {
                    Debug.Assert(ResponseMessageQueue.Count == 0);

                    // Get and process the next message
                    AgentMessage NextMessage = MessageQueuePM.Dequeue();
                    Debug.Assert(NextMessage != null);

                    bool bMessageHandled = false;
                    switch (NextMessage.Type)
                    {
                    case EMessageType.SIGNAL:
                    {
                        if (NextMessage is DisconnectionSignalMessage)
                        {
                            // Mark the connection as inactive
                            DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
                            Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
                            DisconnectMessage.ConnectionToDisconnect.CurrentState     = ConnectionState.DISCONNECTED;
                            DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
                        }

                        // Signal the message and move on
                        AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
                        SignalMessage.ResetEvent.Set();
                        bMessageHandled = true;
                    }
                    break;

                    case EMessageType.TIMING:
                    {
                        Connection FromConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
                        {
                            Connection ToConnection;
                            if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
                                (ToConnection is LocalConnection))
                            {
                                // Handle message
                                AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
                                AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
                                bMessageHandled = true;
                            }
                        }
                    }
                    break;

                    case EMessageType.TASK_REQUEST:
                    {
                        // Look up the requesting connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection RequestingConnection;
                        if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
                        {
                            // Look up the specified Job
                            AgentJob JobToAskForTasks = RequestingConnection.Job;
                            if (JobToAskForTasks != null)
                            {
                                // If we get a valid response back, add it to the queue
                                AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
                                if (Response != null)
                                {
                                    ResponseMessageQueue.Enqueue(Response);

                                    // Specifications and releases are always handled here, but
                                    // reservations are special in that we will send a reservation
                                    // back to local connections but we'll need to make sure the
                                    // message continues on to remote connections.
                                    if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
                                        (Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
                                        ((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
                                         (JobToAskForTasks.Owner is LocalConnection)))
                                    {
                                        bMessageHandled = true;
                                    }
                                }
                            }
                            else
                            {
                                // Unable to find the Job, just send back a release message
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
                                //ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
                                //															ETaskRequestResponseType.RELEASE ) );
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;

                    case EMessageType.TASK_STATE:
                    {
                        // Look up the sending connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection SendingConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
                            (SendingConnection.Job != null))
                        {
                            // Look up the specified Job
                            AgentJob UpdatedJob;
                            if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
                            {
                                AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
                                UpdatedJob.UpdateTaskState(UpdatedTaskState);

                                if (UpdatedJob.Owner is LocalConnection)
                                {
                                    // If the Task state change is of a type potentially interesting to
                                    // the Instigator, return it
                                    switch (UpdatedTaskState.TaskState)
                                    {
                                    case EJobTaskState.TASK_STATE_INVALID:
                                    case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
                                    case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
                                        // For these message types, allow the message to continue on
                                        break;

                                    default:
                                        // Nothing to do otherwise, mark the message as handled
                                        bMessageHandled = true;
                                        break;
                                    }
                                }
                                else
                                {
                                    // Always send messages on for remote connections
                                }
                            }
                            else
                            {
                                // Unable to find the Job, swallow the request
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;
                    }

                    // If the message was not handled completely, send it on
                    if (bMessageHandled == false)
                    {
                        // Look up who the message is being sent to and make sure they're
                        // still active and if not, ignore the message
                        Connection Recipient;
                        Debug.Assert(NextMessage.To != Constants.INVALID);
                        if (Connections.TryGetValue(NextMessage.To,out Recipient))
                        {
                            if (Recipient is LocalConnection)
                            {
                                // If the recipient is local, place it in the proper queue
                                // and signal that a message is ready
                                LocalConnection LocalRecipient = Recipient as LocalConnection;
                                lock (LocalRecipient.MessageQueue)
                                {
                                    LocalRecipient.MessageQueue.Enqueue(NextMessage);

                                    string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
                                                                         NextMessage.To,
                                                                         NextMessage.From,
                                                                         NextMessage.Type,
                                                                         LocalRecipient.MessageQueue.Count);

                                    Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);

                                    LocalRecipient.MessageAvailableSignal();
                                }
                            }
                            else
                            {
                                Debug.Assert(Recipient is RemoteConnection);

                                // If the recipient is remote, send the message via SendMessage
                                // unless the message is a Task being sent back, which is sent
                                // via the dedicated Task API
                                RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
                                if (NextMessage is AgentTaskSpecification)
                                {
                                    // All new Tasks are sent via the dedicated Task API
                                    AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;

                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"]       = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Specification"] = TaskSpecification;
                                    Hashtable RemoteOutParameters = null;

                                    Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
                                    if (Error >= 0)
                                    {
                                        // Perhaps we should be sending an accept message back?
                                    }
                                    else
                                    {
                                        AgentTaskState UpdateMessage;
                                        if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
                                        {
                                            // Special case of the connection dropping while we're adding the
                                            // task, say it's been killed to requeue
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_KILLED);
                                        }
                                        else
                                        {
                                            // All other error cases will be rejections
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_REJECTED);
                                        }
                                        AgentJob Job;
                                        if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
                                        {
                                            Job.UpdateTaskState(UpdateMessage);
                                        }
                                    }
                                }
                                else
                                {
                                    // All standard messages are sent via the SendMessage API
                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Message"] = NextMessage;
                                    Hashtable RemoteOutParameters = null;

                                    RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
                                }

                                string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
                                                                     NextMessage.To,
                                                                     NextMessage.From,
                                                                     NextMessage.Type);

                                Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
                            }
                        }
                        else
                        {
                            Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
                        }
                    }

                    // If there are any responses to the message, send them
                    if (ResponseMessageQueue.Count > 0)
                    {
                        foreach (AgentMessage NextResponse in ResponseMessageQueue)
                        {
                            // For each one of the messages, set the routing fields properly
                            NextResponse.To   = NextMessage.From;
                            NextResponse.From = NextMessage.To;

                            // And then queue the message back up immediately
                            MessageQueuePM.Enqueue(NextResponse);
                        }
                        ResponseMessageQueue.Clear();
                    }
                }

                StopTiming();

                // Wait for a message to become available and once unlocked, swap the queues
                // and check for messages to process. Set a timeout, so we'll wake up every
                // now and then to check for a quit signal at least
                MessageQueueReady.WaitOne(500);
            }
        }
Example #2
0
		public AgentTask( AgentTaskSpecification NewSpecification )
		{
			Specification = NewSpecification;
			CurrentState = new AgentTaskState( NewSpecification.JobGuid, NewSpecification.TaskGuid, EJobTaskState.TASK_STATE_IDLE );
			CurrentOwner = null;
		}
Example #3
0
        /**
         * Takes messages off the internal message queue and handles them by either
         * sending responses, forwarding the message on, or processing it internally
         */
        private void ProcessMessagesThreadProc()
        {
            // A response message queue used to send messages back to the one which sent it
            Queue<AgentMessage> ResponseMessageQueue = new Queue<AgentMessage>();

            while( AgentHasShutDown == false )
            {
                StartTiming( "ProcessMessage-Internal", true );

                lock( MessageQueueLock )
                {
                    // Swap the SM and PM message queue to keep things moving at full speed
                    Queue<AgentMessage> Temp = MessageQueuePM;
                    MessageQueuePM = MessageQueueSM;
                    MessageQueueSM = Temp;
                }

                // Process all messages currently in the queue
                while( MessageQueuePM.Count > 0 )
                {
                    Debug.Assert( ResponseMessageQueue.Count == 0 );

                    // Get and process the next message
                    AgentMessage NextMessage = MessageQueuePM.Dequeue();
                    Debug.Assert( NextMessage != null );

                    bool bMessageHandled = false;
                    switch( NextMessage.Type )
                    {
                        case EMessageType.SIGNAL:
                        {
                            if( NextMessage is DisconnectionSignalMessage )
                            {
                                // Mark the connection as inactive
                                DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
                                Log( EVerbosityLevel.Informative, ELogColour.Green, String.Format( "[CloseConnection] Connection disconnected {0:X8}", DisconnectMessage.ConnectionToDisconnect.Handle ) );
                                DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED;
                                DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
                            }

                            // Signal the message and move on
                            AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
                            SignalMessage.ResetEvent.Set();
                            bMessageHandled = true;
                        }
                        break;

                        case EMessageType.TIMING:
                        {
                            Connection FromConnection;
                            if( ( Connections.TryGetValue( NextMessage.From, out FromConnection ) ) )
                            {
                                Connection ToConnection;
                                if( ( Connections.TryGetValue( NextMessage.To, out ToConnection ) ) &&
                                    ( ToConnection is LocalConnection ) )
                                {
                                    // Handle message
                                    AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
                                    AgentApplication.UpdateMachineState( MachineNameFromConnection( FromConnection ), TimingMessage.ThreadNum, TimingMessage.State );
                                    bMessageHandled = true;
                                }
                            }
                        }
                        break;

                        case EMessageType.TASK_REQUEST:
                        {
                            // Look up the requesting connection
                            Debug.Assert( NextMessage.From != Constants.INVALID );
                            Connection RequestingConnection;
                            if( Connections.TryGetValue( NextMessage.From, out RequestingConnection ) )
                            {
                                // Look up the specified Job
                                AgentJob JobToAskForTasks = RequestingConnection.Job;
                                if( JobToAskForTasks != null )
                                {
                                    // If we get a valid response back, add it to the queue
                                    AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask( RequestingConnection );
                                    if( Response != null )
                                    {
                                        ResponseMessageQueue.Enqueue( Response );

                                        // Specifications and releases are always handled here, but
                                        // reservations are special in that we will send a reservation
                                        // back to local connections but we'll need to make sure the
                                        // message continues on to remote connections.
                                        if( ( Response.ResponseType == ETaskRequestResponseType.SPECIFICATION ) ||
                                            ( Response.ResponseType == ETaskRequestResponseType.RELEASE ) ||
                                            ( ( Response.ResponseType == ETaskRequestResponseType.RESERVATION ) &&
                                              ( JobToAskForTasks.Owner is LocalConnection ) ) )
                                        {
                                            bMessageHandled = true;
                                        }
                                    }
                                }
                                else
                                {
                                    // Unable to find the Job, just send back a release message
                                    Log( EVerbosityLevel.Verbose, ELogColour.Orange, "[ProcessMessage] Unable to find Job for Task Request; may have been closed" );
                                    //ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
                                    //															ETaskRequestResponseType.RELEASE ) );
                                    bMessageHandled = true;
                                }
                            }
                            else
                            {
                                // Unable to find the connection, swallow the request
                                Log( EVerbosityLevel.Verbose, ELogColour.Orange, "[ProcessMessage] Unable to find owning Connection for Task Request" );
                                bMessageHandled = true;
                            }
                        }
                        break;

                        case EMessageType.TASK_STATE:
                        {
                            // Look up the sending connection
                            Debug.Assert( NextMessage.From != Constants.INVALID );
                            Connection SendingConnection;
                            if( ( Connections.TryGetValue( NextMessage.From, out SendingConnection ) ) &&
                                ( SendingConnection.Job != null ) )
                            {
                                // Look up the specified Job
                                AgentJob UpdatedJob;
                                if( ActiveJobs.TryGetValue( SendingConnection.Job.JobGuid, out UpdatedJob ) )
                                {
                                    AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
                                    UpdatedJob.UpdateTaskState( UpdatedTaskState );

                                    if( UpdatedJob.Owner is LocalConnection )
                                    {
                                        // If the Task state change is of a type potentially interesting to
                                        // the Instigator, return it
                                        switch( UpdatedTaskState.TaskState )
                                        {
                                            case EJobTaskState.TASK_STATE_INVALID:
                                            case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
                                            case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
                                                // For these message types, allow the message to continue on
                                                break;

                                            default:
                                                // Nothing to do otherwise, mark the message as handled
                                                bMessageHandled = true;
                                                break;
                                        }
                                    }
                                    else
                                    {
                                        // Always send messages on for remote connections
                                    }
                                }
                                else
                                {
                                    // Unable to find the Job, swallow the request
                                    Log( EVerbosityLevel.Verbose, ELogColour.Orange, "[ProcessMessage] Unable to find Job for Task Request" );
                                    bMessageHandled = true;
                                }
                            }
                            else
                            {
                                // Unable to find the connection, swallow the request
                                Log( EVerbosityLevel.Verbose, ELogColour.Orange, "[ProcessMessage] Unable to find owning Connection for Task Request" );
                                bMessageHandled = true;
                            }
                        }
                        break;
                    }

                    // If the message was not handled completely, send it on
                    if( bMessageHandled == false )
                    {
                        // Look up who the message is being sent to and make sure they're
                        // still active and if not, ignore the message
                        Connection Recipient;
                        Debug.Assert( NextMessage.To != Constants.INVALID );
                        if( Connections.TryGetValue( NextMessage.To, out Recipient ) )
                        {
                            if( Recipient is LocalConnection )
                            {
                                // If the recipient is local, place it in the proper queue
                                // and signal that a message is ready
                                LocalConnection LocalRecipient = Recipient as LocalConnection;
                                lock( LocalRecipient.MessageQueue )
                                {
                                    LocalRecipient.MessageQueue.Enqueue( NextMessage );

                                    string NewLogMessage = String.Format( "Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
                                        NextMessage.To,
                                        NextMessage.From,
                                        NextMessage.Type,
                                        LocalRecipient.MessageQueue.Count );

                                    Log( EVerbosityLevel.SuperVerbose, ELogColour.Green, NewLogMessage );

                                    LocalRecipient.MessageAvailableSignal();
                                }
                            }
                            else
                            {
                                Debug.Assert( Recipient is RemoteConnection );

                                // If the recipient is remote, send the message via SendMessage
                                // unless the message is a Task being sent back, which is sent
                                // via the dedicated Task API
                                RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
                                if( NextMessage is AgentTaskSpecification )
                                {
                                    // All new Tasks are sent via the dedicated Task API
                                    AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;

                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Specification"] = TaskSpecification;
                                    Hashtable RemoteOutParameters = null;

                                    Int32 Error = RemoteRecipient.Interface.AddTask( RemoteRecipient.Handle, RemoteInParameters, ref RemoteOutParameters );
                                    if( Error >= 0 )
                                    {
                                        // Perhaps we should be sending an accept message back?
                                    }
                                    else
                                    {
                                        AgentTaskState UpdateMessage;
                                        if( Error == Constants.ERROR_CONNECTION_DISCONNECTED )
                                        {
                                            // Special case of the connection dropping while we're adding the
                                            // task, say it's been killed to requeue
                                            UpdateMessage = new AgentTaskState( TaskSpecification.JobGuid,
                                                                                TaskSpecification.TaskGuid,
                                                                                EJobTaskState.TASK_STATE_KILLED );
                                        }
                                        else
                                        {
                                            // All other error cases will be rejections
                                            UpdateMessage = new AgentTaskState( TaskSpecification.JobGuid,
                                                                                TaskSpecification.TaskGuid,
                                                                                EJobTaskState.TASK_STATE_REJECTED );
                                        }
                                        AgentJob Job;
                                        if( ActiveJobs.TryGetValue( TaskSpecification.JobGuid, out Job ) )
                                        {
                                            Job.UpdateTaskState( UpdateMessage );
                                        }
                                    }
                                }
                                else
                                {
                                    // All standard messages are sent via the SendMessage API
                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Message"] = NextMessage;
                                    Hashtable RemoteOutParameters = null;

                                    RemoteRecipient.Interface.SendMessage( NextMessage.To, RemoteInParameters, ref RemoteOutParameters );
                                }

                                string NewLogMessage = String.Format( "Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
                                    NextMessage.To,
                                    NextMessage.From,
                                    NextMessage.Type );

                                Log( EVerbosityLevel.SuperVerbose, ELogColour.Green, NewLogMessage );
                            }
                        }
                        else
                        {
                            Log( EVerbosityLevel.Informative, ELogColour.Orange, "ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString() );
                        }
                    }

                    // If there are any responses to the message, send them
                    if( ResponseMessageQueue.Count > 0 )
                    {
                        foreach( AgentMessage NextResponse in ResponseMessageQueue )
                        {
                            // For each one of the messages, set the routing fields properly
                            NextResponse.To = NextMessage.From;
                            NextResponse.From = NextMessage.To;

                            // And then queue the message back up immediately
                            MessageQueuePM.Enqueue( NextResponse );
                        }
                        ResponseMessageQueue.Clear();
                    }
                }

                StopTiming();

                // Wait for a message to become available and once unlocked, swap the queues
                // and check for messages to process. Set a timeout, so we'll wake up every
                // now and then to check for a quit signal at least
                MessageQueueReady.WaitOne( 500 );
            }
        }
Example #4
0
		public void UpdateTaskState( AgentTaskState UpdatedTaskState )
		{
			// Sanity checks
			Debug.Assert( CurrentState != JobState.AGENT_JOB_UNSPECIFIED );
			Debug.Assert( CurrentState != JobState.AGENT_JOB_PENDING );

			AgentTask RunningTask;
			if( RunningTasks.TryGetValue( UpdatedTaskState.TaskGuid, out RunningTask ) )
			{
				// Update the individual Task state
				RunningTask.CurrentState = UpdatedTaskState;
				switch( UpdatedTaskState.TaskState )
				{
					case EJobTaskState.TASK_STATE_ACCEPTED:
						// Nothing to do right now, but we'll need to track start times, etc. later
						break;

					case EJobTaskState.TASK_STATE_RUNNING:
						// Mark the real start time of this task (also set when we give the task out)
						RunningTask.StartTime = DateTime.UtcNow;
						break;

					case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
						UpdateTaskStateAsSuccess( RunningTask );
						break;

					case EJobTaskState.TASK_STATE_REJECTED:
						if( RunningTask.CurrentOwner is RemoteConnection )
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Orange, "[UpdateTaskState]: Task Rejected remotely by " + ( RunningTask.CurrentOwner as RemoteConnection ).Info.Name );
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Orange, "[UpdateTaskState]: Requeueing: " + RunningTask.Specification.Parameters );
							UpdateTaskStateAsRequeued( RunningTask );
						}
						else
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Red, "[UpdateTaskState]: Task Rejected locally by " + Environment.MachineName + ", counted as failure");
							UpdateTaskStateAsFailure( RunningTask );
						}
						break;

					case EJobTaskState.TASK_STATE_KILLED:
						if( RunningTask.CurrentOwner is RemoteConnection )
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Orange, "[UpdateTaskState]: Task Killed remotely by " + ( RunningTask.CurrentOwner as RemoteConnection ).Info.Name );
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Orange, "[UpdateTaskState]: Requeueing: " + RunningTask.Specification.Parameters );
							UpdateTaskStateAsRequeued( RunningTask );
						}
						else
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Red, "[UpdateTaskState]: Task Killed locally by " + Environment.MachineName + ", counted as failure" );
							UpdateTaskStateAsFailure( RunningTask );
						}
						break;

					case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
						if( RunningTask.CurrentOwner is RemoteConnection )
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Red, "[UpdateTaskState]: Task Failed on " + ( RunningTask.CurrentOwner as RemoteConnection ).Info.Name );
						}
						else
						{
							Manager.Log( EVerbosityLevel.Informative, ELogColour.Red, "[UpdateTaskState]: Task Failed on " + Environment.MachineName );
						}
						Manager.Log( EVerbosityLevel.Informative, ELogColour.Red, "[UpdateTaskState]: Task Failed: " + RunningTask.Specification.Parameters );
						UpdateTaskStateAsFailure( RunningTask );
						break;
				}

				// Update the owning Job state, by checking for failures. Success is only
				// determined after all Tasks are assured to be done or orphanable. Only
				// do this one time and let the new state, if there is one, be sticky.
				lock( CurrentSuccessStateLock )
				{
					if( CurrentSuccessState == JobSuccessState.AGENT_JOB_INCOMPLETE )
					{
						// Updtae if any task is a failure
						if( TaskFailureCount > 0 )
						{
							// Update the state and send a message indicating the failure
							CurrentSuccessState = JobSuccessState.AGENT_JOB_FAILURE;
							if( OwnerIsInstigator )
							{
								// Log and send an INFO message describing the failure
								string NewMessageText = "Job has failed! The task failure count is non-zero"; 
								SendJobCompletedMessage( new AgentInfoMessage( NewMessageText ) );
							}
						}
						// Update if all tasks are successful and we're the instigator, since only
						// the instigator can make this determination properly
						else if( ( TaskSuccessCount == TaskCount ) &&
								 ( OwnerIsInstigator ) )
						{
							CurrentSuccessState = JobSuccessState.AGENT_JOB_SUCCESS;
							if( OwnerIsInstigator )
							{
								// Log and send an INFO message describing the success
								string NewMessageText = "Job is a success!";
								SendJobCompletedMessage( new AgentInfoMessage( NewMessageText ) );
							}
						}
					}
				}

				// Update the visualizer if this agent is the Instigator
				if( OwnerIsInstigator )
				{
					AgentApplication.UpdateMachineState( Environment.MachineName, RetiredTasks.Count, EProgressionState.TasksCompleted );
					AgentApplication.UpdateMachineState( Environment.MachineName, RunningTasks.Count, EProgressionState.TasksInProgress );
				}
			}
		}
		/**
		 * Sends a message to an Agent (return messages are sent via the FConnectionCallback)
		 *
		 * @param Message The message being sent
		 *
		 * @return Int32 error code (< 0 is error)
		 */
		public virtual Int32 SendMessage(IntPtr NativeMessagePtr)
		{
			StartTiming("SendMessage-Managed", true);

			FMessage NativeMessage = (FMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FMessage));

			Int32 ReturnValue = Constants.INVALID;
			if (Connection != null)
			{
				AgentMessage ManagedMessage = null;

				// TODO: As we add additional versions, convert to a switch rather than if-else.
				// For now, just use a simple if since we only have one version and a switch is
				// overkill.
				if (NativeMessage.Version == ESwarmVersionValue.VER_1_0)
				{
					switch (NativeMessage.Type)
					{
						case EMessageType.TASK_REQUEST_RESPONSE:
							// Swallow this message, since it should not be sent along to a local connection
							// since all Job and Task information is contained within the Agent itself
						break;

						case EMessageType.TASK_STATE:
						{
							FTaskState NativeTaskStateMessage = (FTaskState)Marshal.PtrToStructure(NativeMessagePtr, typeof(FTaskState));
							AgentGuid ManagedTaskGuid = new AgentGuid(NativeTaskStateMessage.TaskGuid.A,
																	  NativeTaskStateMessage.TaskGuid.B,
																	  NativeTaskStateMessage.TaskGuid.C,
																	  NativeTaskStateMessage.TaskGuid.D);
							EJobTaskState TaskState = (EJobTaskState)NativeTaskStateMessage.TaskState;
							AgentTaskState ManagedTaskStateMessage = new AgentTaskState(null, ManagedTaskGuid, TaskState);

							ManagedTaskStateMessage.TaskExitCode = NativeTaskStateMessage.TaskExitCode;
							ManagedTaskStateMessage.TaskRunningTime = NativeTaskStateMessage.TaskRunningTime;

							// If there is a message, be sure copy and pass it on
							if (NativeTaskStateMessage.TaskMessage != IntPtr.Zero)
							{
								ManagedTaskStateMessage.TaskMessage = FStringMarshaler.MarshalNativeToManaged(NativeTaskStateMessage.TaskMessage);
							}

							ManagedMessage = ManagedTaskStateMessage;
						}
						break;

						case EMessageType.INFO:
						{
							// Create the managed version of the info message
							FInfoMessage NativeInfoMessage = (FInfoMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FInfoMessage));
							AgentInfoMessage ManagedInfoMessage = new AgentInfoMessage();
							if (NativeInfoMessage.TextMessage != IntPtr.Zero)
							{
								ManagedInfoMessage.TextMessage = FStringMarshaler.MarshalNativeToManaged(NativeInfoMessage.TextMessage);
							}
							ManagedMessage = ManagedInfoMessage;
						}
						break;

						case EMessageType.ALERT:
						{
							// Create the managed version of the alert message
							FAlertMessage NativeAlertMessage = (FAlertMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FAlertMessage));
							AgentGuid JobGuid = new AgentGuid(NativeAlertMessage.JobGuid.A,
															  NativeAlertMessage.JobGuid.B,
															  NativeAlertMessage.JobGuid.C,
															  NativeAlertMessage.JobGuid.D);
							AgentAlertMessage ManagedAlertMessage = new AgentAlertMessage(JobGuid);
							ManagedAlertMessage.AlertLevel = (EAlertLevel)(NativeAlertMessage.AlertLevel);
							AgentGuid ObjectGuid = new AgentGuid(NativeAlertMessage.ObjectGuid.A,
																 NativeAlertMessage.ObjectGuid.B,
																 NativeAlertMessage.ObjectGuid.C,
																 NativeAlertMessage.ObjectGuid.D);
							ManagedAlertMessage.ObjectGuid = ObjectGuid;
							ManagedAlertMessage.TypeId = NativeAlertMessage.TypeId;
							if (NativeAlertMessage.TextMessage != IntPtr.Zero)
							{
								ManagedAlertMessage.TextMessage = FStringMarshaler.MarshalNativeToManaged(NativeAlertMessage.TextMessage);
							}
							ManagedMessage = ManagedAlertMessage;
						}
						break;

						case EMessageType.TIMING:
						{
							// Create the managed version of the info message
							FTimingMessage NativeTimingMessage = (FTimingMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FTimingMessage));
							AgentTimingMessage ManagedTimingMessage = new AgentTimingMessage((EProgressionState)NativeTimingMessage.State, NativeTimingMessage.ThreadNum);
							ManagedMessage = ManagedTimingMessage;
						}
						break;

						default:
							// By default, just pass the message version and type through, but
							// any additional payload of a specialized type will be lost
							ManagedMessage = new AgentMessage((EMessageType)NativeMessage.Type);
						break;
					}
				}

				if (ManagedMessage != null)
				{
					try
					{
						// Finally, send the message to the Agent
						StartTiming("SendMessage-Remote", false);
						Connection.SendMessage(ConnectionHandle, ManagedMessage);
						StopTiming();
						ReturnValue = Constants.SUCCESS;
					}
					catch (Exception Ex)
					{
						Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:SendMessage] Error: " + Ex.Message);
						ReturnValue = Constants.ERROR_CONNECTION_DISCONNECTED;
						CleanupClosedConnection();
					}
				}
			}
			else
			{
				ReturnValue = Constants.ERROR_CONNECTION_NOT_FOUND;
			}

			StopTiming();
			return( ReturnValue );
		}