/**
* Takes messages off the internal message queue and handles them by either
* sending responses, forwarding the message on, or processing it internally
*/
private void ProcessMessagesThreadProc()
{
// A response message queue used to send messages back to the one which sent it
Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();
while (AgentHasShutDown == false)
{
StartTiming("ProcessMessage-Internal", true);
lock ( MessageQueueLock )
{
// Swap the SM and PM message queue to keep things moving at full speed
Queue <AgentMessage> Temp = MessageQueuePM;
MessageQueuePM = MessageQueueSM;
MessageQueueSM = Temp;
}
// Process all messages currently in the queue
while (MessageQueuePM.Count > 0)
{
Debug.Assert(ResponseMessageQueue.Count == 0);
// Get and process the next message
AgentMessage NextMessage = MessageQueuePM.Dequeue();
Debug.Assert(NextMessage != null);
bool bMessageHandled = false;
switch (NextMessage.Type)
{
case EMessageType.SIGNAL:
{
if (NextMessage is DisconnectionSignalMessage)
{
// Mark the connection as inactive
DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED;
DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
}
// Signal the message and move on
AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
SignalMessage.ResetEvent.Set();
bMessageHandled = true;
}
break;
case EMessageType.TIMING:
{
Connection FromConnection;
if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
{
Connection ToConnection;
if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
(ToConnection is LocalConnection))
{
// Handle message
AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
bMessageHandled = true;
}
}
}
break;
case EMessageType.TASK_REQUEST:
{
// Look up the requesting connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection RequestingConnection;
if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
{
// Look up the specified Job
AgentJob JobToAskForTasks = RequestingConnection.Job;
if (JobToAskForTasks != null)
{
// If we get a valid response back, add it to the queue
AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
if (Response != null)
{
ResponseMessageQueue.Enqueue(Response);
// Specifications and releases are always handled here, but
// reservations are special in that we will send a reservation
// back to local connections but we'll need to make sure the
// message continues on to remote connections.
if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
(Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
(JobToAskForTasks.Owner is LocalConnection)))
{
bMessageHandled = true;
}
}
}
else
{
// Unable to find the Job, just send back a release message
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
//ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
// ETaskRequestResponseType.RELEASE ) );
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
case EMessageType.TASK_STATE:
{
// Look up the sending connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection SendingConnection;
if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
(SendingConnection.Job != null))
{
// Look up the specified Job
AgentJob UpdatedJob;
if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
{
AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
UpdatedJob.UpdateTaskState(UpdatedTaskState);
if (UpdatedJob.Owner is LocalConnection)
{
// If the Task state change is of a type potentially interesting to
// the Instigator, return it
switch (UpdatedTaskState.TaskState)
{
case EJobTaskState.TASK_STATE_INVALID:
case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
// For these message types, allow the message to continue on
break;
default:
// Nothing to do otherwise, mark the message as handled
bMessageHandled = true;
break;
}
}
else
{
// Always send messages on for remote connections
}
}
else
{
// Unable to find the Job, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
}
// If the message was not handled completely, send it on
if (bMessageHandled == false)
{
// Look up who the message is being sent to and make sure they're
// still active and if not, ignore the message
Connection Recipient;
Debug.Assert(NextMessage.To != Constants.INVALID);
if (Connections.TryGetValue(NextMessage.To,out Recipient))
{
if (Recipient is LocalConnection)
{
// If the recipient is local, place it in the proper queue
// and signal that a message is ready
LocalConnection LocalRecipient = Recipient as LocalConnection;
lock (LocalRecipient.MessageQueue)
{
LocalRecipient.MessageQueue.Enqueue(NextMessage);
string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type,
LocalRecipient.MessageQueue.Count);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
LocalRecipient.MessageAvailableSignal();
}
}
else
{
Debug.Assert(Recipient is RemoteConnection);
// If the recipient is remote, send the message via SendMessage
// unless the message is a Task being sent back, which is sent
// via the dedicated Task API
RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
if (NextMessage is AgentTaskSpecification)
{
// All new Tasks are sent via the dedicated Task API
AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Specification"] = TaskSpecification;
Hashtable RemoteOutParameters = null;
Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
if (Error >= 0)
{
// Perhaps we should be sending an accept message back?
}
else
{
AgentTaskState UpdateMessage;
if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
{
// Special case of the connection dropping while we're adding the
// task, say it's been killed to requeue
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_KILLED);
}
else
{
// All other error cases will be rejections
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_REJECTED);
}
AgentJob Job;
if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
{
Job.UpdateTaskState(UpdateMessage);
}
}
}
else
{
// All standard messages are sent via the SendMessage API
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Message"] = NextMessage;
Hashtable RemoteOutParameters = null;
RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
}
string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
}
}
else
{
Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
}
}
// If there are any responses to the message, send them
if (ResponseMessageQueue.Count > 0)
{
foreach (AgentMessage NextResponse in ResponseMessageQueue)
{
// For each one of the messages, set the routing fields properly
NextResponse.To = NextMessage.From;
NextResponse.From = NextMessage.To;
// And then queue the message back up immediately
MessageQueuePM.Enqueue(NextResponse);
}
ResponseMessageQueue.Clear();
}
}
StopTiming();
// Wait for a message to become available and once unlocked, swap the queues
// and check for messages to process. Set a timeout, so we'll wake up every
// now and then to check for a quit signal at least
MessageQueueReady.WaitOne(500);
}
}
public void CheckForReservations()
{
// We need the state to not change while we're in here
lock( CurrentStateLock )
{
// Depending on the current Job state, handle any outstanding Task reservations
if( ( CurrentState == JobState.AGENT_JOB_UNSPECIFIED ) ||
( CurrentState == JobState.AGENT_JOB_PENDING ) )
{
// As long as the job is PENDING, do nothing. In fact, there shouldn't be
// any reservations at this point. Assert this.
Debug.Assert( TaskReservationCount == 0 );
}
else if( CurrentState == JobState.AGENT_JOB_RUNNING )
{
// If there are any outstanding reservations and tasks to give out...
UInt32 PendingTasksCount = ( UInt32 )PendingTasks.Count;
UInt32 ReservationsToTryToFill = Math.Min( PendingTasksCount, TaskReservationCount );
if( ReservationsToTryToFill > 0 )
{
// Try to fulfill the local ones first
List<LocalConnection> LocalChildren = Owner.LocalChildren.Values;
List<LocalConnection>.Enumerator Children = LocalChildren.GetEnumerator();
while( ( Children.MoveNext() ) &&
( ReservationsToTryToFill > 0 ) )
{
LocalConnection Local = Children.Current;
while( ( Local.ReservationCount > 0 ) &&
( ReservationsToTryToFill > 0 ) )
{
// If we should avoid assigning tasks to this agent,
// simply continue on to the next one (only meaningful
// for local connections since we currently would only
// avoid tasks on the Instigator)
if( AvoidNextTask( Local ) )
{
break;
}
AgentTaskRequestResponse Response = GetNextTask( Local );
if( ( Response != null ) &&
( Response is AgentTaskSpecification ) )
{
AgentTaskSpecification NextTask = Response as AgentTaskSpecification;
// For local connections, send the task message directly
Response.To = Local.Handle;
Response.From = Owner.Handle;
Manager.SendMessageInternal( Owner, Response );
// Consider the reservation satisfied
TaskReservationCount--;
Local.ReservationCount--;
ReservationsToTryToFill--;
}
else
{
// If we didn't get a task, don't keep trying for now
ReservationsToTryToFill = 0;
}
}
}
}
// If we still have pending tasks try to fulfill any outstanding remote reservations
if( ReservationsToTryToFill > 0 )
{
List<RemoteConnection> RemoteChildren = Owner.RemoteChildren.Values;
List<RemoteConnection>.Enumerator Children = RemoteChildren.GetEnumerator();
while( ( Children.MoveNext() ) &&
( ReservationsToTryToFill > 0 ) )
{
RemoteConnection Remote = Children.Current;
while( ( Remote.ReservationCount > 0 ) &&
( ReservationsToTryToFill > 0 ) )
{
AgentTaskRequestResponse Response = GetNextTask( Remote );
if( ( Response != null ) &&
( Response is AgentTaskSpecification ) )
{
AgentTaskSpecification NextTask = Response as AgentTaskSpecification;
// For remote connections, add the task using the job/task API
Hashtable AddTaskInParameters = new Hashtable();
AddTaskInParameters["Version"] = ESwarmVersionValue.VER_1_0;
AddTaskInParameters["Specification"] = NextTask;
Hashtable AddTaskOutParameters = null;
if( Remote.Interface.AddTask( Remote.Handle, AddTaskInParameters, ref AddTaskOutParameters ) >= 0 )
{
// Consider the reservation satisfied
TaskReservationCount--;
Remote.ReservationCount--;
ReservationsToTryToFill--;
}
else
{
// If this fails for any reason, update the task and move on to the next child.
// Note that marking it KILLED will allow it to be requeued for another agent
// and ultimately it'll end up on the local agent, where if it fails, it'll
// be marked a real failure.
UpdateTaskState( new AgentTaskState( NextTask.JobGuid, NextTask.TaskGuid, EJobTaskState.TASK_STATE_KILLED ) );
break;
}
}
else
{
// If we didn't get a task, don't keep trying for now
ReservationsToTryToFill = 0;
}
}
}
}
}
else if( CurrentState == JobState.AGENT_JOB_CLOSED )
{
// If there are any outstanding reservations, we can release them now that we know
// there will be no more tasks added to the Job
if( TaskReservationCount > 0 )
{
foreach( LocalConnection Local in Owner.LocalChildren.Values )
{
// If there are any outstanding reservations, a single RELEASE is sufficient
if( Local.ReservationCount > 0 )
{
AgentTaskRequestResponse ReleaseMessage =
new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE );
ReleaseMessage.To = Local.Handle;
ReleaseMessage.From = Owner.Handle;
Manager.SendMessageInternal( Owner, ReleaseMessage );
}
// Consider all reservations canceled
TaskReservationCount -= Local.ReservationCount;
Local.ReservationCount = 0;
}
foreach( RemoteConnection Remote in Owner.RemoteChildren.Values )
{
// If there are any outstanding reservations, a single RELEASE is sufficient
if( Remote.ReservationCount > 0 )
{
AgentTaskRequestResponse ReleaseMessage =
new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE );
ReleaseMessage.To = Remote.Handle;
ReleaseMessage.From = Owner.Handle;
Manager.SendMessageInternal( Owner, ReleaseMessage );
}
// Consider all reservations canceled
TaskReservationCount -= Remote.ReservationCount;
Remote.ReservationCount = 0;
}
// Sanity check
Debug.Assert( TaskReservationCount == 0 );
}
}
}
}