/**
* Takes messages off the internal message queue and handles them by either
* sending responses, forwarding the message on, or processing it internally
*/
private void ProcessMessagesThreadProc()
{
// A response message queue used to send messages back to the one which sent it
Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();
while (AgentHasShutDown == false)
{
StartTiming("ProcessMessage-Internal", true);
lock ( MessageQueueLock )
{
// Swap the SM and PM message queue to keep things moving at full speed
Queue <AgentMessage> Temp = MessageQueuePM;
MessageQueuePM = MessageQueueSM;
MessageQueueSM = Temp;
}
// Process all messages currently in the queue
while (MessageQueuePM.Count > 0)
{
Debug.Assert(ResponseMessageQueue.Count == 0);
// Get and process the next message
AgentMessage NextMessage = MessageQueuePM.Dequeue();
Debug.Assert(NextMessage != null);
bool bMessageHandled = false;
switch (NextMessage.Type)
{
case EMessageType.SIGNAL:
{
if (NextMessage is DisconnectionSignalMessage)
{
// Mark the connection as inactive
DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED;
DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
}
// Signal the message and move on
AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
SignalMessage.ResetEvent.Set();
bMessageHandled = true;
}
break;
case EMessageType.TIMING:
{
Connection FromConnection;
if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
{
Connection ToConnection;
if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
(ToConnection is LocalConnection))
{
// Handle message
AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
bMessageHandled = true;
}
}
}
break;
case EMessageType.TASK_REQUEST:
{
// Look up the requesting connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection RequestingConnection;
if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
{
// Look up the specified Job
AgentJob JobToAskForTasks = RequestingConnection.Job;
if (JobToAskForTasks != null)
{
// If we get a valid response back, add it to the queue
AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
if (Response != null)
{
ResponseMessageQueue.Enqueue(Response);
// Specifications and releases are always handled here, but
// reservations are special in that we will send a reservation
// back to local connections but we'll need to make sure the
// message continues on to remote connections.
if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
(Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
(JobToAskForTasks.Owner is LocalConnection)))
{
bMessageHandled = true;
}
}
}
else
{
// Unable to find the Job, just send back a release message
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
//ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
// ETaskRequestResponseType.RELEASE ) );
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
case EMessageType.TASK_STATE:
{
// Look up the sending connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection SendingConnection;
if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
(SendingConnection.Job != null))
{
// Look up the specified Job
AgentJob UpdatedJob;
if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
{
AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
UpdatedJob.UpdateTaskState(UpdatedTaskState);
if (UpdatedJob.Owner is LocalConnection)
{
// If the Task state change is of a type potentially interesting to
// the Instigator, return it
switch (UpdatedTaskState.TaskState)
{
case EJobTaskState.TASK_STATE_INVALID:
case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
// For these message types, allow the message to continue on
break;
default:
// Nothing to do otherwise, mark the message as handled
bMessageHandled = true;
break;
}
}
else
{
// Always send messages on for remote connections
}
}
else
{
// Unable to find the Job, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
}
// If the message was not handled completely, send it on
if (bMessageHandled == false)
{
// Look up who the message is being sent to and make sure they're
// still active and if not, ignore the message
Connection Recipient;
Debug.Assert(NextMessage.To != Constants.INVALID);
if (Connections.TryGetValue(NextMessage.To,out Recipient))
{
if (Recipient is LocalConnection)
{
// If the recipient is local, place it in the proper queue
// and signal that a message is ready
LocalConnection LocalRecipient = Recipient as LocalConnection;
lock (LocalRecipient.MessageQueue)
{
LocalRecipient.MessageQueue.Enqueue(NextMessage);
string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type,
LocalRecipient.MessageQueue.Count);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
LocalRecipient.MessageAvailableSignal();
}
}
else
{
Debug.Assert(Recipient is RemoteConnection);
// If the recipient is remote, send the message via SendMessage
// unless the message is a Task being sent back, which is sent
// via the dedicated Task API
RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
if (NextMessage is AgentTaskSpecification)
{
// All new Tasks are sent via the dedicated Task API
AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Specification"] = TaskSpecification;
Hashtable RemoteOutParameters = null;
Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
if (Error >= 0)
{
// Perhaps we should be sending an accept message back?
}
else
{
AgentTaskState UpdateMessage;
if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
{
// Special case of the connection dropping while we're adding the
// task, say it's been killed to requeue
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_KILLED);
}
else
{
// All other error cases will be rejections
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_REJECTED);
}
AgentJob Job;
if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
{
Job.UpdateTaskState(UpdateMessage);
}
}
}
else
{
// All standard messages are sent via the SendMessage API
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Message"] = NextMessage;
Hashtable RemoteOutParameters = null;
RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
}
string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
}
}
else
{
Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
}
}
// If there are any responses to the message, send them
if (ResponseMessageQueue.Count > 0)
{
foreach (AgentMessage NextResponse in ResponseMessageQueue)
{
// For each one of the messages, set the routing fields properly
NextResponse.To = NextMessage.From;
NextResponse.From = NextMessage.To;
// And then queue the message back up immediately
MessageQueuePM.Enqueue(NextResponse);
}
ResponseMessageQueue.Clear();
}
}
StopTiming();
// Wait for a message to become available and once unlocked, swap the queues
// and check for messages to process. Set a timeout, so we'll wake up every
// now and then to check for a quit signal at least
MessageQueueReady.WaitOne(500);
}
}
/**
* Sends a message to an Agent (return messages are sent via the FConnectionCallback)
*
* @param Message The message being sent
*
* @return Int32 error code (< 0 is error)
*/
public virtual Int32 SendMessage(IntPtr NativeMessagePtr)
{
StartTiming("SendMessage-Managed", true);
FMessage NativeMessage = (FMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FMessage));
Int32 ReturnValue = Constants.INVALID;
if (Connection != null)
{
AgentMessage ManagedMessage = null;
// TODO: As we add additional versions, convert to a switch rather than if-else.
// For now, just use a simple if since we only have one version and a switch is
// overkill.
if (NativeMessage.Version == ESwarmVersionValue.VER_1_0)
{
switch (NativeMessage.Type)
{
case EMessageType.TASK_REQUEST_RESPONSE:
// Swallow this message, since it should not be sent along to a local connection
// since all Job and Task information is contained within the Agent itself
break;
case EMessageType.TASK_STATE:
{
FTaskState NativeTaskStateMessage = (FTaskState)Marshal.PtrToStructure(NativeMessagePtr, typeof(FTaskState));
AgentGuid ManagedTaskGuid = new AgentGuid(NativeTaskStateMessage.TaskGuid.A,
NativeTaskStateMessage.TaskGuid.B,
NativeTaskStateMessage.TaskGuid.C,
NativeTaskStateMessage.TaskGuid.D);
EJobTaskState TaskState = (EJobTaskState)NativeTaskStateMessage.TaskState;
AgentTaskState ManagedTaskStateMessage = new AgentTaskState(null, ManagedTaskGuid, TaskState);
ManagedTaskStateMessage.TaskExitCode = NativeTaskStateMessage.TaskExitCode;
ManagedTaskStateMessage.TaskRunningTime = NativeTaskStateMessage.TaskRunningTime;
// If there is a message, be sure copy and pass it on
if (NativeTaskStateMessage.TaskMessage != IntPtr.Zero)
{
ManagedTaskStateMessage.TaskMessage = FStringMarshaler.MarshalNativeToManaged(NativeTaskStateMessage.TaskMessage);
}
ManagedMessage = ManagedTaskStateMessage;
}
break;
case EMessageType.INFO:
{
// Create the managed version of the info message
FInfoMessage NativeInfoMessage = (FInfoMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FInfoMessage));
AgentInfoMessage ManagedInfoMessage = new AgentInfoMessage();
if (NativeInfoMessage.TextMessage != IntPtr.Zero)
{
ManagedInfoMessage.TextMessage = FStringMarshaler.MarshalNativeToManaged(NativeInfoMessage.TextMessage);
}
ManagedMessage = ManagedInfoMessage;
}
break;
case EMessageType.ALERT:
{
// Create the managed version of the alert message
FAlertMessage NativeAlertMessage = (FAlertMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FAlertMessage));
AgentGuid JobGuid = new AgentGuid(NativeAlertMessage.JobGuid.A,
NativeAlertMessage.JobGuid.B,
NativeAlertMessage.JobGuid.C,
NativeAlertMessage.JobGuid.D);
AgentAlertMessage ManagedAlertMessage = new AgentAlertMessage(JobGuid);
ManagedAlertMessage.AlertLevel = (EAlertLevel)(NativeAlertMessage.AlertLevel);
AgentGuid ObjectGuid = new AgentGuid(NativeAlertMessage.ObjectGuid.A,
NativeAlertMessage.ObjectGuid.B,
NativeAlertMessage.ObjectGuid.C,
NativeAlertMessage.ObjectGuid.D);
ManagedAlertMessage.ObjectGuid = ObjectGuid;
ManagedAlertMessage.TypeId = NativeAlertMessage.TypeId;
if (NativeAlertMessage.TextMessage != IntPtr.Zero)
{
ManagedAlertMessage.TextMessage = FStringMarshaler.MarshalNativeToManaged(NativeAlertMessage.TextMessage);
}
ManagedMessage = ManagedAlertMessage;
}
break;
case EMessageType.TIMING:
{
// Create the managed version of the info message
FTimingMessage NativeTimingMessage = (FTimingMessage)Marshal.PtrToStructure(NativeMessagePtr, typeof(FTimingMessage));
AgentTimingMessage ManagedTimingMessage = new AgentTimingMessage((EProgressionState)NativeTimingMessage.State, NativeTimingMessage.ThreadNum);
ManagedMessage = ManagedTimingMessage;
}
break;
default:
// By default, just pass the message version and type through, but
// any additional payload of a specialized type will be lost
ManagedMessage = new AgentMessage((EMessageType)NativeMessage.Type);
break;
}
}
if (ManagedMessage != null)
{
try
{
// Finally, send the message to the Agent
StartTiming("SendMessage-Remote", false);
Connection.SendMessage(ConnectionHandle, ManagedMessage);
StopTiming();
ReturnValue = Constants.SUCCESS;
}
catch (Exception Ex)
{
Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:SendMessage] Error: " + Ex.Message);
ReturnValue = Constants.ERROR_CONNECTION_DISCONNECTED;
CleanupClosedConnection();
}
}
}
else
{
ReturnValue = Constants.ERROR_CONNECTION_NOT_FOUND;
}
StopTiming();
return( ReturnValue );
}
