/**
* Takes messages off the internal message queue and handles them by either
* sending responses, forwarding the message on, or processing it internally
*/
private void ProcessMessagesThreadProc()
{
// A response message queue used to send messages back to the one which sent it
Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();
while (AgentHasShutDown == false)
{
StartTiming("ProcessMessage-Internal", true);
lock ( MessageQueueLock )
{
// Swap the SM and PM message queue to keep things moving at full speed
Queue <AgentMessage> Temp = MessageQueuePM;
MessageQueuePM = MessageQueueSM;
MessageQueueSM = Temp;
}
// Process all messages currently in the queue
while (MessageQueuePM.Count > 0)
{
Debug.Assert(ResponseMessageQueue.Count == 0);
// Get and process the next message
AgentMessage NextMessage = MessageQueuePM.Dequeue();
Debug.Assert(NextMessage != null);
bool bMessageHandled = false;
switch (NextMessage.Type)
{
case EMessageType.SIGNAL:
{
if (NextMessage is DisconnectionSignalMessage)
{
// Mark the connection as inactive
DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED;
DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
}
// Signal the message and move on
AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
SignalMessage.ResetEvent.Set();
bMessageHandled = true;
}
break;
case EMessageType.TIMING:
{
Connection FromConnection;
if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
{
Connection ToConnection;
if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
(ToConnection is LocalConnection))
{
// Handle message
AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
bMessageHandled = true;
}
}
}
break;
case EMessageType.TASK_REQUEST:
{
// Look up the requesting connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection RequestingConnection;
if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
{
// Look up the specified Job
AgentJob JobToAskForTasks = RequestingConnection.Job;
if (JobToAskForTasks != null)
{
// If we get a valid response back, add it to the queue
AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
if (Response != null)
{
ResponseMessageQueue.Enqueue(Response);
// Specifications and releases are always handled here, but
// reservations are special in that we will send a reservation
// back to local connections but we'll need to make sure the
// message continues on to remote connections.
if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
(Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
(JobToAskForTasks.Owner is LocalConnection)))
{
bMessageHandled = true;
}
}
}
else
{
// Unable to find the Job, just send back a release message
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
//ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
// ETaskRequestResponseType.RELEASE ) );
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
case EMessageType.TASK_STATE:
{
// Look up the sending connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection SendingConnection;
if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
(SendingConnection.Job != null))
{
// Look up the specified Job
AgentJob UpdatedJob;
if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
{
AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
UpdatedJob.UpdateTaskState(UpdatedTaskState);
if (UpdatedJob.Owner is LocalConnection)
{
// If the Task state change is of a type potentially interesting to
// the Instigator, return it
switch (UpdatedTaskState.TaskState)
{
case EJobTaskState.TASK_STATE_INVALID:
case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
// For these message types, allow the message to continue on
break;
default:
// Nothing to do otherwise, mark the message as handled
bMessageHandled = true;
break;
}
}
else
{
// Always send messages on for remote connections
}
}
else
{
// Unable to find the Job, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
}
// If the message was not handled completely, send it on
if (bMessageHandled == false)
{
// Look up who the message is being sent to and make sure they're
// still active and if not, ignore the message
Connection Recipient;
Debug.Assert(NextMessage.To != Constants.INVALID);
if (Connections.TryGetValue(NextMessage.To,out Recipient))
{
if (Recipient is LocalConnection)
{
// If the recipient is local, place it in the proper queue
// and signal that a message is ready
LocalConnection LocalRecipient = Recipient as LocalConnection;
lock (LocalRecipient.MessageQueue)
{
LocalRecipient.MessageQueue.Enqueue(NextMessage);
string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type,
LocalRecipient.MessageQueue.Count);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
LocalRecipient.MessageAvailableSignal();
}
}
else
{
Debug.Assert(Recipient is RemoteConnection);
// If the recipient is remote, send the message via SendMessage
// unless the message is a Task being sent back, which is sent
// via the dedicated Task API
RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
if (NextMessage is AgentTaskSpecification)
{
// All new Tasks are sent via the dedicated Task API
AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Specification"] = TaskSpecification;
Hashtable RemoteOutParameters = null;
Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
if (Error >= 0)
{
// Perhaps we should be sending an accept message back?
}
else
{
AgentTaskState UpdateMessage;
if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
{
// Special case of the connection dropping while we're adding the
// task, say it's been killed to requeue
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_KILLED);
}
else
{
// All other error cases will be rejections
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_REJECTED);
}
AgentJob Job;
if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
{
Job.UpdateTaskState(UpdateMessage);
}
}
}
else
{
// All standard messages are sent via the SendMessage API
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Message"] = NextMessage;
Hashtable RemoteOutParameters = null;
RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
}
string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
}
}
else
{
Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
}
}
// If there are any responses to the message, send them
if (ResponseMessageQueue.Count > 0)
{
foreach (AgentMessage NextResponse in ResponseMessageQueue)
{
// For each one of the messages, set the routing fields properly
NextResponse.To = NextMessage.From;
NextResponse.From = NextMessage.To;
// And then queue the message back up immediately
MessageQueuePM.Enqueue(NextResponse);
}
ResponseMessageQueue.Clear();
}
}
StopTiming();
// Wait for a message to become available and once unlocked, swap the queues
// and check for messages to process. Set a timeout, so we'll wake up every
// now and then to check for a quit signal at least
MessageQueueReady.WaitOne(500);
}
}
public Int32 AddTask( AgentTaskSpecification NewTaskSpecification )
{
Int32 ErrorCode = Constants.INVALID;
// Using the provided specification, request all necessary Task files. For local
// Agents, the executable and dependencies must already exist in the cache. For
// remote Agents, we'll need request the necessary files from the Instigator.
if( RequestTaskFiles( Owner, NewTaskSpecification ) )
{
lock( PendingTasks )
{
Manager.Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Job] AddTask: Adding \"" + NewTaskSpecification.Parameters + "\" (" + NewTaskSpecification.TaskGuid.ToString() + ") with cost " + NewTaskSpecification.Cost.ToString() );
// Queue the Task
PendingTasks.Push( new AgentTask( NewTaskSpecification ) );
TaskCount++;
// Sanity check a couple assumptions here
if( PendingTasks.Count > 1 )
{
Debug.Assert( TaskReservationCount == 0 );
}
if( TaskReservationCount > 0 )
{
Debug.Assert( PendingTasks.Count == 1 );
}
}
// After adding a new Task, check for any outstanding reservations
CheckForReservations();
// Done
ErrorCode = Constants.SUCCESS;
}
else
{
Manager.Log( EVerbosityLevel.Critical, ELogColour.Red, "[Job] AddTask: Failed to cache necessary task files" );
ErrorCode = Constants.ERROR_CHANNEL_NOT_FOUND;
}
return ( ErrorCode );
}
private bool RequestTaskFiles( Connection RequestingConnection, AgentTaskSpecification TaskSpecification )
{
// Check for and possibly request each dependency
if( TaskSpecification.Dependencies != null )
{
foreach( string Dependency in TaskSpecification.Dependencies )
{
if( !RequestDependency( RequestingConnection, Dependency, true ) )
{
return false;
}
}
}
return true;
}
public AgentTask( AgentTaskSpecification NewSpecification )
{
Specification = NewSpecification;
CurrentState = new AgentTaskState( NewSpecification.JobGuid, NewSpecification.TaskGuid, EJobTaskState.TASK_STATE_IDLE );
CurrentOwner = null;
}
Int32 CacheAllFiles(AgentTaskSpecification TaskSpecification)
{
Int32 ReturnValue = Constants.INVALID;
try
{
TaskSpecification.DependenciesOriginalNames = null;
if ((TaskSpecification.Dependencies != null) &&
(TaskSpecification.Dependencies.Count > 0))
{
// Allocate the dictionary we'll use for the name mapping
Dictionary<String, String> DependenciesOriginalNames = new Dictionary<String, String>();
// Check and cache all the dependencies
for (Int32 i = 0; i < TaskSpecification.Dependencies.Count; i++)
{
String OriginalDependencyFullPath = TaskSpecification.Dependencies[i];
String CachedDependencyName;
ReturnValue = CacheFileAndConvertName(OriginalDependencyFullPath, out CachedDependencyName, false);
if (ReturnValue < 0)
{
// Failed to cache the dependency, return failure
Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:CacheAllFiles] Failed to cache the dependency: " + OriginalDependencyFullPath);
return ReturnValue;
}
TaskSpecification.Dependencies[i] = CachedDependencyName;
String OriginalDependencyName = Path.GetFileName(OriginalDependencyFullPath);
DependenciesOriginalNames.Add(CachedDependencyName, OriginalDependencyName);
}
// Set the newly created name mapping dictionary
TaskSpecification.DependenciesOriginalNames = DependenciesOriginalNames;
}
ReturnValue = Constants.SUCCESS;
}
catch (Exception Ex)
{
Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:CacheAllFiles] Error: " + Ex.Message);
ReturnValue = Constants.ERROR_CONNECTION_DISCONNECTED;
CleanupClosedConnection();
}
return ReturnValue;
}
/**
* Adds a Task to the current Job
*
* @param Specification A structure describing the new Task
*
* @return Int32 Error code (< 0 is an error)
*/
public virtual Int32 AddTask(FTaskSpecification Specification)
{
StartTiming("AddTask-Managed", true);
Int32 ReturnValue = Constants.INVALID;
if (Connection != null)
{
if (ConnectionConfiguration.AgentJobGuid != null)
{
// Convert the parameters from native to managed
AgentGuid TaskGuid = new AgentGuid(Specification.TaskGuid.A,
Specification.TaskGuid.B,
Specification.TaskGuid.C,
Specification.TaskGuid.D);
String Parameters = Specification.Parameters;
List<String> Dependencies = null;
if (Specification.DependencyCount > 0)
{
Dependencies = new List<String>();
for (UInt32 i = 0; i < Specification.DependencyCount; i++)
{
Dependencies.Add(Specification.Dependencies[i]);
}
}
AgentTaskSpecification NewSpecification =
new AgentTaskSpecification(ConnectionConfiguration.AgentJobGuid, TaskGuid, (Int32)Specification.Flags, Parameters, (Int32)Specification.Cost, Dependencies);
// Ensure all the files are in the cache with the right cache compatible name
ReturnValue = CacheAllFiles(NewSpecification);
if (ReturnValue >= 0)
{
// Queue up all tasks until the specification is complete and submit them all at once
PendingTasks.Add(NewSpecification);
}
}
else
{
ReturnValue = Constants.ERROR_JOB_NOT_FOUND;
}
}
else
{
ReturnValue = Constants.ERROR_CONNECTION_NOT_FOUND;
}
StopTiming();
return ReturnValue;
}
