public Int32 OpenJob( Connection NewOwner, AgentGuid NewJobGuid )
{
// Create the bi-directional link between the job and its owner
NewOwner.Job = this;
Owner = NewOwner;
// Determine if the owner is the Instigator, and if so, perform
// any additional work necessary
if( NewOwner is LocalConnection )
{
OwnerIsInstigator = true;
// If the owner of the job is the Instigator, set the current working
// directory to be where the Instigator process is executing
LocalConnection LocalNewOwner = NewOwner as LocalConnection;
Manager.SetCurrentDirectoryByProcessID( LocalNewOwner.ProcessID );
// Update the visualizer
AgentApplication.UpdateMachineState( Environment.MachineName, -1, EProgressionState.InstigatorConnected );
}
// Add to the Agent-wide list of active jobs
JobGuid = NewJobGuid;
Manager.ActiveJobs.Add( NewJobGuid, this );
return Constants.SUCCESS;
}
/**
* Flush the message queue - note this does not assure that it's empty, it just
* makes sure that all messages that are in the queue at the time of this method
* call are processed. This call is blocking.
*/
public bool FlushMessageQueue( Connection RequestingConnection, bool WithDisconnect )
{
Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Draining message queue for {0:X8}", RequestingConnection.Handle ) );
// This special call will act as a "writer" since we only want to
// allow one of these to act at a time. By doing this, we ensure
// that all "readers" that use this lock are finished.
SendMessageLock.AcquireWriterLock( Timeout.Infinite );
Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock acquired for {0:X8}", RequestingConnection.Handle ) );
// Flush the message queue with a signal message, optionally with a disconnection event
AgentSignalMessage SignalMessage;
if( WithDisconnect )
{
SignalMessage = new DisconnectionSignalMessage( RequestingConnection );
}
else
{
SignalMessage = new AgentSignalMessage();
}
Int32 SignalMessageSent = SendMessageInternal( RequestingConnection, SignalMessage );
// Release our "writer" lock
SendMessageLock.ReleaseWriterLock();
Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock released for {0:X8}", RequestingConnection.Handle ) );
if( SignalMessageSent == Constants.SUCCESS )
{
SignalMessage.ResetEvent.WaitOne( Timeout.Infinite );
Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Drain complete for {0:X8}", RequestingConnection.Handle ) );
return true;
}
else
{
Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, "[FlushMessageQueue] Drain failed because of an error sending the signal message" );
return false;
}
}
private Int32 SetChannelHandleAndAdd( Connection ConnectionThatOwnsChannel, Channel NewChannel )
{
Int32 NewHandleValue;
lock( SetChannelHandleAndAddLock )
{
// Generate a new random value for the handle
do
{
NewHandleValue = SetChannelHandleAndAddGenerator.Next();
}
// Keep generating new values until we find one not already in use
while( ConnectionThatOwnsChannel.OpenChannels.ContainsKey( NewHandleValue ) );
// Now set the new handle value and add to the set
NewChannel.SetHandle( NewHandleValue );
ConnectionThatOwnsChannel.OpenChannels.Add( NewHandleValue, NewChannel );
}
return NewHandleValue;
}
private bool RequestTaskFiles( Connection RequestingConnection, AgentTaskSpecification TaskSpecification )
{
// Check for and possibly request each dependency
if( TaskSpecification.Dependencies != null )
{
foreach( string Dependency in TaskSpecification.Dependencies )
{
if( !RequestDependency( RequestingConnection, Dependency, true ) )
{
return false;
}
}
}
return true;
}
private bool RequestJobFiles( Connection RequestingConnection, AgentJobSpecification JobSpecification )
{
// Check for and possibly request the executable
if( !RequestDependency( RequestingConnection, JobSpecification.ExecutableName, true ) )
{
return false;
}
// Check for and possibly request each dependency
if( JobSpecification.RequiredDependencies != null )
{
foreach( string Dependency in JobSpecification.RequiredDependencies )
{
if( !RequestDependency( RequestingConnection, Dependency, true ) )
{
return false;
}
}
}
if( JobSpecification.OptionalDependencies != null )
{
foreach( string Dependency in JobSpecification.OptionalDependencies )
{
if( !RequestDependency( RequestingConnection, Dependency, false ) )
{
return false;
}
}
}
return true;
}
public void CancelReservations( Connection ReservationHolder )
{
// We need the state to not change while we're in here
lock( CurrentStateLock )
{
TaskReservationCount -= ReservationHolder.ReservationCount;
ReservationHolder.ReservationCount = 0;
}
}
public AgentTaskRequestResponse GetNextTask( Connection RequestingConnection )
{
// We need the state to not change while we're in here
lock( CurrentStateLock )
{
// Take the next Task off of the Pending list and add it to the Active list
if( CurrentState == JobState.AGENT_JOB_RUNNING )
{
lock( PendingTasks )
{
AgentTask NextTask = null;
// Check for whether we're running in deterministic mode
if( DeterministicModeEnabled )
{
// If so, look up the requesting connection's task queue
Queue<AgentTask> TaskQueue;
string RequestorName = Manager.MachineNameFromConnection( RequestingConnection );
if( Manager.LastSuccessfulJobRecord.AgentToTaskQueueMapping.TryGetValue( RequestorName, out TaskQueue ) )
{
if( TaskQueue.Count > 0 )
{
// Grab the next pending task
NextTask = TaskQueue.Dequeue();
}
}
}
// Otherwise, we're in a standard distribution mode, see if there
// are tasks available to hand out
else if( PendingTasks.Count > 0 )
{
// Check to see if should avoid the next task
if( !AvoidNextTask( RequestingConnection ) )
{
NextTask = PendingTasks.Pop();
}
}
if( NextTask != null )
{
// Assign the new owner and add the task to the running sets
NextTask.CurrentOwner = RequestingConnection;
// Set the assign time of the task as well as the start time,
// in case they don't send back a RUNNING message, which will
// set the proper start time
NextTask.AssignTime = DateTime.UtcNow;
NextTask.StartTime = DateTime.UtcNow;
// Update additional stats
if( RequestingConnection is RemoteConnection )
{
TaskCountRemote++;
}
AgentTaskSpecification TaskSpecification = NextTask.Specification;
RequestingConnection.RunningTasks.Add( TaskSpecification.TaskGuid, NextTask );
RunningTasks.Add( TaskSpecification.TaskGuid, NextTask );
// Send back the now ready task specification
return TaskSpecification;
}
// If we're not the final authority of this job, then we'll send back a
// reservation and allow the Instigator send the real answer later.
// Alternately, if the owner is the Instigator, we also need to send
// back a reservation to make sure the local job application runs until
// the job is done.
else if( ( OwnerIsInstigator == false ) ||
( RequestingConnection is LocalConnection ) )
{
// Always track outstanding reservations
RequestingConnection.ReservationCount++;
TaskReservationCount++;
// For local connections, send a RESERVATION response which indicates that we
// have no work currently to hand out, but we might in the future, so sit
// tight. For remote connections, we only need to track the reservation
if( RequestingConnection is LocalConnection )
{
return ( new AgentTaskRequestResponse( JobGuid,
ETaskRequestResponseType.RESERVATION ) );
}
}
// Otherwise, we're all out of tasks, so release the worker
else
{
// No more distributed tasks to hand out, send back a release
return ( new AgentTaskRequestResponse( JobGuid,
ETaskRequestResponseType.RELEASE ) );
}
}
}
else if( CurrentState == JobState.AGENT_JOB_CLOSED )
{
// No more distributed tasks to hand out, send back a release
return ( new AgentTaskRequestResponse( JobGuid,
ETaskRequestResponseType.RELEASE ) );
}
}
return null;
}
private bool AvoidNextTask( Connection RequestingConection )
{
Debug.Assert( RequestingConection.Job != null );
bool SafeToAvoidNextTask =
( AgentApplication.Options.AvoidLocalExecution ) &&
( RequestingConection is LocalConnection ) &&
( RequestingConection.Job.Owner.RemoteChildren.Count > 0 );
return SafeToAvoidNextTask;
}
private bool RequestDependency(Connection RequestingConnection, string Dependency, bool bIsRequired)
{
Hashtable InParameters = new Hashtable();
InParameters["Version"] = ESwarmVersionValue.VER_1_0;
InParameters["ChannelName"] = Dependency;
Hashtable OutParameters = null;
if (Manager.TestChannel(RequestingConnection.Handle, InParameters, ref OutParameters) < 0)
{
if (RequestingConnection is RemoteConnection)
{
// For remote connection misses, try to pull the file from the instigator
Manager.Log(EVerbosityLevel.Verbose, ELogColour.Green, "[Job] Attempting to pull file from remote Agent: " + Dependency);
if (Manager.PullChannel(RequestingConnection as RemoteConnection, Dependency, null, 5) == false)
{
if (bIsRequired)
{
Manager.Log(EVerbosityLevel.Informative, ELogColour.Red, "[Job] Error: Failed to pull a required file from remote Agent: " + Dependency);
return false;
}
else
{
Manager.Log(EVerbosityLevel.Verbose, ELogColour.Orange, "[Job] Warning: Failed to pull an optional file from remote Agent: " + Dependency);
}
}
}
else
{
Debug.Assert(RequestingConnection is LocalConnection);
if (bIsRequired)
{
// Always fail on local connection misses
Manager.Log(EVerbosityLevel.Informative, ELogColour.Red, "[Job] Error: Failed to find required file: " + Dependency);
return false;
}
else
{
Manager.Log(EVerbosityLevel.Verbose, ELogColour.Orange, "[Job] Warning: Failed to find an optional file: " + Dependency);
}
}
}
return true;
}
/*
* Get the machine IP address from the connection structure
*/
public string MachineIPAddressFromConnection( Connection RequestingConnection )
{
string MachineIPAddress = "";
RemoteConnection Remote = RequestingConnection as RemoteConnection;
if( Remote == null )
{
MachineIPAddress = "127.0.0.1";
}
else
{
MachineIPAddress = Remote.Interface.RemoteAgentIPAddress;
}
return ( MachineIPAddress );
}
public DisconnectionSignalMessage( Connection InConnectionToDisconnect )
{
ConnectionToDisconnect = InConnectionToDisconnect;
}
/**
* Actual work performed by SendMessage happens here
*/
public Int32 SendMessageInternal( Connection Sender, AgentMessage NewMessage )
{
Int32 ErrorCode = Constants.INVALID;
// We assume the message is valid, but if somewhere below we change our
// mind, this value will be set to false and we'll eat the message
bool bMessageIsValid = true;
// Logic for the setting of the To and From fields (if not already set)
//
// All connections sending messages are implicitly sending them to the
// Instigator of the Job they're working on. Depending on where the
// message is coming from, we might need to patch up the To field a
// little bit to ensure it's heading to a Local connection when it
// needs to be. The only case this applies is when we receive a message
// from a Remote connection, directed toward a Remote connection, which
// happens when we get a message from a Remote Agent from an even more
// Remote connection (i.e. a Job connection on the remote machine):
//
// To Local, From Local -> routing of main message and replies are ok
// To Local, From Remote -> routing of main message and replies are ok
// To Remote, From Local -> routing of main message and replies are ok
// To Remote, From Remote -> routing of replies is ok, but the main
// message is in trouble if it's not completely handled within the
// Agent's ProcessMessages routine. It would be forwarded on to the
// To field connection which is Remote and we'd end up bouncing the
// message back and forth forever. Need to adjust the To field to
// point to the parent of the To connection (which is where it's
// intended to go anyway). See further below for where we handle
// this case.
// If the From field is not set, give it the connection handle value by
// default so that any response message will automatically be routed back
// to it whether it's the sender or the recipient
if( NewMessage.From == Constants.INVALID )
{
NewMessage.From = Sender.Handle;
}
// If the connection used to send the message is Remote, check for the
// Remote -> Remote case described above
else if( Sender is RemoteConnection )
{
// If the From field is already set, see if we've already registered
// the connection this message is being sent from
Connection FromConnection;
if( !Connections.TryGetValue( NewMessage.From, out FromConnection ) )
{
// This is a new one - make it an alias for the sender so that any
// responses will be directed back via its sending interface
RemoteConnection RemoteSender = Sender as RemoteConnection;
RemoteSender.Aliases.Add( NewMessage.From );
// There are times we want to ensure no new connections are coming online
// where we'll lock the Connections dictionary (see MaintainCache)
lock( Connections )
{
Connections.Add( NewMessage.From, RemoteSender );
}
FromConnection = RemoteSender;
string LogMessage = String.Format( "[SendMessage] Added alias for remote connection: {0:X8} is an alias for {1:X8}",
NewMessage.From,
Sender.Handle );
Log( EVerbosityLevel.Informative, ELogColour.Green, LogMessage );
}
// If this is a Remote -> Remote situation, the proper place to route
// the message to the parent of the remote connection since the Agents
// generally act as glue between connections
if( FromConnection is RemoteConnection )
{
Debug.Assert( NewMessage.To != Constants.INVALID );
Connection ToConnection;
if( (Connections.TryGetValue( NewMessage.To, out ToConnection )) &&
(ToConnection is RemoteConnection) )
{
Connection ToConnectionParent = ToConnection.Parent;
if( ToConnectionParent != null )
{
NewMessage.To = ToConnectionParent.Handle;
}
}
}
}
// If the To field is not set, assign it based on the message type
if( NewMessage.To == Constants.INVALID )
{
// TODO: As we add additional versions, convert to a switch rather than if-else.
// For now, just use a simple if since we only have one version and a switch is
// overkill.
if( NewMessage.Version == ESwarmVersionValue.VER_1_0 )
{
// The default is for messages to be ultimately routed to the Instigator
// unless the message is one of a certain set of types that route
// directly to the connection specified
switch( NewMessage.Type )
{
// These message types need to be routed to the connection specified
// either because they are meant to be simple round-trip messages or
// because they are sent from within Swarm directly to the connection
// and should not be routed anywhere else
case EMessageType.QUIT:
case EMessageType.PING:
case EMessageType.SIGNAL:
NewMessage.To = Sender.Handle;
break;
// These message types need to be routed eventually to the Instigator
// connection, which is the ultimate ancestor up the parent chain, so
// simply assign the most senior parent we have
case EMessageType.INFO:
case EMessageType.ALERT:
case EMessageType.TIMING:
case EMessageType.TASK_REQUEST:
case EMessageType.TASK_STATE:
case EMessageType.JOB_STATE:
// By default, make the sender the recipient for these cases, in
// case the parent is no longer active
NewMessage.To = Sender.Handle;
Connection SenderParent = Sender.Parent;
if( SenderParent != null )
{
// If we have a parent connection and it's active, then
// assign it as the recipient
if( SenderParent.CurrentState == ConnectionState.CONNECTED )
{
NewMessage.To = SenderParent.Handle;
}
}
break;
// These message types are not expected and are each error cases
case EMessageType.NONE: // Should never be set to this
case EMessageType.JOB_SPECIFICATION: // Only used for messages going directly into OpenJob
case EMessageType.TASK_REQUEST_RESPONSE: // Should always have the To field set already
default:
Log( EVerbosityLevel.Informative, ELogColour.Orange, "SendMessage: Invalid message type received, ignoring " + NewMessage.Type.ToString() );
break;
}
// If still no assigned To field, consider it an error
if( NewMessage.To == Constants.INVALID )
{
Log( EVerbosityLevel.Informative, ELogColour.Orange, "SendMessage: No proper recipient found, ignoring " + NewMessage.Type.ToString() );
bMessageIsValid = false;
}
}
}
// If the message remains valid, post it to the queue
if( bMessageIsValid )
{
lock( MessageQueueLock )
{
Debug.Assert( NewMessage != null );
MessageQueueSM.Enqueue( NewMessage );
string NewLogMessage = String.Format( "Step 1 of N for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Agent)",
NewMessage.To,
NewMessage.From,
NewMessage.Type,
MessageQueueSM.Count );
Log( EVerbosityLevel.SuperVerbose, ELogColour.Green, NewLogMessage );
MessageQueueReady.Set();
}
ErrorCode = Constants.SUCCESS;
}
else
{
Log( EVerbosityLevel.Informative, ELogColour.Orange, String.Format( "SendMessage: Discarded message \"{0}\"", NewMessage.Type ) );
}
return ( ErrorCode );
}
/*
* Get the machine name from the connection structure
*/
public string MachineNameFromConnection( Connection RequestingConnection )
{
string Name = "";
RemoteConnection Remote = RequestingConnection as RemoteConnection;
if( Remote == null )
{
Name = Environment.MachineName;
}
else
{
Name = Remote.Info.Name;
}
return ( Name );
}
public AgentTask( AgentTaskSpecification NewSpecification )
{
Specification = NewSpecification;
CurrentState = new AgentTaskState( NewSpecification.JobGuid, NewSpecification.TaskGuid, EJobTaskState.TASK_STATE_IDLE );
CurrentOwner = null;
}
