public void AbandonPotentialRemoteAgents( LocalConnection ParentConnection )
{
lock( AbandonPotentialRemoteAgentsLock )
{
// Walk through the list of potential agents and disconnect each one
foreach( AgentInfo NextRemoteAgent in ParentConnection.PotentialRemoteAgents )
{
AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.RemoteDisconnected );
}
// Done with the queue now, clear it
ParentConnection.PotentialRemoteAgents.Clear();
foreach( AgentInfo NextRemoteAgent in ParentConnection.UnavailableRemoteAgents )
{
AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.RemoteDisconnected );
}
// Done with the queue now, clear it
ParentConnection.UnavailableRemoteAgents.Clear();
// Finished with remote agents, set the flag
ParentConnection.RemoteAgentsAbandoned = true;
}
}
/**
* Pulls the next agent off the list of potentials and tries to open a connection
* until it either opens one successfully or runs out of agents
*/
public RemoteConnection GetNextRemoteAgent( LocalConnection ParentConnection )
{
// Walk through the list of potential agents and try to open connections
// until we either find one or run out of agents
RemoteConnection Remote = null;
AgentJob Job = ParentConnection.Job;
while( ( Job != null ) &&
( Job.CurrentState == AgentJob.JobState.AGENT_JOB_RUNNING ) &&
( ParentConnection.PotentialRemoteAgents.Count > 0 ) &&
( Remote == null ) )
{
AgentInfo NextRemoteAgent = ParentConnection.PotentialRemoteAgents.Dequeue();
bool IsAssigned = false;
DateTime AssignedTimestamp = DateTime.MinValue;
if( ( NextRemoteAgent.Configuration.ContainsKey( "AssignedTo" ) ) &&
( ( NextRemoteAgent.Configuration["AssignedTo"] as string ) == Environment.MachineName ) )
{
IsAssigned = true;
AssignedTimestamp = ( DateTime )NextRemoteAgent.Configuration["AssignedTime"];
}
// If we failed to open the connection, but the agent is alive, add it to the list to try again later
if( ( TryOpenRemoteConnection( ParentConnection, NextRemoteAgent, IsAssigned, AssignedTimestamp, out Remote ) == Constants.SUCCESS ) &&
( Remote == null ) )
{
AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.Blocked );
ParentConnection.UnavailableRemoteAgents.Enqueue( NextRemoteAgent );
}
}
// Return whatever we found
return Remote;
}
/**
* Used by local processes to open a connection to Swarm
*/
private Int32 OpenConnection_1_0( Hashtable InParameters, ref Hashtable OutParameters )
{
// Always wait until the agent is fully initialized
Initialized.WaitOne();
// Unpack the input parameters
Int32 LocalProcessID = ( Int32 )InParameters["ProcessID"];
ELogFlags LoggingFlags = ( ELogFlags )InParameters["LoggingFlags"];
// Check for optional parameters
if( InParameters.Contains( "ProcessIsOwner" ) )
{
bool LocalProcessIsOwner = ( bool )InParameters["ProcessIsOwner"];
if( LocalProcessIsOwner )
{
OwnerProcessID = LocalProcessID;
}
}
// If a restart or an exit has been requested, stop accepting new connections
if( AgentIsShuttingDown )
{
return Constants.INVALID;
}
// If this is a "first" new local connection, clear the log window
LocalConnectionRequestWaiting.Set();
lock( Connections )
{
LocalConnectionRequestWaiting.Reset();
Int32 LocalConnectionCount = 0;
foreach( Connection NextConnection in Connections.Values )
{
if( NextConnection is LocalConnection )
{
LocalConnectionCount++;
}
}
if( LocalConnectionCount == 0 )
{
AgentApplication.ClearLogWindow();
}
}
CreateTimings( LoggingFlags );
// Get a new unique handle that this potential connection will be known by
Int32 NewConnectionHandle = GetUniqueHandle();
// Local connection request, the connection ID is the process ID of the caller
LocalConnection NewLocalConnection = new LocalConnection( NewConnectionHandle, LocalProcessID );
// Determine if this connection is a replacement of another, existing connection
foreach( Connection ExistingConnection in Connections.Values )
{
if( ExistingConnection is LocalConnection )
{
LocalConnection ExistingLocalConnection = ExistingConnection as LocalConnection;
if( ExistingLocalConnection.ProcessID == LocalProcessID )
{
// If this process already has a connection, close the older one
// since generally the only way this happens is when a connection
// dies and is re-established later
Log( EVerbosityLevel.Informative, ELogColour.Orange, "[Connection] Detected new local connection from same process ID as an existing one, closing the old one" );
Hashtable OldInParameters = null;
Hashtable OldOutParameters = null;
CloseConnection( ExistingConnection.Handle, OldInParameters, ref OldOutParameters );
}
}
}
// Determine if this connection is a child of another, existing connection
Connection ParentConnection = null;
AgentJob ParentJob = null;
foreach( AgentJob Job in ActiveJobs.Values )
{
if( Job.CurrentState == AgentJob.JobState.AGENT_JOB_RUNNING )
{
// If the incoming connection is from a process spawned by a Job
// associated with an existing connection, note the parent-child
// relationship
lock( Job.ProcessObjectLock )
{
if( ( Job.ProcessObject != null ) &&
( Job.ProcessObject.Id == LocalProcessID ) )
{
// Grab the parent and its Job
ParentConnection = Job.Owner;
ParentJob = Job;
// Found what we want, break out
break;
}
}
// If the Job was marked as manually started, then try to match
// based on the full path name of the executable
EJobTaskFlags JobFlags = Job.Specification.JobFlags;
if( ( JobFlags & EJobTaskFlags.FLAG_MANUAL_START ) != 0 )
{
Process ProcessObject = Process.GetProcessById( LocalProcessID );
string ProcessFilename = Path.GetFileName( ProcessObject.MainModule.FileName );
string OriginalExecutableName;
if( Job.Specification.DependenciesOriginalNames.TryGetValue( Job.Specification.ExecutableName, out OriginalExecutableName ) )
{
// Compare in a way that allows matching with the debug executable, which contains the release executable name
//@todo - detect debug executable in a robust method, possibly through optional dependencies
string OriginalExecutableWithoutExtention = System.IO.Path.GetFileNameWithoutExtension(OriginalExecutableName);
if (OriginalExecutableWithoutExtention.Contains(ProcessFilename) || ProcessFilename.Contains(OriginalExecutableWithoutExtention))
{
Log( EVerbosityLevel.Informative, ELogColour.Green, "[Job] Attaching new process handle to existing Job" );
ProcessObject.EnableRaisingEvents = true;
ProcessObject.Exited += new EventHandler( Job.ExitedProcessEventHandler );
Job.ProcessObject = ProcessObject;
// Grab the parent and its Job
ParentConnection = Job.Owner;
ParentJob = Job;
// Found what we want, break out
break;
}
}
}
}
}
// If we found a parent connection, establish the parent-child relationship
if( ParentConnection != null )
{
Log( EVerbosityLevel.Informative, ELogColour.Green, "[Job] Found a parent connection for PID " + LocalProcessID );
Log( EVerbosityLevel.Informative, ELogColour.Green, String.Format( "[Job] {0:X8} -> {1:X8}", ParentConnection.Handle, NewConnectionHandle ) );
NewLocalConnection.Parent = ParentConnection;
NewLocalConnection.Job = ParentJob;
ParentConnection.LocalChildren.Add( NewConnectionHandle, NewLocalConnection );
ParentConnection.LocalChildrenSeen++;
ParentConnection.ChildrenSeen++;
}
// There are times we want to ensure no new connections are coming online
// where we'll lock the Connections dictionary (see MaintainCache)
LocalConnectionRequestWaiting.Set();
lock( Connections )
{
LocalConnectionRequestWaiting.Reset();
// Add the new local connection to the list of tracked connections
Connections.Add( NewConnectionHandle, NewLocalConnection );
NewLocalConnection.CurrentState = ConnectionState.CONNECTED;
}
// If this is an Instigator's connection, update some state
if( ParentConnection == null )
{
// Update our state and ping the coordinator, if this is an instigating connection
CurrentState = AgentState.Working;
WorkingFor = Environment.MachineName;
PingCoordinator( true );
}
// Fill in the connection configuration output parameter
OutParameters = new Hashtable();
OutParameters["Version"] = ESwarmVersionValue.VER_1_0;
OutParameters["AgentProcessID"] = AgentProcessID;
OutParameters["AgentCachePath"] = GetCacheLocation();
// There are two requirements to be a pure local connection: be local and
// have no remote parents. The main benefit of a pure local connection is
// that they get to avoid calling into the Agent for the Channel API.
bool IsPureLocalConnection = true;
if( NewLocalConnection.Job != null )
{
OutParameters["AgentJobGuid"] = NewLocalConnection.Job.JobGuid;
if( NewLocalConnection.Job.Owner is RemoteConnection )
{
IsPureLocalConnection = false;
}
}
OutParameters["IsPureLocalConnection"] = IsPureLocalConnection;
// Return the handle for the new local connection
return NewConnectionHandle;
}
/**
* Used internally to try to open connections to remote agents
*/
public Int32 TryOpenRemoteConnection( LocalConnection ParentConnection, AgentInfo RemoteAgentInfo, bool IsAssigned, DateTime AssignedTimestamp, out RemoteConnection OpenedRemoteConnection )
{
// Initialize the return values
Int32 ErrorCode = Constants.INVALID;
OpenedRemoteConnection = null;
string RemoteAgentName = RemoteAgentInfo.Name;
string RemoteAgentIPAddress = RemoteAgentInfo.Configuration["IPAddress"].ToString();
Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Connect] Trying to open a remote connection to " + RemoteAgentName + " at " + RemoteAgentIPAddress );
bool bHostNameValid = true;
try
{
ValidateHostName(RemoteAgentIPAddress);
}
catch (Exception)
{
ErrorCode = Constants.ERROR_EXCEPTION;
bHostNameValid = false;
}
// First, ping the remote host to make sure we can talk to it
if (bHostNameValid && PingRemoteHost(RemoteAgentName, RemoteAgentIPAddress))
{
// Get the new unique handle that this connection will be known by
Int32 NewConnectionHandle = GetUniqueHandle();
// Get the remote Agent interface object
string RemoteAgentURL = String.Format( "tcp://{0}:{1}/SwarmAgent",
RemoteAgentIPAddress,
Properties.Settings.Default.AgentRemotingPort.ToString() );
Agent RemoteAgentInterface = ( Agent )Activator.GetObject( typeof( Agent ), RemoteAgentURL );
RemoteConnectionInterfaceWrapper WrappedRemoteAgentInterface = new RemoteConnectionInterfaceWrapper( NewConnectionHandle, RemoteAgentName, RemoteAgentIPAddress, RemoteAgentInterface );
Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Connect] Remote agent interface object obtained for " + RemoteAgentName + " at " + RemoteAgentIPAddress );
// With the interface in hand, try to open a bi-directional connection.
// Create the new RemoteConnection object and add it to the set
// of pending connections, waiting for the incoming confirmation
// which will move it to the active connection set. See
// ConfirmRemoteConnection for where this happens.
RemoteConnection NewRemoteConnection = new RemoteConnection( NewConnectionHandle,
RemoteAgentInfo,
WrappedRemoteAgentInterface );
PendingConnections.Add( NewConnectionHandle, NewRemoteConnection );
// Try to open the connection
try
{
// Send the connection handle and the machine name that will
// be used to construct the return URL for bi-directional
// connection
if( WrappedRemoteAgentInterface.OpenRemoteConnection( Environment.MachineName, NewConnectionHandle, LoggingFlags, IsAssigned, AssignedTimestamp ) >= 0 )
{
// Successful confirmation, double check that the connection has moved
if( PendingConnections.ContainsKey( NewConnectionHandle ) == false )
{
if( Connections.ContainsKey( NewConnectionHandle ) )
{
// Connection was successfully moved to the active connection set
// thus the bi-directional communication link is established
Log( EVerbosityLevel.Informative, ELogColour.Green, "[Connect] Successfully opened a remote connection with " + RemoteAgentName );
// Establish the parent-child relationship
NewRemoteConnection.Parent = ParentConnection;
NewRemoteConnection.Job = ParentConnection.Job;
ParentConnection.RemoteChildren.Add( NewConnectionHandle, NewRemoteConnection );
ParentConnection.RemoteChildrenSeen++;
ParentConnection.ChildrenSeen++;
// Update the visualizer indicating that another remote connection is online
if( NewRemoteConnection.Job.OwnerIsInstigator )
{
AgentApplication.UpdateMachineState( RemoteAgentName, -1, EProgressionState.RemoteConnected );
}
// Return the new handle for this connection
OpenedRemoteConnection = NewRemoteConnection;
ErrorCode = Constants.SUCCESS;
}
else
{
// Woah. Bad news.
}
}
}
else
{
// The connection failed to open, but didn't timeout and die, so it's likely just busy.
// Return with a success code, but with no remote agent, signaling that it's possible
// to try this one again later.
ErrorCode = Constants.SUCCESS;
}
}
catch( Exception Ex )
{
Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Exception: " + RemoteAgentName + ": " + Ex.ToString() );
ErrorCode = Constants.ERROR_EXCEPTION;
}
if( ErrorCode < 0 )
{
Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Failed to open a remote connection with " + RemoteAgentName );
}
// Regardless of how we got here, make sure the connection has been removed
// from the pending set (this is a safe call to make anytime)
PendingConnections.Remove( NewConnectionHandle );
}
else
{
// Failed to ping, simply return with an appropriate error code
ErrorCode = Constants.ERROR_CONNECTION_NOT_FOUND;
}
return ErrorCode;
}
/**
* Requeues the previously unavailable agent list
*/
public bool RetryUnavailableRemoteAgent( LocalConnection ParentConnection )
{
if( ParentConnection.UnavailableRemoteAgents.Count > 0 )
{
// Queue up the next unavailable agent
ParentConnection.PotentialRemoteAgents.Enqueue( ParentConnection.UnavailableRemoteAgents.Dequeue() );
return true;
}
return false;
}
/**
* Refresh the list of active remote agents
*/
public bool ResetPotentialRemoteAgents( LocalConnection ParentConnection )
{
StartTiming( "GetRemoteAgents-Internal", true );
// Start everything off empty
List<AgentInfo> UnattachedRemoteAgents = new List<AgentInfo>();
if( ( Coordinator != null ) &&
( CoordinatorResponding ) &&
( AgentApplication.Options.EnableStandaloneMode == false ) )
{
List<AgentInfo> PotentialRemoteAgents = null;
try
{
// Get all alive agents - but don't connect to them just yet
Hashtable RequestedConfiguration = new Hashtable();
RequestedConfiguration["Version"] = CurrentVersion;
RequestedConfiguration["GroupName"] = AgentApplication.Options.AllowedRemoteAgentGroup;
RequestedConfiguration["RequestingAgentName"] = Environment.MachineName;
RequestedConfiguration["RequestAssignmentFor"] = Environment.MachineName;
PotentialRemoteAgents = Coordinator.GetAvailableAgents( RequestedConfiguration );
}
catch( Exception )
{
// Until all open connections are closed, the coordinator is abandoned
CoordinatorResponding = false;
}
// If we survived the call to the coordinator, work the list
if( ( CoordinatorResponding ) &&
( PotentialRemoteAgents != null ) )
{
// Filter the list by all currently attached remote agents and ourself
foreach( AgentInfo Info in PotentialRemoteAgents )
{
bool WorkerAlreadyConnectedOrDisallowed = false;
if( ( Info.Name == Environment.MachineName ) ||
( !AgentNamePassesAllowedAgentsFilter( Info.Name ) ) )
{
// Trivial case
WorkerAlreadyConnectedOrDisallowed = true;
}
else
{
// Walk through the list of already attached workers
foreach( RemoteConnection Remote in ParentConnection.RemoteChildren.Values )
{
if( Remote.Info.Name == Info.Name )
{
WorkerAlreadyConnectedOrDisallowed = true;
break;
}
}
}
// If not already attached, add it to the new list of potentials
if( !WorkerAlreadyConnectedOrDisallowed )
{
UnattachedRemoteAgents.Add( Info );
}
}
}
}
// Set the newly crafted list of potential workers
ParentConnection.PotentialRemoteAgents = new Queue<AgentInfo>( UnattachedRemoteAgents );
ParentConnection.UnavailableRemoteAgents.Clear();
// Log out which other machines we might be able to ask for help
foreach( AgentInfo NextAgent in UnattachedRemoteAgents )
{
Log( EVerbosityLevel.Complex, ELogColour.Blue, String.Format( "[Connect] Coordinator named remote agent {0} which is {1}", NextAgent.Name, NextAgent.State ) );
}
StopTiming();
return ( UnattachedRemoteAgents.Count > 0 );
}
