public void AbandonPotentialRemoteAgents( LocalConnection ParentConnection ) { lock( AbandonPotentialRemoteAgentsLock ) { // Walk through the list of potential agents and disconnect each one foreach( AgentInfo NextRemoteAgent in ParentConnection.PotentialRemoteAgents ) { AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.RemoteDisconnected ); } // Done with the queue now, clear it ParentConnection.PotentialRemoteAgents.Clear(); foreach( AgentInfo NextRemoteAgent in ParentConnection.UnavailableRemoteAgents ) { AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.RemoteDisconnected ); } // Done with the queue now, clear it ParentConnection.UnavailableRemoteAgents.Clear(); // Finished with remote agents, set the flag ParentConnection.RemoteAgentsAbandoned = true; } }
/** * Pulls the next agent off the list of potentials and tries to open a connection * until it either opens one successfully or runs out of agents */ public RemoteConnection GetNextRemoteAgent( LocalConnection ParentConnection ) { // Walk through the list of potential agents and try to open connections // until we either find one or run out of agents RemoteConnection Remote = null; AgentJob Job = ParentConnection.Job; while( ( Job != null ) && ( Job.CurrentState == AgentJob.JobState.AGENT_JOB_RUNNING ) && ( ParentConnection.PotentialRemoteAgents.Count > 0 ) && ( Remote == null ) ) { AgentInfo NextRemoteAgent = ParentConnection.PotentialRemoteAgents.Dequeue(); bool IsAssigned = false; DateTime AssignedTimestamp = DateTime.MinValue; if( ( NextRemoteAgent.Configuration.ContainsKey( "AssignedTo" ) ) && ( ( NextRemoteAgent.Configuration["AssignedTo"] as string ) == Environment.MachineName ) ) { IsAssigned = true; AssignedTimestamp = ( DateTime )NextRemoteAgent.Configuration["AssignedTime"]; } // If we failed to open the connection, but the agent is alive, add it to the list to try again later if( ( TryOpenRemoteConnection( ParentConnection, NextRemoteAgent, IsAssigned, AssignedTimestamp, out Remote ) == Constants.SUCCESS ) && ( Remote == null ) ) { AgentApplication.UpdateMachineState( NextRemoteAgent.Name, -1, EProgressionState.Blocked ); ParentConnection.UnavailableRemoteAgents.Enqueue( NextRemoteAgent ); } } // Return whatever we found return Remote; }
/** * Used by local processes to open a connection to Swarm */ private Int32 OpenConnection_1_0( Hashtable InParameters, ref Hashtable OutParameters ) { // Always wait until the agent is fully initialized Initialized.WaitOne(); // Unpack the input parameters Int32 LocalProcessID = ( Int32 )InParameters["ProcessID"]; ELogFlags LoggingFlags = ( ELogFlags )InParameters["LoggingFlags"]; // Check for optional parameters if( InParameters.Contains( "ProcessIsOwner" ) ) { bool LocalProcessIsOwner = ( bool )InParameters["ProcessIsOwner"]; if( LocalProcessIsOwner ) { OwnerProcessID = LocalProcessID; } } // If a restart or an exit has been requested, stop accepting new connections if( AgentIsShuttingDown ) { return Constants.INVALID; } // If this is a "first" new local connection, clear the log window LocalConnectionRequestWaiting.Set(); lock( Connections ) { LocalConnectionRequestWaiting.Reset(); Int32 LocalConnectionCount = 0; foreach( Connection NextConnection in Connections.Values ) { if( NextConnection is LocalConnection ) { LocalConnectionCount++; } } if( LocalConnectionCount == 0 ) { AgentApplication.ClearLogWindow(); } } CreateTimings( LoggingFlags ); // Get a new unique handle that this potential connection will be known by Int32 NewConnectionHandle = GetUniqueHandle(); // Local connection request, the connection ID is the process ID of the caller LocalConnection NewLocalConnection = new LocalConnection( NewConnectionHandle, LocalProcessID ); // Determine if this connection is a replacement of another, existing connection foreach( Connection ExistingConnection in Connections.Values ) { if( ExistingConnection is LocalConnection ) { LocalConnection ExistingLocalConnection = ExistingConnection as LocalConnection; if( ExistingLocalConnection.ProcessID == LocalProcessID ) { // If this process already has a connection, close the older one // since generally the only way this happens is when a connection // dies and is re-established later Log( EVerbosityLevel.Informative, ELogColour.Orange, "[Connection] Detected new local connection from same process ID as an existing one, closing the old one" ); Hashtable OldInParameters = null; Hashtable OldOutParameters = null; CloseConnection( ExistingConnection.Handle, OldInParameters, ref OldOutParameters ); } } } // Determine if this connection is a child of another, existing connection Connection ParentConnection = null; AgentJob ParentJob = null; foreach( AgentJob Job in ActiveJobs.Values ) { if( Job.CurrentState == AgentJob.JobState.AGENT_JOB_RUNNING ) { // If the incoming connection is from a process spawned by a Job // associated with an existing connection, note the parent-child // relationship lock( Job.ProcessObjectLock ) { if( ( Job.ProcessObject != null ) && ( Job.ProcessObject.Id == LocalProcessID ) ) { // Grab the parent and its Job ParentConnection = Job.Owner; ParentJob = Job; // Found what we want, break out break; } } // If the Job was marked as manually started, then try to match // based on the full path name of the executable EJobTaskFlags JobFlags = Job.Specification.JobFlags; if( ( JobFlags & EJobTaskFlags.FLAG_MANUAL_START ) != 0 ) { Process ProcessObject = Process.GetProcessById( LocalProcessID ); string ProcessFilename = Path.GetFileName( ProcessObject.MainModule.FileName ); string OriginalExecutableName; if( Job.Specification.DependenciesOriginalNames.TryGetValue( Job.Specification.ExecutableName, out OriginalExecutableName ) ) { // Compare in a way that allows matching with the debug executable, which contains the release executable name //@todo - detect debug executable in a robust method, possibly through optional dependencies string OriginalExecutableWithoutExtention = System.IO.Path.GetFileNameWithoutExtension(OriginalExecutableName); if (OriginalExecutableWithoutExtention.Contains(ProcessFilename) || ProcessFilename.Contains(OriginalExecutableWithoutExtention)) { Log( EVerbosityLevel.Informative, ELogColour.Green, "[Job] Attaching new process handle to existing Job" ); ProcessObject.EnableRaisingEvents = true; ProcessObject.Exited += new EventHandler( Job.ExitedProcessEventHandler ); Job.ProcessObject = ProcessObject; // Grab the parent and its Job ParentConnection = Job.Owner; ParentJob = Job; // Found what we want, break out break; } } } } } // If we found a parent connection, establish the parent-child relationship if( ParentConnection != null ) { Log( EVerbosityLevel.Informative, ELogColour.Green, "[Job] Found a parent connection for PID " + LocalProcessID ); Log( EVerbosityLevel.Informative, ELogColour.Green, String.Format( "[Job] {0:X8} -> {1:X8}", ParentConnection.Handle, NewConnectionHandle ) ); NewLocalConnection.Parent = ParentConnection; NewLocalConnection.Job = ParentJob; ParentConnection.LocalChildren.Add( NewConnectionHandle, NewLocalConnection ); ParentConnection.LocalChildrenSeen++; ParentConnection.ChildrenSeen++; } // There are times we want to ensure no new connections are coming online // where we'll lock the Connections dictionary (see MaintainCache) LocalConnectionRequestWaiting.Set(); lock( Connections ) { LocalConnectionRequestWaiting.Reset(); // Add the new local connection to the list of tracked connections Connections.Add( NewConnectionHandle, NewLocalConnection ); NewLocalConnection.CurrentState = ConnectionState.CONNECTED; } // If this is an Instigator's connection, update some state if( ParentConnection == null ) { // Update our state and ping the coordinator, if this is an instigating connection CurrentState = AgentState.Working; WorkingFor = Environment.MachineName; PingCoordinator( true ); } // Fill in the connection configuration output parameter OutParameters = new Hashtable(); OutParameters["Version"] = ESwarmVersionValue.VER_1_0; OutParameters["AgentProcessID"] = AgentProcessID; OutParameters["AgentCachePath"] = GetCacheLocation(); // There are two requirements to be a pure local connection: be local and // have no remote parents. The main benefit of a pure local connection is // that they get to avoid calling into the Agent for the Channel API. bool IsPureLocalConnection = true; if( NewLocalConnection.Job != null ) { OutParameters["AgentJobGuid"] = NewLocalConnection.Job.JobGuid; if( NewLocalConnection.Job.Owner is RemoteConnection ) { IsPureLocalConnection = false; } } OutParameters["IsPureLocalConnection"] = IsPureLocalConnection; // Return the handle for the new local connection return NewConnectionHandle; }
/** * Used internally to try to open connections to remote agents */ public Int32 TryOpenRemoteConnection( LocalConnection ParentConnection, AgentInfo RemoteAgentInfo, bool IsAssigned, DateTime AssignedTimestamp, out RemoteConnection OpenedRemoteConnection ) { // Initialize the return values Int32 ErrorCode = Constants.INVALID; OpenedRemoteConnection = null; string RemoteAgentName = RemoteAgentInfo.Name; string RemoteAgentIPAddress = RemoteAgentInfo.Configuration["IPAddress"].ToString(); Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Connect] Trying to open a remote connection to " + RemoteAgentName + " at " + RemoteAgentIPAddress ); bool bHostNameValid = true; try { ValidateHostName(RemoteAgentIPAddress); } catch (Exception) { ErrorCode = Constants.ERROR_EXCEPTION; bHostNameValid = false; } // First, ping the remote host to make sure we can talk to it if (bHostNameValid && PingRemoteHost(RemoteAgentName, RemoteAgentIPAddress)) { // Get the new unique handle that this connection will be known by Int32 NewConnectionHandle = GetUniqueHandle(); // Get the remote Agent interface object string RemoteAgentURL = String.Format( "tcp://{0}:{1}/SwarmAgent", RemoteAgentIPAddress, Properties.Settings.Default.AgentRemotingPort.ToString() ); Agent RemoteAgentInterface = ( Agent )Activator.GetObject( typeof( Agent ), RemoteAgentURL ); RemoteConnectionInterfaceWrapper WrappedRemoteAgentInterface = new RemoteConnectionInterfaceWrapper( NewConnectionHandle, RemoteAgentName, RemoteAgentIPAddress, RemoteAgentInterface ); Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Connect] Remote agent interface object obtained for " + RemoteAgentName + " at " + RemoteAgentIPAddress ); // With the interface in hand, try to open a bi-directional connection. // Create the new RemoteConnection object and add it to the set // of pending connections, waiting for the incoming confirmation // which will move it to the active connection set. See // ConfirmRemoteConnection for where this happens. RemoteConnection NewRemoteConnection = new RemoteConnection( NewConnectionHandle, RemoteAgentInfo, WrappedRemoteAgentInterface ); PendingConnections.Add( NewConnectionHandle, NewRemoteConnection ); // Try to open the connection try { // Send the connection handle and the machine name that will // be used to construct the return URL for bi-directional // connection if( WrappedRemoteAgentInterface.OpenRemoteConnection( Environment.MachineName, NewConnectionHandle, LoggingFlags, IsAssigned, AssignedTimestamp ) >= 0 ) { // Successful confirmation, double check that the connection has moved if( PendingConnections.ContainsKey( NewConnectionHandle ) == false ) { if( Connections.ContainsKey( NewConnectionHandle ) ) { // Connection was successfully moved to the active connection set // thus the bi-directional communication link is established Log( EVerbosityLevel.Informative, ELogColour.Green, "[Connect] Successfully opened a remote connection with " + RemoteAgentName ); // Establish the parent-child relationship NewRemoteConnection.Parent = ParentConnection; NewRemoteConnection.Job = ParentConnection.Job; ParentConnection.RemoteChildren.Add( NewConnectionHandle, NewRemoteConnection ); ParentConnection.RemoteChildrenSeen++; ParentConnection.ChildrenSeen++; // Update the visualizer indicating that another remote connection is online if( NewRemoteConnection.Job.OwnerIsInstigator ) { AgentApplication.UpdateMachineState( RemoteAgentName, -1, EProgressionState.RemoteConnected ); } // Return the new handle for this connection OpenedRemoteConnection = NewRemoteConnection; ErrorCode = Constants.SUCCESS; } else { // Woah. Bad news. } } } else { // The connection failed to open, but didn't timeout and die, so it's likely just busy. // Return with a success code, but with no remote agent, signaling that it's possible // to try this one again later. ErrorCode = Constants.SUCCESS; } } catch( Exception Ex ) { Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Exception: " + RemoteAgentName + ": " + Ex.ToString() ); ErrorCode = Constants.ERROR_EXCEPTION; } if( ErrorCode < 0 ) { Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Failed to open a remote connection with " + RemoteAgentName ); } // Regardless of how we got here, make sure the connection has been removed // from the pending set (this is a safe call to make anytime) PendingConnections.Remove( NewConnectionHandle ); } else { // Failed to ping, simply return with an appropriate error code ErrorCode = Constants.ERROR_CONNECTION_NOT_FOUND; } return ErrorCode; }
/** * Requeues the previously unavailable agent list */ public bool RetryUnavailableRemoteAgent( LocalConnection ParentConnection ) { if( ParentConnection.UnavailableRemoteAgents.Count > 0 ) { // Queue up the next unavailable agent ParentConnection.PotentialRemoteAgents.Enqueue( ParentConnection.UnavailableRemoteAgents.Dequeue() ); return true; } return false; }
/** * Refresh the list of active remote agents */ public bool ResetPotentialRemoteAgents( LocalConnection ParentConnection ) { StartTiming( "GetRemoteAgents-Internal", true ); // Start everything off empty List<AgentInfo> UnattachedRemoteAgents = new List<AgentInfo>(); if( ( Coordinator != null ) && ( CoordinatorResponding ) && ( AgentApplication.Options.EnableStandaloneMode == false ) ) { List<AgentInfo> PotentialRemoteAgents = null; try { // Get all alive agents - but don't connect to them just yet Hashtable RequestedConfiguration = new Hashtable(); RequestedConfiguration["Version"] = CurrentVersion; RequestedConfiguration["GroupName"] = AgentApplication.Options.AllowedRemoteAgentGroup; RequestedConfiguration["RequestingAgentName"] = Environment.MachineName; RequestedConfiguration["RequestAssignmentFor"] = Environment.MachineName; PotentialRemoteAgents = Coordinator.GetAvailableAgents( RequestedConfiguration ); } catch( Exception ) { // Until all open connections are closed, the coordinator is abandoned CoordinatorResponding = false; } // If we survived the call to the coordinator, work the list if( ( CoordinatorResponding ) && ( PotentialRemoteAgents != null ) ) { // Filter the list by all currently attached remote agents and ourself foreach( AgentInfo Info in PotentialRemoteAgents ) { bool WorkerAlreadyConnectedOrDisallowed = false; if( ( Info.Name == Environment.MachineName ) || ( !AgentNamePassesAllowedAgentsFilter( Info.Name ) ) ) { // Trivial case WorkerAlreadyConnectedOrDisallowed = true; } else { // Walk through the list of already attached workers foreach( RemoteConnection Remote in ParentConnection.RemoteChildren.Values ) { if( Remote.Info.Name == Info.Name ) { WorkerAlreadyConnectedOrDisallowed = true; break; } } } // If not already attached, add it to the new list of potentials if( !WorkerAlreadyConnectedOrDisallowed ) { UnattachedRemoteAgents.Add( Info ); } } } } // Set the newly crafted list of potential workers ParentConnection.PotentialRemoteAgents = new Queue<AgentInfo>( UnattachedRemoteAgents ); ParentConnection.UnavailableRemoteAgents.Clear(); // Log out which other machines we might be able to ask for help foreach( AgentInfo NextAgent in UnattachedRemoteAgents ) { Log( EVerbosityLevel.Complex, ELogColour.Blue, String.Format( "[Connect] Coordinator named remote agent {0} which is {1}", NextAgent.Name, NextAgent.State ) ); } StopTiming(); return ( UnattachedRemoteAgents.Count > 0 ); }