public Int32 OpenJob( Connection NewOwner, AgentGuid NewJobGuid ) { // Create the bi-directional link between the job and its owner NewOwner.Job = this; Owner = NewOwner; // Determine if the owner is the Instigator, and if so, perform // any additional work necessary if( NewOwner is LocalConnection ) { OwnerIsInstigator = true; // If the owner of the job is the Instigator, set the current working // directory to be where the Instigator process is executing LocalConnection LocalNewOwner = NewOwner as LocalConnection; Manager.SetCurrentDirectoryByProcessID( LocalNewOwner.ProcessID ); // Update the visualizer AgentApplication.UpdateMachineState( Environment.MachineName, -1, EProgressionState.InstigatorConnected ); } // Add to the Agent-wide list of active jobs JobGuid = NewJobGuid; Manager.ActiveJobs.Add( NewJobGuid, this ); return Constants.SUCCESS; }
/** * Flush the message queue - note this does not assure that it's empty, it just * makes sure that all messages that are in the queue at the time of this method * call are processed. This call is blocking. */ public bool FlushMessageQueue( Connection RequestingConnection, bool WithDisconnect ) { Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Draining message queue for {0:X8}", RequestingConnection.Handle ) ); // This special call will act as a "writer" since we only want to // allow one of these to act at a time. By doing this, we ensure // that all "readers" that use this lock are finished. SendMessageLock.AcquireWriterLock( Timeout.Infinite ); Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock acquired for {0:X8}", RequestingConnection.Handle ) ); // Flush the message queue with a signal message, optionally with a disconnection event AgentSignalMessage SignalMessage; if( WithDisconnect ) { SignalMessage = new DisconnectionSignalMessage( RequestingConnection ); } else { SignalMessage = new AgentSignalMessage(); } Int32 SignalMessageSent = SendMessageInternal( RequestingConnection, SignalMessage ); // Release our "writer" lock SendMessageLock.ReleaseWriterLock(); Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Lock released for {0:X8}", RequestingConnection.Handle ) ); if( SignalMessageSent == Constants.SUCCESS ) { SignalMessage.ResetEvent.WaitOne( Timeout.Infinite ); Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, String.Format( "[FlushMessageQueue] Drain complete for {0:X8}", RequestingConnection.Handle ) ); return true; } else { Log( EVerbosityLevel.ExtraVerbose, ELogColour.Green, "[FlushMessageQueue] Drain failed because of an error sending the signal message" ); return false; } }
private Int32 SetChannelHandleAndAdd( Connection ConnectionThatOwnsChannel, Channel NewChannel ) { Int32 NewHandleValue; lock( SetChannelHandleAndAddLock ) { // Generate a new random value for the handle do { NewHandleValue = SetChannelHandleAndAddGenerator.Next(); } // Keep generating new values until we find one not already in use while( ConnectionThatOwnsChannel.OpenChannels.ContainsKey( NewHandleValue ) ); // Now set the new handle value and add to the set NewChannel.SetHandle( NewHandleValue ); ConnectionThatOwnsChannel.OpenChannels.Add( NewHandleValue, NewChannel ); } return NewHandleValue; }
private bool RequestTaskFiles( Connection RequestingConnection, AgentTaskSpecification TaskSpecification ) { // Check for and possibly request each dependency if( TaskSpecification.Dependencies != null ) { foreach( string Dependency in TaskSpecification.Dependencies ) { if( !RequestDependency( RequestingConnection, Dependency, true ) ) { return false; } } } return true; }
private bool RequestJobFiles( Connection RequestingConnection, AgentJobSpecification JobSpecification ) { // Check for and possibly request the executable if( !RequestDependency( RequestingConnection, JobSpecification.ExecutableName, true ) ) { return false; } // Check for and possibly request each dependency if( JobSpecification.RequiredDependencies != null ) { foreach( string Dependency in JobSpecification.RequiredDependencies ) { if( !RequestDependency( RequestingConnection, Dependency, true ) ) { return false; } } } if( JobSpecification.OptionalDependencies != null ) { foreach( string Dependency in JobSpecification.OptionalDependencies ) { if( !RequestDependency( RequestingConnection, Dependency, false ) ) { return false; } } } return true; }
public void CancelReservations( Connection ReservationHolder ) { // We need the state to not change while we're in here lock( CurrentStateLock ) { TaskReservationCount -= ReservationHolder.ReservationCount; ReservationHolder.ReservationCount = 0; } }
public AgentTaskRequestResponse GetNextTask( Connection RequestingConnection ) { // We need the state to not change while we're in here lock( CurrentStateLock ) { // Take the next Task off of the Pending list and add it to the Active list if( CurrentState == JobState.AGENT_JOB_RUNNING ) { lock( PendingTasks ) { AgentTask NextTask = null; // Check for whether we're running in deterministic mode if( DeterministicModeEnabled ) { // If so, look up the requesting connection's task queue Queue<AgentTask> TaskQueue; string RequestorName = Manager.MachineNameFromConnection( RequestingConnection ); if( Manager.LastSuccessfulJobRecord.AgentToTaskQueueMapping.TryGetValue( RequestorName, out TaskQueue ) ) { if( TaskQueue.Count > 0 ) { // Grab the next pending task NextTask = TaskQueue.Dequeue(); } } } // Otherwise, we're in a standard distribution mode, see if there // are tasks available to hand out else if( PendingTasks.Count > 0 ) { // Check to see if should avoid the next task if( !AvoidNextTask( RequestingConnection ) ) { NextTask = PendingTasks.Pop(); } } if( NextTask != null ) { // Assign the new owner and add the task to the running sets NextTask.CurrentOwner = RequestingConnection; // Set the assign time of the task as well as the start time, // in case they don't send back a RUNNING message, which will // set the proper start time NextTask.AssignTime = DateTime.UtcNow; NextTask.StartTime = DateTime.UtcNow; // Update additional stats if( RequestingConnection is RemoteConnection ) { TaskCountRemote++; } AgentTaskSpecification TaskSpecification = NextTask.Specification; RequestingConnection.RunningTasks.Add( TaskSpecification.TaskGuid, NextTask ); RunningTasks.Add( TaskSpecification.TaskGuid, NextTask ); // Send back the now ready task specification return TaskSpecification; } // If we're not the final authority of this job, then we'll send back a // reservation and allow the Instigator send the real answer later. // Alternately, if the owner is the Instigator, we also need to send // back a reservation to make sure the local job application runs until // the job is done. else if( ( OwnerIsInstigator == false ) || ( RequestingConnection is LocalConnection ) ) { // Always track outstanding reservations RequestingConnection.ReservationCount++; TaskReservationCount++; // For local connections, send a RESERVATION response which indicates that we // have no work currently to hand out, but we might in the future, so sit // tight. For remote connections, we only need to track the reservation if( RequestingConnection is LocalConnection ) { return ( new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RESERVATION ) ); } } // Otherwise, we're all out of tasks, so release the worker else { // No more distributed tasks to hand out, send back a release return ( new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE ) ); } } } else if( CurrentState == JobState.AGENT_JOB_CLOSED ) { // No more distributed tasks to hand out, send back a release return ( new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE ) ); } } return null; }
private bool AvoidNextTask( Connection RequestingConection ) { Debug.Assert( RequestingConection.Job != null ); bool SafeToAvoidNextTask = ( AgentApplication.Options.AvoidLocalExecution ) && ( RequestingConection is LocalConnection ) && ( RequestingConection.Job.Owner.RemoteChildren.Count > 0 ); return SafeToAvoidNextTask; }
private bool RequestDependency(Connection RequestingConnection, string Dependency, bool bIsRequired) { Hashtable InParameters = new Hashtable(); InParameters["Version"] = ESwarmVersionValue.VER_1_0; InParameters["ChannelName"] = Dependency; Hashtable OutParameters = null; if (Manager.TestChannel(RequestingConnection.Handle, InParameters, ref OutParameters) < 0) { if (RequestingConnection is RemoteConnection) { // For remote connection misses, try to pull the file from the instigator Manager.Log(EVerbosityLevel.Verbose, ELogColour.Green, "[Job] Attempting to pull file from remote Agent: " + Dependency); if (Manager.PullChannel(RequestingConnection as RemoteConnection, Dependency, null, 5) == false) { if (bIsRequired) { Manager.Log(EVerbosityLevel.Informative, ELogColour.Red, "[Job] Error: Failed to pull a required file from remote Agent: " + Dependency); return false; } else { Manager.Log(EVerbosityLevel.Verbose, ELogColour.Orange, "[Job] Warning: Failed to pull an optional file from remote Agent: " + Dependency); } } } else { Debug.Assert(RequestingConnection is LocalConnection); if (bIsRequired) { // Always fail on local connection misses Manager.Log(EVerbosityLevel.Informative, ELogColour.Red, "[Job] Error: Failed to find required file: " + Dependency); return false; } else { Manager.Log(EVerbosityLevel.Verbose, ELogColour.Orange, "[Job] Warning: Failed to find an optional file: " + Dependency); } } } return true; }
/* * Get the machine IP address from the connection structure */ public string MachineIPAddressFromConnection( Connection RequestingConnection ) { string MachineIPAddress = ""; RemoteConnection Remote = RequestingConnection as RemoteConnection; if( Remote == null ) { MachineIPAddress = "127.0.0.1"; } else { MachineIPAddress = Remote.Interface.RemoteAgentIPAddress; } return ( MachineIPAddress ); }
public DisconnectionSignalMessage( Connection InConnectionToDisconnect ) { ConnectionToDisconnect = InConnectionToDisconnect; }
/** * Actual work performed by SendMessage happens here */ public Int32 SendMessageInternal( Connection Sender, AgentMessage NewMessage ) { Int32 ErrorCode = Constants.INVALID; // We assume the message is valid, but if somewhere below we change our // mind, this value will be set to false and we'll eat the message bool bMessageIsValid = true; // Logic for the setting of the To and From fields (if not already set) // // All connections sending messages are implicitly sending them to the // Instigator of the Job they're working on. Depending on where the // message is coming from, we might need to patch up the To field a // little bit to ensure it's heading to a Local connection when it // needs to be. The only case this applies is when we receive a message // from a Remote connection, directed toward a Remote connection, which // happens when we get a message from a Remote Agent from an even more // Remote connection (i.e. a Job connection on the remote machine): // // To Local, From Local -> routing of main message and replies are ok // To Local, From Remote -> routing of main message and replies are ok // To Remote, From Local -> routing of main message and replies are ok // To Remote, From Remote -> routing of replies is ok, but the main // message is in trouble if it's not completely handled within the // Agent's ProcessMessages routine. It would be forwarded on to the // To field connection which is Remote and we'd end up bouncing the // message back and forth forever. Need to adjust the To field to // point to the parent of the To connection (which is where it's // intended to go anyway). See further below for where we handle // this case. // If the From field is not set, give it the connection handle value by // default so that any response message will automatically be routed back // to it whether it's the sender or the recipient if( NewMessage.From == Constants.INVALID ) { NewMessage.From = Sender.Handle; } // If the connection used to send the message is Remote, check for the // Remote -> Remote case described above else if( Sender is RemoteConnection ) { // If the From field is already set, see if we've already registered // the connection this message is being sent from Connection FromConnection; if( !Connections.TryGetValue( NewMessage.From, out FromConnection ) ) { // This is a new one - make it an alias for the sender so that any // responses will be directed back via its sending interface RemoteConnection RemoteSender = Sender as RemoteConnection; RemoteSender.Aliases.Add( NewMessage.From ); // There are times we want to ensure no new connections are coming online // where we'll lock the Connections dictionary (see MaintainCache) lock( Connections ) { Connections.Add( NewMessage.From, RemoteSender ); } FromConnection = RemoteSender; string LogMessage = String.Format( "[SendMessage] Added alias for remote connection: {0:X8} is an alias for {1:X8}", NewMessage.From, Sender.Handle ); Log( EVerbosityLevel.Informative, ELogColour.Green, LogMessage ); } // If this is a Remote -> Remote situation, the proper place to route // the message to the parent of the remote connection since the Agents // generally act as glue between connections if( FromConnection is RemoteConnection ) { Debug.Assert( NewMessage.To != Constants.INVALID ); Connection ToConnection; if( (Connections.TryGetValue( NewMessage.To, out ToConnection )) && (ToConnection is RemoteConnection) ) { Connection ToConnectionParent = ToConnection.Parent; if( ToConnectionParent != null ) { NewMessage.To = ToConnectionParent.Handle; } } } } // If the To field is not set, assign it based on the message type if( NewMessage.To == Constants.INVALID ) { // TODO: As we add additional versions, convert to a switch rather than if-else. // For now, just use a simple if since we only have one version and a switch is // overkill. if( NewMessage.Version == ESwarmVersionValue.VER_1_0 ) { // The default is for messages to be ultimately routed to the Instigator // unless the message is one of a certain set of types that route // directly to the connection specified switch( NewMessage.Type ) { // These message types need to be routed to the connection specified // either because they are meant to be simple round-trip messages or // because they are sent from within Swarm directly to the connection // and should not be routed anywhere else case EMessageType.QUIT: case EMessageType.PING: case EMessageType.SIGNAL: NewMessage.To = Sender.Handle; break; // These message types need to be routed eventually to the Instigator // connection, which is the ultimate ancestor up the parent chain, so // simply assign the most senior parent we have case EMessageType.INFO: case EMessageType.ALERT: case EMessageType.TIMING: case EMessageType.TASK_REQUEST: case EMessageType.TASK_STATE: case EMessageType.JOB_STATE: // By default, make the sender the recipient for these cases, in // case the parent is no longer active NewMessage.To = Sender.Handle; Connection SenderParent = Sender.Parent; if( SenderParent != null ) { // If we have a parent connection and it's active, then // assign it as the recipient if( SenderParent.CurrentState == ConnectionState.CONNECTED ) { NewMessage.To = SenderParent.Handle; } } break; // These message types are not expected and are each error cases case EMessageType.NONE: // Should never be set to this case EMessageType.JOB_SPECIFICATION: // Only used for messages going directly into OpenJob case EMessageType.TASK_REQUEST_RESPONSE: // Should always have the To field set already default: Log( EVerbosityLevel.Informative, ELogColour.Orange, "SendMessage: Invalid message type received, ignoring " + NewMessage.Type.ToString() ); break; } // If still no assigned To field, consider it an error if( NewMessage.To == Constants.INVALID ) { Log( EVerbosityLevel.Informative, ELogColour.Orange, "SendMessage: No proper recipient found, ignoring " + NewMessage.Type.ToString() ); bMessageIsValid = false; } } } // If the message remains valid, post it to the queue if( bMessageIsValid ) { lock( MessageQueueLock ) { Debug.Assert( NewMessage != null ); MessageQueueSM.Enqueue( NewMessage ); string NewLogMessage = String.Format( "Step 1 of N for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Agent)", NewMessage.To, NewMessage.From, NewMessage.Type, MessageQueueSM.Count ); Log( EVerbosityLevel.SuperVerbose, ELogColour.Green, NewLogMessage ); MessageQueueReady.Set(); } ErrorCode = Constants.SUCCESS; } else { Log( EVerbosityLevel.Informative, ELogColour.Orange, String.Format( "SendMessage: Discarded message \"{0}\"", NewMessage.Type ) ); } return ( ErrorCode ); }
/* * Get the machine name from the connection structure */ public string MachineNameFromConnection( Connection RequestingConnection ) { string Name = ""; RemoteConnection Remote = RequestingConnection as RemoteConnection; if( Remote == null ) { Name = Environment.MachineName; } else { Name = Remote.Info.Name; } return ( Name ); }
public AgentTask( AgentTaskSpecification NewSpecification ) { Specification = NewSpecification; CurrentState = new AgentTaskState( NewSpecification.JobGuid, NewSpecification.TaskGuid, EJobTaskState.TASK_STATE_IDLE ); CurrentOwner = null; }