/** * Takes messages off the internal message queue and handles them by either * sending responses, forwarding the message on, or processing it internally */ private void ProcessMessagesThreadProc() { // A response message queue used to send messages back to the one which sent it Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>(); while (AgentHasShutDown == false) { StartTiming("ProcessMessage-Internal", true); lock ( MessageQueueLock ) { // Swap the SM and PM message queue to keep things moving at full speed Queue <AgentMessage> Temp = MessageQueuePM; MessageQueuePM = MessageQueueSM; MessageQueueSM = Temp; } // Process all messages currently in the queue while (MessageQueuePM.Count > 0) { Debug.Assert(ResponseMessageQueue.Count == 0); // Get and process the next message AgentMessage NextMessage = MessageQueuePM.Dequeue(); Debug.Assert(NextMessage != null); bool bMessageHandled = false; switch (NextMessage.Type) { case EMessageType.SIGNAL: { if (NextMessage is DisconnectionSignalMessage) { // Mark the connection as inactive DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage; Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle)); DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED; DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow; } // Signal the message and move on AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage; SignalMessage.ResetEvent.Set(); bMessageHandled = true; } break; case EMessageType.TIMING: { Connection FromConnection; if ((Connections.TryGetValue(NextMessage.From,out FromConnection))) { Connection ToConnection; if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) && (ToConnection is LocalConnection)) { // Handle message AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage; AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State); bMessageHandled = true; } } } break; case EMessageType.TASK_REQUEST: { // Look up the requesting connection Debug.Assert(NextMessage.From != Constants.INVALID); Connection RequestingConnection; if (Connections.TryGetValue(NextMessage.From,out RequestingConnection)) { // Look up the specified Job AgentJob JobToAskForTasks = RequestingConnection.Job; if (JobToAskForTasks != null) { // If we get a valid response back, add it to the queue AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection); if (Response != null) { ResponseMessageQueue.Enqueue(Response); // Specifications and releases are always handled here, but // reservations are special in that we will send a reservation // back to local connections but we'll need to make sure the // message continues on to remote connections. if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) || (Response.ResponseType == ETaskRequestResponseType.RELEASE) || ((Response.ResponseType == ETaskRequestResponseType.RESERVATION) && (JobToAskForTasks.Owner is LocalConnection))) { bMessageHandled = true; } } } else { // Unable to find the Job, just send back a release message Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed"); //ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid, // ETaskRequestResponseType.RELEASE ) ); bMessageHandled = true; } } else { // Unable to find the connection, swallow the request Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request"); bMessageHandled = true; } } break; case EMessageType.TASK_STATE: { // Look up the sending connection Debug.Assert(NextMessage.From != Constants.INVALID); Connection SendingConnection; if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) && (SendingConnection.Job != null)) { // Look up the specified Job AgentJob UpdatedJob; if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob)) { AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState; UpdatedJob.UpdateTaskState(UpdatedTaskState); if (UpdatedJob.Owner is LocalConnection) { // If the Task state change is of a type potentially interesting to // the Instigator, return it switch (UpdatedTaskState.TaskState) { case EJobTaskState.TASK_STATE_INVALID: case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS: case EJobTaskState.TASK_STATE_COMPLETE_FAILURE: // For these message types, allow the message to continue on break; default: // Nothing to do otherwise, mark the message as handled bMessageHandled = true; break; } } else { // Always send messages on for remote connections } } else { // Unable to find the Job, swallow the request Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request"); bMessageHandled = true; } } else { // Unable to find the connection, swallow the request Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request"); bMessageHandled = true; } } break; } // If the message was not handled completely, send it on if (bMessageHandled == false) { // Look up who the message is being sent to and make sure they're // still active and if not, ignore the message Connection Recipient; Debug.Assert(NextMessage.To != Constants.INVALID); if (Connections.TryGetValue(NextMessage.To,out Recipient)) { if (Recipient is LocalConnection) { // If the recipient is local, place it in the proper queue // and signal that a message is ready LocalConnection LocalRecipient = Recipient as LocalConnection; lock (LocalRecipient.MessageQueue) { LocalRecipient.MessageQueue.Enqueue(NextMessage); string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)", NextMessage.To, NextMessage.From, NextMessage.Type, LocalRecipient.MessageQueue.Count); Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage); LocalRecipient.MessageAvailableSignal(); } } else { Debug.Assert(Recipient is RemoteConnection); // If the recipient is remote, send the message via SendMessage // unless the message is a Task being sent back, which is sent // via the dedicated Task API RemoteConnection RemoteRecipient = Recipient as RemoteConnection; if (NextMessage is AgentTaskSpecification) { // All new Tasks are sent via the dedicated Task API AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification; Hashtable RemoteInParameters = new Hashtable(); RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0; RemoteInParameters["Specification"] = TaskSpecification; Hashtable RemoteOutParameters = null; Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters); if (Error >= 0) { // Perhaps we should be sending an accept message back? } else { AgentTaskState UpdateMessage; if (Error == Constants.ERROR_CONNECTION_DISCONNECTED) { // Special case of the connection dropping while we're adding the // task, say it's been killed to requeue UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid, TaskSpecification.TaskGuid, EJobTaskState.TASK_STATE_KILLED); } else { // All other error cases will be rejections UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid, TaskSpecification.TaskGuid, EJobTaskState.TASK_STATE_REJECTED); } AgentJob Job; if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job)) { Job.UpdateTaskState(UpdateMessage); } } } else { // All standard messages are sent via the SendMessage API Hashtable RemoteInParameters = new Hashtable(); RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0; RemoteInParameters["Message"] = NextMessage; Hashtable RemoteOutParameters = null; RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters); } string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)", NextMessage.To, NextMessage.From, NextMessage.Type); Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage); } } else { Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString()); } } // If there are any responses to the message, send them if (ResponseMessageQueue.Count > 0) { foreach (AgentMessage NextResponse in ResponseMessageQueue) { // For each one of the messages, set the routing fields properly NextResponse.To = NextMessage.From; NextResponse.From = NextMessage.To; // And then queue the message back up immediately MessageQueuePM.Enqueue(NextResponse); } ResponseMessageQueue.Clear(); } } StopTiming(); // Wait for a message to become available and once unlocked, swap the queues // and check for messages to process. Set a timeout, so we'll wake up every // now and then to check for a quit signal at least MessageQueueReady.WaitOne(500); } }
public Int32 AddTask( AgentTaskSpecification NewTaskSpecification ) { Int32 ErrorCode = Constants.INVALID; // Using the provided specification, request all necessary Task files. For local // Agents, the executable and dependencies must already exist in the cache. For // remote Agents, we'll need request the necessary files from the Instigator. if( RequestTaskFiles( Owner, NewTaskSpecification ) ) { lock( PendingTasks ) { Manager.Log( EVerbosityLevel.Verbose, ELogColour.Green, "[Job] AddTask: Adding \"" + NewTaskSpecification.Parameters + "\" (" + NewTaskSpecification.TaskGuid.ToString() + ") with cost " + NewTaskSpecification.Cost.ToString() ); // Queue the Task PendingTasks.Push( new AgentTask( NewTaskSpecification ) ); TaskCount++; // Sanity check a couple assumptions here if( PendingTasks.Count > 1 ) { Debug.Assert( TaskReservationCount == 0 ); } if( TaskReservationCount > 0 ) { Debug.Assert( PendingTasks.Count == 1 ); } } // After adding a new Task, check for any outstanding reservations CheckForReservations(); // Done ErrorCode = Constants.SUCCESS; } else { Manager.Log( EVerbosityLevel.Critical, ELogColour.Red, "[Job] AddTask: Failed to cache necessary task files" ); ErrorCode = Constants.ERROR_CHANNEL_NOT_FOUND; } return ( ErrorCode ); }
private bool RequestTaskFiles( Connection RequestingConnection, AgentTaskSpecification TaskSpecification ) { // Check for and possibly request each dependency if( TaskSpecification.Dependencies != null ) { foreach( string Dependency in TaskSpecification.Dependencies ) { if( !RequestDependency( RequestingConnection, Dependency, true ) ) { return false; } } } return true; }
public AgentTask( AgentTaskSpecification NewSpecification ) { Specification = NewSpecification; CurrentState = new AgentTaskState( NewSpecification.JobGuid, NewSpecification.TaskGuid, EJobTaskState.TASK_STATE_IDLE ); CurrentOwner = null; }
Int32 CacheAllFiles(AgentTaskSpecification TaskSpecification) { Int32 ReturnValue = Constants.INVALID; try { TaskSpecification.DependenciesOriginalNames = null; if ((TaskSpecification.Dependencies != null) && (TaskSpecification.Dependencies.Count > 0)) { // Allocate the dictionary we'll use for the name mapping Dictionary<String, String> DependenciesOriginalNames = new Dictionary<String, String>(); // Check and cache all the dependencies for (Int32 i = 0; i < TaskSpecification.Dependencies.Count; i++) { String OriginalDependencyFullPath = TaskSpecification.Dependencies[i]; String CachedDependencyName; ReturnValue = CacheFileAndConvertName(OriginalDependencyFullPath, out CachedDependencyName, false); if (ReturnValue < 0) { // Failed to cache the dependency, return failure Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:CacheAllFiles] Failed to cache the dependency: " + OriginalDependencyFullPath); return ReturnValue; } TaskSpecification.Dependencies[i] = CachedDependencyName; String OriginalDependencyName = Path.GetFileName(OriginalDependencyFullPath); DependenciesOriginalNames.Add(CachedDependencyName, OriginalDependencyName); } // Set the newly created name mapping dictionary TaskSpecification.DependenciesOriginalNames = DependenciesOriginalNames; } ReturnValue = Constants.SUCCESS; } catch (Exception Ex) { Log(EVerbosityLevel.Critical, ELogColour.Red, "[Interface:CacheAllFiles] Error: " + Ex.Message); ReturnValue = Constants.ERROR_CONNECTION_DISCONNECTED; CleanupClosedConnection(); } return ReturnValue; }
/** * Adds a Task to the current Job * * @param Specification A structure describing the new Task * * @return Int32 Error code (< 0 is an error) */ public virtual Int32 AddTask(FTaskSpecification Specification) { StartTiming("AddTask-Managed", true); Int32 ReturnValue = Constants.INVALID; if (Connection != null) { if (ConnectionConfiguration.AgentJobGuid != null) { // Convert the parameters from native to managed AgentGuid TaskGuid = new AgentGuid(Specification.TaskGuid.A, Specification.TaskGuid.B, Specification.TaskGuid.C, Specification.TaskGuid.D); String Parameters = Specification.Parameters; List<String> Dependencies = null; if (Specification.DependencyCount > 0) { Dependencies = new List<String>(); for (UInt32 i = 0; i < Specification.DependencyCount; i++) { Dependencies.Add(Specification.Dependencies[i]); } } AgentTaskSpecification NewSpecification = new AgentTaskSpecification(ConnectionConfiguration.AgentJobGuid, TaskGuid, (Int32)Specification.Flags, Parameters, (Int32)Specification.Cost, Dependencies); // Ensure all the files are in the cache with the right cache compatible name ReturnValue = CacheAllFiles(NewSpecification); if (ReturnValue >= 0) { // Queue up all tasks until the specification is complete and submit them all at once PendingTasks.Add(NewSpecification); } } else { ReturnValue = Constants.ERROR_JOB_NOT_FOUND; } } else { ReturnValue = Constants.ERROR_CONNECTION_NOT_FOUND; } StopTiming(); return ReturnValue; }