Exemplo n.º 1
0
        /**
         * Takes messages off the internal message queue and handles them by either
         * sending responses, forwarding the message on, or processing it internally
         */
        private void ProcessMessagesThreadProc()
        {
            // A response message queue used to send messages back to the one which sent it
            Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();

            while (AgentHasShutDown == false)
            {
                StartTiming("ProcessMessage-Internal", true);

                lock ( MessageQueueLock )
                {
                    // Swap the SM and PM message queue to keep things moving at full speed
                    Queue <AgentMessage> Temp = MessageQueuePM;
                    MessageQueuePM = MessageQueueSM;
                    MessageQueueSM = Temp;
                }

                // Process all messages currently in the queue
                while (MessageQueuePM.Count > 0)
                {
                    Debug.Assert(ResponseMessageQueue.Count == 0);

                    // Get and process the next message
                    AgentMessage NextMessage = MessageQueuePM.Dequeue();
                    Debug.Assert(NextMessage != null);

                    bool bMessageHandled = false;
                    switch (NextMessage.Type)
                    {
                    case EMessageType.SIGNAL:
                    {
                        if (NextMessage is DisconnectionSignalMessage)
                        {
                            // Mark the connection as inactive
                            DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
                            Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
                            DisconnectMessage.ConnectionToDisconnect.CurrentState     = ConnectionState.DISCONNECTED;
                            DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
                        }

                        // Signal the message and move on
                        AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
                        SignalMessage.ResetEvent.Set();
                        bMessageHandled = true;
                    }
                    break;

                    case EMessageType.TIMING:
                    {
                        Connection FromConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
                        {
                            Connection ToConnection;
                            if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
                                (ToConnection is LocalConnection))
                            {
                                // Handle message
                                AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
                                AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
                                bMessageHandled = true;
                            }
                        }
                    }
                    break;

                    case EMessageType.TASK_REQUEST:
                    {
                        // Look up the requesting connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection RequestingConnection;
                        if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
                        {
                            // Look up the specified Job
                            AgentJob JobToAskForTasks = RequestingConnection.Job;
                            if (JobToAskForTasks != null)
                            {
                                // If we get a valid response back, add it to the queue
                                AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
                                if (Response != null)
                                {
                                    ResponseMessageQueue.Enqueue(Response);

                                    // Specifications and releases are always handled here, but
                                    // reservations are special in that we will send a reservation
                                    // back to local connections but we'll need to make sure the
                                    // message continues on to remote connections.
                                    if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
                                        (Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
                                        ((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
                                         (JobToAskForTasks.Owner is LocalConnection)))
                                    {
                                        bMessageHandled = true;
                                    }
                                }
                            }
                            else
                            {
                                // Unable to find the Job, just send back a release message
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
                                //ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
                                //															ETaskRequestResponseType.RELEASE ) );
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;

                    case EMessageType.TASK_STATE:
                    {
                        // Look up the sending connection
                        Debug.Assert(NextMessage.From != Constants.INVALID);
                        Connection SendingConnection;
                        if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
                            (SendingConnection.Job != null))
                        {
                            // Look up the specified Job
                            AgentJob UpdatedJob;
                            if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
                            {
                                AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
                                UpdatedJob.UpdateTaskState(UpdatedTaskState);

                                if (UpdatedJob.Owner is LocalConnection)
                                {
                                    // If the Task state change is of a type potentially interesting to
                                    // the Instigator, return it
                                    switch (UpdatedTaskState.TaskState)
                                    {
                                    case EJobTaskState.TASK_STATE_INVALID:
                                    case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
                                    case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
                                        // For these message types, allow the message to continue on
                                        break;

                                    default:
                                        // Nothing to do otherwise, mark the message as handled
                                        bMessageHandled = true;
                                        break;
                                    }
                                }
                                else
                                {
                                    // Always send messages on for remote connections
                                }
                            }
                            else
                            {
                                // Unable to find the Job, swallow the request
                                Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
                                bMessageHandled = true;
                            }
                        }
                        else
                        {
                            // Unable to find the connection, swallow the request
                            Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
                            bMessageHandled = true;
                        }
                    }
                    break;
                    }

                    // If the message was not handled completely, send it on
                    if (bMessageHandled == false)
                    {
                        // Look up who the message is being sent to and make sure they're
                        // still active and if not, ignore the message
                        Connection Recipient;
                        Debug.Assert(NextMessage.To != Constants.INVALID);
                        if (Connections.TryGetValue(NextMessage.To,out Recipient))
                        {
                            if (Recipient is LocalConnection)
                            {
                                // If the recipient is local, place it in the proper queue
                                // and signal that a message is ready
                                LocalConnection LocalRecipient = Recipient as LocalConnection;
                                lock (LocalRecipient.MessageQueue)
                                {
                                    LocalRecipient.MessageQueue.Enqueue(NextMessage);

                                    string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
                                                                         NextMessage.To,
                                                                         NextMessage.From,
                                                                         NextMessage.Type,
                                                                         LocalRecipient.MessageQueue.Count);

                                    Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);

                                    LocalRecipient.MessageAvailableSignal();
                                }
                            }
                            else
                            {
                                Debug.Assert(Recipient is RemoteConnection);

                                // If the recipient is remote, send the message via SendMessage
                                // unless the message is a Task being sent back, which is sent
                                // via the dedicated Task API
                                RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
                                if (NextMessage is AgentTaskSpecification)
                                {
                                    // All new Tasks are sent via the dedicated Task API
                                    AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;

                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"]       = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Specification"] = TaskSpecification;
                                    Hashtable RemoteOutParameters = null;

                                    Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
                                    if (Error >= 0)
                                    {
                                        // Perhaps we should be sending an accept message back?
                                    }
                                    else
                                    {
                                        AgentTaskState UpdateMessage;
                                        if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
                                        {
                                            // Special case of the connection dropping while we're adding the
                                            // task, say it's been killed to requeue
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_KILLED);
                                        }
                                        else
                                        {
                                            // All other error cases will be rejections
                                            UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
                                                                               TaskSpecification.TaskGuid,
                                                                               EJobTaskState.TASK_STATE_REJECTED);
                                        }
                                        AgentJob Job;
                                        if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
                                        {
                                            Job.UpdateTaskState(UpdateMessage);
                                        }
                                    }
                                }
                                else
                                {
                                    // All standard messages are sent via the SendMessage API
                                    Hashtable RemoteInParameters = new Hashtable();
                                    RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
                                    RemoteInParameters["Message"] = NextMessage;
                                    Hashtable RemoteOutParameters = null;

                                    RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
                                }

                                string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
                                                                     NextMessage.To,
                                                                     NextMessage.From,
                                                                     NextMessage.Type);

                                Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
                            }
                        }
                        else
                        {
                            Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
                        }
                    }

                    // If there are any responses to the message, send them
                    if (ResponseMessageQueue.Count > 0)
                    {
                        foreach (AgentMessage NextResponse in ResponseMessageQueue)
                        {
                            // For each one of the messages, set the routing fields properly
                            NextResponse.To   = NextMessage.From;
                            NextResponse.From = NextMessage.To;

                            // And then queue the message back up immediately
                            MessageQueuePM.Enqueue(NextResponse);
                        }
                        ResponseMessageQueue.Clear();
                    }
                }

                StopTiming();

                // Wait for a message to become available and once unlocked, swap the queues
                // and check for messages to process. Set a timeout, so we'll wake up every
                // now and then to check for a quit signal at least
                MessageQueueReady.WaitOne(500);
            }
        }
Exemplo n.º 2
0
		public void CheckForReservations()
		{
			// We need the state to not change while we're in here
			lock( CurrentStateLock )
			{
				// Depending on the current Job state, handle any outstanding Task reservations
				if( ( CurrentState == JobState.AGENT_JOB_UNSPECIFIED ) ||
					( CurrentState == JobState.AGENT_JOB_PENDING ) )
				{
					// As long as the job is PENDING, do nothing. In fact, there shouldn't be
					// any reservations at this point. Assert this.
					Debug.Assert( TaskReservationCount == 0 );
				}
				else if( CurrentState == JobState.AGENT_JOB_RUNNING )
				{
					// If there are any outstanding reservations and tasks to give out...
					UInt32 PendingTasksCount = ( UInt32 )PendingTasks.Count;
					UInt32 ReservationsToTryToFill = Math.Min( PendingTasksCount, TaskReservationCount );

					if( ReservationsToTryToFill > 0 )
					{
						// Try to fulfill the local ones first
						List<LocalConnection> LocalChildren = Owner.LocalChildren.Values;
						List<LocalConnection>.Enumerator Children = LocalChildren.GetEnumerator();

						while( ( Children.MoveNext() ) &&
							   ( ReservationsToTryToFill > 0 ) )
						{
							LocalConnection Local = Children.Current;
							while( ( Local.ReservationCount > 0 ) &&
								   ( ReservationsToTryToFill > 0 ) )
							{
								// If we should avoid assigning tasks to this agent,
								// simply continue on to the next one (only meaningful
								// for local connections since we currently would only
								// avoid tasks on the Instigator)
								if( AvoidNextTask( Local ) )
								{
									break;
								}

								AgentTaskRequestResponse Response = GetNextTask( Local );
								if( ( Response != null ) &&
									( Response is AgentTaskSpecification ) )
								{
									AgentTaskSpecification NextTask = Response as AgentTaskSpecification;

									// For local connections, send the task message directly
									Response.To = Local.Handle;
									Response.From = Owner.Handle;
									Manager.SendMessageInternal( Owner, Response );

									// Consider the reservation satisfied
									TaskReservationCount--;
									Local.ReservationCount--;
									ReservationsToTryToFill--;
								}
								else
								{
									// If we didn't get a task, don't keep trying for now
									ReservationsToTryToFill = 0;
								}
							}
						}
					}

					// If we still have pending tasks try to fulfill any outstanding remote reservations
					if( ReservationsToTryToFill > 0 )
					{
						List<RemoteConnection> RemoteChildren = Owner.RemoteChildren.Values;
						List<RemoteConnection>.Enumerator Children = RemoteChildren.GetEnumerator();

						while( ( Children.MoveNext() ) &&
							   ( ReservationsToTryToFill > 0 ) )
						{
							RemoteConnection Remote = Children.Current;
							while( ( Remote.ReservationCount > 0 ) &&
								   ( ReservationsToTryToFill > 0 ) )
							{
								AgentTaskRequestResponse Response = GetNextTask( Remote );
								if( ( Response != null ) &&
									( Response is AgentTaskSpecification ) )
								{
									AgentTaskSpecification NextTask = Response as AgentTaskSpecification;

									// For remote connections, add the task using the job/task API
									Hashtable AddTaskInParameters = new Hashtable();
									AddTaskInParameters["Version"] = ESwarmVersionValue.VER_1_0;
									AddTaskInParameters["Specification"] = NextTask;
									Hashtable AddTaskOutParameters = null;

									if( Remote.Interface.AddTask( Remote.Handle, AddTaskInParameters, ref AddTaskOutParameters ) >= 0 )
									{
										// Consider the reservation satisfied
										TaskReservationCount--;
										Remote.ReservationCount--;
										ReservationsToTryToFill--;
									}
									else
									{
										// If this fails for any reason, update the task and move on to the next child.
										// Note that marking it KILLED will allow it to be requeued for another agent
										// and ultimately it'll end up on the local agent, where if it fails, it'll
										// be marked a real failure.
										UpdateTaskState( new AgentTaskState( NextTask.JobGuid, NextTask.TaskGuid, EJobTaskState.TASK_STATE_KILLED ) );
										break;
									}
								}
								else
								{
									// If we didn't get a task, don't keep trying for now
									ReservationsToTryToFill = 0;
								}
							}
						}
					}
				}
				else if( CurrentState == JobState.AGENT_JOB_CLOSED )
				{
					// If there are any outstanding reservations, we can release them now that we know
					// there will be no more tasks added to the Job
					if( TaskReservationCount > 0 )
					{
						foreach( LocalConnection Local in Owner.LocalChildren.Values )
						{
							// If there are any outstanding reservations, a single RELEASE is sufficient
							if( Local.ReservationCount > 0 )
							{
								AgentTaskRequestResponse ReleaseMessage =
									new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE );
								ReleaseMessage.To = Local.Handle;
								ReleaseMessage.From = Owner.Handle;
								Manager.SendMessageInternal( Owner, ReleaseMessage );
							}
							// Consider all reservations canceled
							TaskReservationCount -= Local.ReservationCount;
							Local.ReservationCount = 0;
						}

						foreach( RemoteConnection Remote in Owner.RemoteChildren.Values )
						{
							// If there are any outstanding reservations, a single RELEASE is sufficient
							if( Remote.ReservationCount > 0 )
							{
								AgentTaskRequestResponse ReleaseMessage =
									new AgentTaskRequestResponse( JobGuid, ETaskRequestResponseType.RELEASE );
								ReleaseMessage.To = Remote.Handle;
								ReleaseMessage.From = Owner.Handle;
								Manager.SendMessageInternal( Owner, ReleaseMessage );
							}
							// Consider all reservations canceled
							TaskReservationCount -= Remote.ReservationCount;
							Remote.ReservationCount = 0;
						}

						// Sanity check
						Debug.Assert( TaskReservationCount == 0 );
					}
				}
			}
		}