private void DeveloperRestartWorkerAgentsMenuItem_Click(object sender, EventArgs e)
{
if (AgentApplication.Options.EnableStandaloneMode)
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Not restarting worker agents, standalone mode enabled");
}
else
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Restarting worker agents...");
AgentApplication.RequestRestartWorkerAgents();
}
}
private void SwarmAgentWindowClosing(object sender, FormClosingEventArgs e)
{
if (e.CloseReason == CloseReason.UserClosing)
{
e.Cancel = true;
Hide();
}
else
{
AgentApplication.RequestQuit();
}
}
private void NetworkPingRemoteAgentsMenuItem_Click(object sender, EventArgs e)
{
if (AgentApplication.Options.EnableStandaloneMode)
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Not pinging remote agents, standalone mode enabled");
}
else
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Pinging remote agents...");
AgentApplication.RequestPingRemoteAgents();
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Remote Agent ping complete");
}
}
private void NetworkPingCoordinatorMenuItem_Click(object sender, EventArgs e)
{
if (AgentApplication.Options.EnableStandaloneMode)
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Not pinging coordinator, standalone mode enabled");
}
else
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Pinging Coordinator...");
if (AgentApplication.RequestPingCoordinator())
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Coordinator has responded normally");
}
else
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, "[Network] Coordinator has failed to respond");
}
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Green, "[Network] Coordinator ping complete");
}
}
private void OptionsValueChanged(object s, PropertyValueChangedEventArgs e)
{
GridItem ChangedProperty = e.ChangedItem;
if (ChangedProperty != null)
{
Type T = ChangedProperty.Value.GetType();
if (T == typeof(SwarmAgentWindow.DialogFont))
{
UpdateFonts();
}
else if (T == typeof(Color))
{
CreateBarColours(this);
}
else if ((ChangedProperty.Label == "EnableStandaloneMode") ||
(ChangedProperty.Label == "AgentGroupName"))
{
AgentApplication.RequestPingCoordinator();
}
else if (ChangedProperty.Label == "CoordinatorRemotingHost")
{
AgentApplication.RequestInitCoordinator();
}
else if (ChangedProperty.Label == "ShowDeveloperMenu")
{
DeveloperMenuItem.Visible = ( bool )ChangedProperty.Value;
}
else if (ChangedProperty.Label == "CacheFolder")
{
AgentApplication.RequestCacheRelocation();
}
// Always write out the latest options when anything changes
SaveOptions();
}
}
/**
* Takes messages off the internal message queue and handles them by either
* sending responses, forwarding the message on, or processing it internally
*/
private void ProcessMessagesThreadProc()
{
// A response message queue used to send messages back to the one which sent it
Queue <AgentMessage> ResponseMessageQueue = new Queue <AgentMessage>();
while (AgentHasShutDown == false)
{
StartTiming("ProcessMessage-Internal", true);
lock ( MessageQueueLock )
{
// Swap the SM and PM message queue to keep things moving at full speed
Queue <AgentMessage> Temp = MessageQueuePM;
MessageQueuePM = MessageQueueSM;
MessageQueueSM = Temp;
}
// Process all messages currently in the queue
while (MessageQueuePM.Count > 0)
{
Debug.Assert(ResponseMessageQueue.Count == 0);
// Get and process the next message
AgentMessage NextMessage = MessageQueuePM.Dequeue();
Debug.Assert(NextMessage != null);
bool bMessageHandled = false;
switch (NextMessage.Type)
{
case EMessageType.SIGNAL:
{
if (NextMessage is DisconnectionSignalMessage)
{
// Mark the connection as inactive
DisconnectionSignalMessage DisconnectMessage = NextMessage as DisconnectionSignalMessage;
Log(EVerbosityLevel.Informative,ELogColour.Green,String.Format("[CloseConnection] Connection disconnected {0:X8}",DisconnectMessage.ConnectionToDisconnect.Handle));
DisconnectMessage.ConnectionToDisconnect.CurrentState = ConnectionState.DISCONNECTED;
DisconnectMessage.ConnectionToDisconnect.DisconnectedTime = DateTime.UtcNow;
}
// Signal the message and move on
AgentSignalMessage SignalMessage = NextMessage as AgentSignalMessage;
SignalMessage.ResetEvent.Set();
bMessageHandled = true;
}
break;
case EMessageType.TIMING:
{
Connection FromConnection;
if ((Connections.TryGetValue(NextMessage.From,out FromConnection)))
{
Connection ToConnection;
if ((Connections.TryGetValue(NextMessage.To,out ToConnection)) &&
(ToConnection is LocalConnection))
{
// Handle message
AgentTimingMessage TimingMessage = NextMessage as AgentTimingMessage;
AgentApplication.UpdateMachineState(MachineNameFromConnection(FromConnection),TimingMessage.ThreadNum,TimingMessage.State);
bMessageHandled = true;
}
}
}
break;
case EMessageType.TASK_REQUEST:
{
// Look up the requesting connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection RequestingConnection;
if (Connections.TryGetValue(NextMessage.From,out RequestingConnection))
{
// Look up the specified Job
AgentJob JobToAskForTasks = RequestingConnection.Job;
if (JobToAskForTasks != null)
{
// If we get a valid response back, add it to the queue
AgentTaskRequestResponse Response = JobToAskForTasks.GetNextTask(RequestingConnection);
if (Response != null)
{
ResponseMessageQueue.Enqueue(Response);
// Specifications and releases are always handled here, but
// reservations are special in that we will send a reservation
// back to local connections but we'll need to make sure the
// message continues on to remote connections.
if ((Response.ResponseType == ETaskRequestResponseType.SPECIFICATION) ||
(Response.ResponseType == ETaskRequestResponseType.RELEASE) ||
((Response.ResponseType == ETaskRequestResponseType.RESERVATION) &&
(JobToAskForTasks.Owner is LocalConnection)))
{
bMessageHandled = true;
}
}
}
else
{
// Unable to find the Job, just send back a release message
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request; may have been closed");
//ResponseMessageQueue.Enqueue( new AgentTaskRequestResponse( RequestingConnection.Job.JobGuid,
// ETaskRequestResponseType.RELEASE ) );
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
case EMessageType.TASK_STATE:
{
// Look up the sending connection
Debug.Assert(NextMessage.From != Constants.INVALID);
Connection SendingConnection;
if ((Connections.TryGetValue(NextMessage.From,out SendingConnection)) &&
(SendingConnection.Job != null))
{
// Look up the specified Job
AgentJob UpdatedJob;
if (ActiveJobs.TryGetValue(SendingConnection.Job.JobGuid,out UpdatedJob))
{
AgentTaskState UpdatedTaskState = NextMessage as AgentTaskState;
UpdatedJob.UpdateTaskState(UpdatedTaskState);
if (UpdatedJob.Owner is LocalConnection)
{
// If the Task state change is of a type potentially interesting to
// the Instigator, return it
switch (UpdatedTaskState.TaskState)
{
case EJobTaskState.TASK_STATE_INVALID:
case EJobTaskState.TASK_STATE_COMPLETE_SUCCESS:
case EJobTaskState.TASK_STATE_COMPLETE_FAILURE:
// For these message types, allow the message to continue on
break;
default:
// Nothing to do otherwise, mark the message as handled
bMessageHandled = true;
break;
}
}
else
{
// Always send messages on for remote connections
}
}
else
{
// Unable to find the Job, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find Job for Task Request");
bMessageHandled = true;
}
}
else
{
// Unable to find the connection, swallow the request
Log(EVerbosityLevel.Verbose,ELogColour.Orange,"[ProcessMessage] Unable to find owning Connection for Task Request");
bMessageHandled = true;
}
}
break;
}
// If the message was not handled completely, send it on
if (bMessageHandled == false)
{
// Look up who the message is being sent to and make sure they're
// still active and if not, ignore the message
Connection Recipient;
Debug.Assert(NextMessage.To != Constants.INVALID);
if (Connections.TryGetValue(NextMessage.To,out Recipient))
{
if (Recipient is LocalConnection)
{
// If the recipient is local, place it in the proper queue
// and signal that a message is ready
LocalConnection LocalRecipient = Recipient as LocalConnection;
lock (LocalRecipient.MessageQueue)
{
LocalRecipient.MessageQueue.Enqueue(NextMessage);
string NewLogMessage = String.Format("Step 2 of 4 for message: ({0:X8} -> {1:X8}), {2}, Message Count {3} (Local Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type,
LocalRecipient.MessageQueue.Count);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
LocalRecipient.MessageAvailableSignal();
}
}
else
{
Debug.Assert(Recipient is RemoteConnection);
// If the recipient is remote, send the message via SendMessage
// unless the message is a Task being sent back, which is sent
// via the dedicated Task API
RemoteConnection RemoteRecipient = Recipient as RemoteConnection;
if (NextMessage is AgentTaskSpecification)
{
// All new Tasks are sent via the dedicated Task API
AgentTaskSpecification TaskSpecification = NextMessage as AgentTaskSpecification;
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Specification"] = TaskSpecification;
Hashtable RemoteOutParameters = null;
Int32 Error = RemoteRecipient.Interface.AddTask(RemoteRecipient.Handle,RemoteInParameters,ref RemoteOutParameters);
if (Error >= 0)
{
// Perhaps we should be sending an accept message back?
}
else
{
AgentTaskState UpdateMessage;
if (Error == Constants.ERROR_CONNECTION_DISCONNECTED)
{
// Special case of the connection dropping while we're adding the
// task, say it's been killed to requeue
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_KILLED);
}
else
{
// All other error cases will be rejections
UpdateMessage = new AgentTaskState(TaskSpecification.JobGuid,
TaskSpecification.TaskGuid,
EJobTaskState.TASK_STATE_REJECTED);
}
AgentJob Job;
if (ActiveJobs.TryGetValue(TaskSpecification.JobGuid,out Job))
{
Job.UpdateTaskState(UpdateMessage);
}
}
}
else
{
// All standard messages are sent via the SendMessage API
Hashtable RemoteInParameters = new Hashtable();
RemoteInParameters["Version"] = ESwarmVersionValue.VER_1_0;
RemoteInParameters["Message"] = NextMessage;
Hashtable RemoteOutParameters = null;
RemoteRecipient.Interface.SendMessage(NextMessage.To,RemoteInParameters,ref RemoteOutParameters);
}
string NewLogMessage = String.Format("Step 2 of 2 for message: ({0:X8} -> {1:X8}), {2}, (Remote Connection)",
NextMessage.To,
NextMessage.From,
NextMessage.Type);
Log(EVerbosityLevel.SuperVerbose,ELogColour.Green,NewLogMessage);
}
}
else
{
Log(EVerbosityLevel.Informative,ELogColour.Orange,"ProcessMessage: Message sent to invalid connection, ignoring: " + NextMessage.Type.ToString());
}
}
// If there are any responses to the message, send them
if (ResponseMessageQueue.Count > 0)
{
foreach (AgentMessage NextResponse in ResponseMessageQueue)
{
// For each one of the messages, set the routing fields properly
NextResponse.To = NextMessage.From;
NextResponse.From = NextMessage.To;
// And then queue the message back up immediately
MessageQueuePM.Enqueue(NextResponse);
}
ResponseMessageQueue.Clear();
}
}
StopTiming();
// Wait for a message to become available and once unlocked, swap the queues
// and check for messages to process. Set a timeout, so we'll wake up every
// now and then to check for a quit signal at least
MessageQueueReady.WaitOne(500);
}
}
static void Main( string[] args )
{
// Read args
ParseArgs(args);
// Start up the GUI thread
InitGUIThread();
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Green, "Starting up SwarmAgent ..." );
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Green, " ... registering SwarmAgent with remoting service" );
// Register the local agent singleton
RemotingConfiguration.RegisterWellKnownServiceType( typeof( Agent ), "SwarmAgent", WellKnownObjectMode.Singleton );
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Green, " ... registering SwarmAgent network channels" );
// We're going to have two channels for the Agent: one for the actual Agent
// application (IPC with infinite timeout) and one for all other remoting
// traffic (TCP with infinite timeout that we monitor for drops)
IpcChannel AgentIPCChannel = null;
TcpChannel AgentTCPChannel = null;
while( ( Ticking == true ) &&
( ( AgentIPCChannel == null ) ||
( AgentTCPChannel == null ) ) )
{
try
{
if( AgentIPCChannel == null )
{
// Register the IPC connection to the local agent
string IPCChannelPortName = String.Format( "127.0.0.1:{0}", Properties.Settings.Default.AgentRemotingPort );
AgentIPCChannel = new IpcChannel( IPCChannelPortName );
ChannelServices.RegisterChannel( AgentIPCChannel, false );
}
if( AgentTCPChannel == null )
{
// Register the TCP connection to the local agent
AgentTCPChannel = new TcpChannel( Properties.Settings.Default.AgentRemotingPort );
ChannelServices.RegisterChannel( AgentTCPChannel, false );
}
}
catch (RemotingException Ex)
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, "[ERROR] Channel already registered, suggesting another SwarmAgent or client is running.");
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, "[ERROR] If you feel this is in error, check your running process list for additional copies of");
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, "[ERROR] SwarmAgent or UnrealLightmass (or other client) and consider killing them.");
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, "[ERROR] Sleeping for a few seconds and trying again...");
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, string.Format("[ERROR] Channel registration failed. Reason: {0}\n, Callstack: {1}.", Ex.Message, Ex.StackTrace));
Thread.Sleep(3000);
}
catch (Exception Ex)
{
AgentApplication.Log(EVerbosityLevel.Informative, ELogColour.Orange, string.Format("[ERROR] Channel registration failed. Reason: {0}\n, Callstack: {1}.", Ex.Message, Ex.StackTrace));
Thread.Sleep(3000);
}
}
// if we're still ticking, we should have both of our channels initialized
if( Ticking )
{
Debug.Assert( AgentIPCChannel != null );
Debug.Assert( AgentTCPChannel != null );
}
else
{
// Otherwise, we can simply return to exit
return;
}
// Get the agent interface object using the IPC channel
string LocalAgentURL = String.Format( "ipc://127.0.0.1:{0}/SwarmAgent", Properties.Settings.Default.AgentRemotingPort.ToString() );
LocalAgent = ( Agent )Activator.GetObject( typeof( Agent ), LocalAgentURL );
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Green, " ... initializing SwarmAgent" );
// Init the local agent object (if this is the first call to it, it will be created now)
bool AgentInitializedSuccessfully = false;
try
{
AgentInitializedSuccessfully = LocalAgent.Init( Process.GetCurrentProcess().Id );
}
catch( Exception Ex )
{
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Red, "[ERROR] Local agent failed to initialize with an IPC channel! Falling back to TCP..." );
AgentApplication.Log( EVerbosityLevel.Verbose, ELogColour.Red, "[ERROR] Exception details: " + Ex.ToString() );
// Try again with the TCP channel, which is slower but should work
LocalAgentURL = String.Format( "tcp://127.0.0.1:{0}/SwarmAgent", Properties.Settings.Default.AgentRemotingPort.ToString() );
LocalAgent = ( Agent )Activator.GetObject( typeof( Agent ), LocalAgentURL );
try
{
AgentInitializedSuccessfully = LocalAgent.Init( Process.GetCurrentProcess().Id );
if( AgentInitializedSuccessfully )
{
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Red, "[ERROR] RECOVERED by using TCP channel!" );
}
}
catch( Exception Ex2 )
{
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Red, "[ERROR] Local agent failed to initialize with a TCP channel! Fatal error." );
AgentApplication.Log( EVerbosityLevel.Verbose, ELogColour.Red, "[ERROR] Exception details: " + Ex2.ToString() );
ChannelServices.UnregisterChannel( AgentTCPChannel );
ChannelServices.UnregisterChannel( AgentIPCChannel );
return;
}
}
// Only continue if we have a fully initialized agent
if( ( LocalAgent != null ) && AgentInitializedSuccessfully )
{
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Green, " ... initialization successful, SwarmAgent now running" );
// Loop until a quit/restart event has been requested
while( !LocalAgent.ShuttingDown() )
{
try
{
// If we've stopped ticking, notify the agent that we want to shutdown
if( !Ticking )
{
LocalAgent.RequestShutdown();
}
// Maintain the agent itself
LocalAgent.MaintainAgent();
// Maintain any running active connections
LocalAgent.MaintainConnections();
// Maintain the Agent's cache
if( CacheRelocationRequested )
{
LocalAgent.RequestCacheRelocation();
CacheRelocationRequested = false;
}
if( CacheClearRequested )
{
LocalAgent.RequestCacheClear();
CacheClearRequested = false;
}
if( CacheValidateRequested )
{
LocalAgent.RequestCacheValidate();
CacheValidateRequested = false;
}
LocalAgent.MaintainCache();
// Maintain any running jobs
LocalAgent.MaintainJobs();
// If this is a deployed application which is configured to auto-update,
// we'll check for any updates and, if there are any, request a restart
// which will install them prior to restarting
#if !__MonoCS__
if( ( AgentApplication.DeveloperOptions.UpdateAutomatically ) &&
( ApplicationDeployment.IsNetworkDeployed ) &&
(DateTime.UtcNow > NextUpdateCheckTime))
{
if( CheckForUpdates() )
{
LocalAgent.RequestRestart();
}
NextUpdateCheckTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
}
#endif
}
catch( Exception Ex )
{
AgentApplication.Log( EVerbosityLevel.Informative, ELogColour.Red, "[ERROR] UNHANDLED EXCEPTION: " + Ex.Message );
AgentApplication.Log( EVerbosityLevel.ExtraVerbose, ELogColour.Red, "[ERROR] UNHANDLED EXCEPTION: " + Ex.ToString() );
}
// Sleep for a little bit
Thread.Sleep( 500 );
}
// Let the GUI destroy itself
RequestQuit();
bool AgentIsRestarting = LocalAgent.Restarting();
// Do any required cleanup
LocalAgent.Destroy();
ChannelServices.UnregisterChannel( AgentTCPChannel );
ChannelServices.UnregisterChannel( AgentIPCChannel );
// Now that everything is shut down, restart if requested
if( AgentIsRestarting )
{
ClearCache();
InstallAllUpdates();
Application.Restart();
}
}
}
///////////////////////////////////////////////////////////////////////////
/*
* Logs a line of text to the agent log window
*/
public Int32 Log(EVerbosityLevel Verbosity, ELogColour TextColour, string Line)
{
AgentApplication.Log(Verbosity, TextColour, Line);
return(Constants.SUCCESS);
}
private void CacheValidateClick(object sender, EventArgs e)
{
AgentApplication.RequestCacheValidation();
}
private void CacheClearClick(object sender, EventArgs e)
{
AgentApplication.RequestCacheClear();
}
private void ClickExitMenu(object sender, EventArgs e)
{
Hide();
AgentApplication.RequestQuit();
}
