private void CreateTimings(ELogFlags InLoggingFlags)
{
LoggingFlags = InLoggingFlags;
if ((LoggingFlags & ELogFlags.LOG_TIMINGS) != 0)
{
bPerfTimerInstanceActive = true;
}
}
static int SwarmOpenConnection(FConnectionCallback CallbackFunc, IntPtr CallbackData, ELogFlags LoggingFlags, IntPtr OptionsFolder)
{
try
{
return GInstance.OpenConnection(CallbackFunc, CallbackData, (ELogFlags)LoggingFlags, FStringMarshaler.MarshalNativeToManaged(OptionsFolder));
}
catch(Exception Ex)
{
DebugLog.Write(Ex.Message + "\n" + Ex.ToString());
return 0;
}
}
Int32 TryOpenConnection(FConnectionCallback CallbackFunc, IntPtr CallbackData, ELogFlags LoggingFlags)
{
try
{
// Allocate our new connection wrapper object
Connection = new IAgentInterfaceWrapper();
// Make sure the agent is alive and responsive before continuing
EditorLog(EVerbosityLevel.Informative, "[TryOpenConnection] Testing the Agent");
Hashtable InParameters = null;
Hashtable OutParameters = null;
bool AgentIsReady = false;
while (!AgentIsReady)
{
try
{
// Simply try to call the method and if it doesn't throw
// an exception, consider it a success
Connection.Method(0, InParameters, ref OutParameters);
AgentIsReady = true;
}
catch (Exception ex)
{
// Wait a little longer
EditorLog(EVerbosityLevel.Critical, "[TryOpenConnection] Waiting for the agent to start up ...");
EditorLog(EVerbosityLevel.Critical, ex.ToString());
Thread.Sleep(5000);
}
}
// Request an official connection to the Agent
EditorLog(EVerbosityLevel.Informative, "[TryOpenConnection] Opening Connection to Agent");
EditorLog(EVerbosityLevel.Informative, "[TryOpenConnection] Local Process ID is " + Process.GetCurrentProcess().Id.ToString());
StartTiming("OpenConnection-Remote", false);
ConnectionHandle = Connection.OpenConnection(AgentProcess, AgentProcessOwner, Process.GetCurrentProcess().Id, LoggingFlags, out ConnectionConfiguration);
StopTiming();
if (ConnectionHandle >= 0)
{
Log(EVerbosityLevel.Informative, ELogColour.Green, "[Interface:TryOpenConnection] Local connection established");
// Spawn a thread to monitor the message queue
MessageThreadData ThreadData = new MessageThreadData();
ThreadData.Owner = this;
ThreadData.Connection = Connection;
ThreadData.ConnectionHandle = ConnectionHandle;
ThreadData.ConnectionCallback = CallbackFunc;
ThreadData.ConnectionCallbackData = CallbackData;
ThreadData.ConnectionConfiguration = ConnectionConfiguration;
// Launch the message queue thread
ConnectionMessageThread = new Thread(new ParameterizedThreadStart(MessageThreadProc));
ConnectionMessageThread.Name = "ConnectionMessageThread";
ConnectionMessageThread.Start( ThreadData );
// Launch the agent monitor thread
ConnectionMonitorThread = new Thread(new ParameterizedThreadStart(MonitorThreadProc));
ConnectionMonitorThread.Name = "ConnectionMonitorThread";
ConnectionMonitorThread.Start(ThreadData);
// Save the user's callback routine
ConnectionCallback = CallbackFunc;
ConnectionCallbackData = CallbackData;
ConnectionLoggingFlags = LoggingFlags;
}
}
catch (Exception Ex)
{
EditorLog(EVerbosityLevel.Critical, "[TryOpenConnection] Error: " + Ex.Message);
EditorLog(EVerbosityLevel.Critical, Ex.ToString());
ConnectionHandle = Constants.INVALID;
Connection = null;
}
return ConnectionHandle;
}
///////////////////////////////////////////////////////////////////////////
// A duplication of the IAgentInterface API which this class wraps
public Int32 OpenConnection(Process AgentProcess, bool AgentProcessOwner, Int32 LocalProcessID, ELogFlags LoggingFlags, out AgentConfiguration NewConfiguration)
{
OpenConnectionDelegate DOpenConnection = Connection.OpenConnection;
// Set up the versioned hashtable input parameters
Hashtable InParameters = new Hashtable();
InParameters["Version"] = ESwarmVersionValue.VER_1_0;
InParameters["ProcessID"] = LocalProcessID;
InParameters["ProcessIsOwner"] = (AgentProcessOwner == true ? true : false);
InParameters["LoggingFlags"] = LoggingFlags;
Hashtable OutParameters = null;
IAsyncResult Result = DOpenConnection.BeginInvoke(InParameters, ref OutParameters, null, null);
// This will wait with an occasional wake up to check to see if the
// agent process is still alive and kicking (avoids infinite wait,
// allows for very long start up times while debugging)
Int32 StartupSleep = 1000;
while ((Result.AsyncWaitHandle.WaitOne(StartupSleep) == false) &&
(AgentProcess.HasExited == false) &&
(AgentProcess.Responding == true))
{
// While the application is alive and responding, wait
DebugLog.Write("[OpenConnection] Waiting for agent to respond ...");
}
if (Result.IsCompleted)
{
// If the invocation didn't fail, end to get the result
Int32 ReturnValue = DOpenConnection.EndInvoke(ref OutParameters, Result);
if (OutParameters != null)
{
if ((ESwarmVersionValue)OutParameters["Version"] == ESwarmVersionValue.VER_1_0)
{
NewConfiguration = new AgentConfiguration();
NewConfiguration.AgentProcessID = (Int32 )OutParameters["AgentProcessID"];
NewConfiguration.AgentCachePath = (String )OutParameters["AgentCachePath"];
NewConfiguration.AgentJobGuid = (AgentGuid )OutParameters["AgentJobGuid"];
if (OutParameters.ContainsKey("IsPureLocalConnection"))
{
NewConfiguration.IsPureLocalConnection = (bool)OutParameters["IsPureLocalConnection"];
}
// Complete and successful
return ReturnValue;
}
}
}
// Otherwise, error
NewConfiguration = null;
return Constants.ERROR_CONNECTION_DISCONNECTED;
}
/**
* Opens a new connection to the Swarm
*
* @param CallbackFunc The callback function Swarm will use to communicate back to the Instigator
*
* @return An INT containing the error code (if < 0) or the handle (>= 0) which is useful for debugging only
*/
public virtual Int32 OpenConnection(FConnectionCallback CallbackFunc, IntPtr CallbackData, ELogFlags LoggingFlags, string OptionsFolder)
{
// Checked here so we can time OpenConnection
if ((LoggingFlags & ELogFlags.LOG_TIMINGS) == ELogFlags.LOG_TIMINGS)
{
PerfTimerInstance = new PerfTimer();
}
StartTiming("OpenConnection-Managed", true);
// Establish a connection to the local Agent server object
ConnectionHandle = Constants.INVALID;
Int32 ReturnValue = Constants.INVALID;
try
{
EditorLog(EVerbosityLevel.Informative, "[OpenConnection] Registering TCP channel ...");
// Start up network services, by opening a network communication channel
NetworkChannel = new TcpClientChannel();
ChannelServices.RegisterChannel(NetworkChannel, false);
// See if an agent is already running, and if not, launch one
EnsureAgentIsRunning(OptionsFolder);
if (AgentProcess != null)
{
EditorLog(EVerbosityLevel.Informative, "[OpenConnection] Connecting to agent ...");
ReturnValue = TryOpenConnection(CallbackFunc, CallbackData, LoggingFlags);
if (ReturnValue >= 0)
{
AgentCacheFolder = ConnectionConfiguration.AgentCachePath;
if (AgentCacheFolder.Length == 0)
{
EditorLog(EVerbosityLevel.Critical, "[OpenConnection] Agent cache folder with 0 length.");
CloseConnection();
ReturnValue = Constants.ERROR_FILE_FOUND_NOT;
}
}
}
else
{
EditorLog(EVerbosityLevel.Critical, "[OpenConnection] Failed to find Swarm Agent");
ReturnValue = Constants.ERROR_FILE_FOUND_NOT;
}
}
catch (Exception Ex)
{
EditorLog(EVerbosityLevel.Critical, "[OpenConnection] Error: " + Ex.Message);
ReturnValue = Constants.ERROR_EXCEPTION;
}
// Finally, if there have been no errors, assign the connection handle
if (ReturnValue >= 0)
{
ConnectionHandle = ReturnValue;
}
else
{
// If we've failed for any reason, call the clean up routine
CleanupClosedConnection();
}
StopTiming();
return ReturnValue;
}
/**
* Used by an agent to open a remote connection to another agent. Remote
* connections are bi-directional and this routine, along with its pair,
* ConfirmRemoteConnection, are used to resolve all handshaking. Remote
* connections are used by Agents to work on Tasks for Jobs they're
* managing on behalf of a LocalConnection.
*/
public Int32 OpenRemoteConnection( string RequestingAgentName, Int32 ConnectionHandle, ELogFlags LoggingFlags, bool IsAssigned, DateTime AssignedTimestamp )
{
// Always wait until the agent is fully initialized
Initialized.WaitOne();
Int32 RemoteConnectionHandle = Constants.INVALID;
CreateTimings( LoggingFlags );
StartTiming( "OpenRemoteConn-Internal", true );
// Make sure we only have one thread in here at a time
lock( Connections )
{
// Reject any incoming remote connection if:
// we're in standalone mode
// of if the CPU is already busy
// or if a restart/exit has been requested
if( ( AgentApplication.Options.EnableStandaloneMode ) ||
( CurrentState == AgentState.Busy ) ||
( AgentIsShuttingDown ) )
{
return Constants.INVALID;
}
else
{
// Determine if this is a request for a connection or an assignment
if( IsAssigned )
{
// Make sure the assignment isn't stale
if( AssignedTimestamp <= LastAssignedTime )
{
return Constants.INVALID;
}
if( Connections.Count != 0 )
{
// If there are any active jobs running in which the owner is the instigator,
// meaning that this agent is connected directly to the instigator, reject
foreach( AgentJob Job in ActiveJobs.Values )
{
if( Job.OwnerIsInstigator )
{
return Constants.INVALID;
}
}
// Close any existing remote connections in favor of this incoming one
foreach( Connection ExistingConnection in Connections.Values )
{
if( ExistingConnection is RemoteConnection )
{
Log( EVerbosityLevel.Informative, ELogColour.Orange, "[Connection] Detected incoming assigned remote connection, closing existing remote connection" );
Hashtable OldInParameters = null;
Hashtable OldOutParameters = null;
CloseConnection( ExistingConnection.Handle, OldInParameters, ref OldOutParameters );
}
}
}
}
else
{
// For connection requests, any other connection at all will cause a rejection
if( Connections.Count != 0 )
{
return Constants.INVALID;
}
}
}
// First, ping the remote host to make sure we can talk to it,
// starting with verifying the Agent with the Coordinator
try
{
if( ( Coordinator != null ) &&
( CoordinatorResponding ) &&
( AgentApplication.Options.EnableStandaloneMode == false ) )
{
AgentInfo RequestingAgentInfo = null;
Hashtable RequestedConfiguration = new Hashtable();
RequestedConfiguration["Version"] = new Version( 0, 0, 0, 0 );
RequestedConfiguration["RequestingAgentName"] = Environment.MachineName;
List<AgentInfo> PotentialRemoteAgents = Coordinator.GetAvailableAgents( RequestedConfiguration );
foreach( AgentInfo NextAgentInfo in PotentialRemoteAgents )
{
if( NextAgentInfo.Name == RequestingAgentName )
{
RequestingAgentInfo = NextAgentInfo;
break;
}
}
if( RequestingAgentInfo != null )
{
Debug.Assert( RequestingAgentInfo.Configuration.ContainsKey( "IPAddress" ) );
string RequestingAgentIPAddress = RequestingAgentInfo.Configuration["IPAddress"].ToString();
ValidateHostName(RequestingAgentIPAddress);
// Now, ping the Agent
if( PingRemoteHost( RequestingAgentName, RequestingAgentIPAddress ) )
{
// Get the remote agent's interface object and wrap it
string RemoteAgentURL = String.Format( "tcp://{0}:{1}/SwarmAgent",
RequestingAgentIPAddress,
Properties.Settings.Default.AgentRemotingPort.ToString() );
Agent RequestingAgentInterface = ( Agent )Activator.GetObject( typeof( Agent ), RemoteAgentURL );
RemoteConnectionInterfaceWrapper WrappedRequestingAgentInterface = new RemoteConnectionInterfaceWrapper( ConnectionHandle, RequestingAgentName, RequestingAgentIPAddress, RequestingAgentInterface );
Log( EVerbosityLevel.Informative, ELogColour.Green, "[Connect] Remote agent connection object obtained: " + RequestingAgentName );
// Confirm the connection
try
{
// Send the connection GUID back to the requesting agent as
// confirmation that we've received the request and plan to
// accept it after the handshaking is complete
RemoteConnectionHandle = WrappedRequestingAgentInterface.ConfirmRemoteConnection( ConnectionHandle );
if( RemoteConnectionHandle >= 0 )
{
// Create the new remote connection object
RemoteConnection NewRemoteConnection = new RemoteConnection( ConnectionHandle,
RequestingAgentInfo,
WrappedRequestingAgentInterface );
// There are times we want to ensure no new connections are coming online
// where we'll lock the Connections dictionary (see MaintainCache). Note
// that we're already in the lock above...
Connections.Add( ConnectionHandle, NewRemoteConnection );
NewRemoteConnection.CurrentState = ConnectionState.CONNECTED;
// Update our state and ping the coordinator
CurrentState = AgentState.Working;
WorkingFor = RequestingAgentName;
if( IsAssigned )
{
// Successful assignment
LastAssignedTime = AssignedTimestamp;
}
PingCoordinator( true );
Log( EVerbosityLevel.Informative, ELogColour.Green, "[Connect] Remote agent connection confirmed: " + RequestingAgentName );
}
}
catch( Exception Ex )
{
Log( EVerbosityLevel.Informative, ELogColour.Red, "[Connect] OpenRemoteConnection: " + Ex.ToString() );
RemoteConnectionHandle = Constants.ERROR_EXCEPTION;
}
}
else
{
// Failed to ping, simply return with an appropriate error code
Log( EVerbosityLevel.Informative, ELogColour.Red, "[Connect] Failed to ping " + RequestingAgentName + " at " + RequestingAgentIPAddress + " to confirm the remote connection" );
RemoteConnectionHandle = Constants.ERROR_CONNECTION_NOT_FOUND;
}
}
else
{
// Failed to find the Agent info for the requesting Agent, simply return with an appropriate error code
Log( EVerbosityLevel.Informative, ELogColour.Red, "[Connect] Failed to lookup " + RequestingAgentName + " in the Coordinator" );
RemoteConnectionHandle = Constants.ERROR_CONNECTION_NOT_FOUND;
}
}
else
{
// Could not contact the Coordinator, simply return with an appropriate error code
Log( EVerbosityLevel.Informative, ELogColour.Red, "[Connect] Failed to contact the Coordinator" );
RemoteConnectionHandle = Constants.ERROR_CONNECTION_NOT_FOUND;
}
}
catch( Exception Ex )
{
Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Remote agent connection failed: " + RequestingAgentName );
Log( EVerbosityLevel.Verbose, ELogColour.Red, "[Connect] Exception details: " + Ex.ToString() );
}
if( RemoteConnectionHandle < 0 )
{
// If we get here, we have failed to create the connection
Log( EVerbosityLevel.Informative, ELogColour.Red, "[Connect] Remote agent connection failed: " + RequestingAgentName );
}
StopTiming();
}
return RemoteConnectionHandle;
}
public Int32 OpenRemoteConnection( string RequestingAgentName, Int32 ConnectionHandle, ELogFlags LoggingFlags, bool IsAssigned, DateTime AssignedTimestamp )
{
Int32 ReturnCode = Constants.ERROR_CONNECTION_DISCONNECTED;
if( RemoteInterfaceAlive )
{
try
{
OpenRemoteConnectionDelegate DOpenRemoteConnection = new OpenRemoteConnectionDelegate( RemoteInterface.OpenRemoteConnection );
IAsyncResult Result = DOpenRemoteConnection.BeginInvoke( RequestingAgentName, ConnectionHandle, LoggingFlags, IsAssigned, AssignedTimestamp, null, null );
// Double the timeout, since this requires round-trip communication
Int32 RemoteAgentTimeout = ( AgentApplication.DeveloperOptions.RemoteAgentTimeout >= 0 ? 2 * AgentApplication.DeveloperOptions.RemoteAgentTimeout * 1000 : Timeout.Infinite );
WaitHandle.WaitAny( new WaitHandle[2] { Result.AsyncWaitHandle, RemoteInterfaceDropped }, RemoteAgentTimeout );
if( Result.IsCompleted )
{
ReturnCode = DOpenRemoteConnection.EndInvoke( Result );
// If opening the remote connection is successful, start monitoring for drops
if( ReturnCode >= 0 )
{
BeginMonitoring();
}
}
}
catch( Exception )
{
SignalConnectionDropped();
}
}
return ReturnCode;
}
