/// <summary> /// Generate broker launcher epr /// </summary> /// <param name="machineName">machine name</param> /// <returns>broker launcher epr</returns> internal static string GenerateBrokerLauncherEpr(string endpointPrefix, string machineName, TransportScheme scheme) { if (endpointPrefix.Equals(HttpsPrefix, StringComparison.InvariantCultureIgnoreCase)) { return(SoaHelper.GetBrokerLauncherAddress(machineName, scheme)); } else if (endpointPrefix.Equals(NettcpPrefix, StringComparison.InvariantCultureIgnoreCase)) { return(SoaHelper.GetBrokerLauncherAddress(machineName)); } throw new ArgumentException(); }
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath) { // string sessionId = LocalSessionConfiguration.GetNextSessionId(); string cmd; if (true) { cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath}"; } else { cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath} --AzureStorageConnectionString {LocalSessionConfiguration.BrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True"; } string sessionId = SessionStartInfo.StandaloneSessionId; this.brokerLauncherProcess = Process.Start( LocalSessionConfiguration.BrokerLauncherExePath, cmd); this.svcHostProcess = Process.Start(LocalSessionConfiguration.ServiceHostExePath, "-standalone"); sessionAllocateInfo.Id = sessionId; // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken }; sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress("localhost") }; return(sessionAllocateInfo); }
/// <summary> /// Get broker nodes inside Azure cluster. /// </summary> /// <param name="nodesInfo">node info list</param> /// <param name="incrementIndex">true to increment the round robin index.</param> /// <returns>The returned array sorts the addresses by round robin fashion.</returns> public string[] GetAvailableAzureBrokerEPRs(out List <NodeInfo> nodesInfo, bool incrementIndex = true) { List <string> eprs = new List <string>(); nodesInfo = new List <NodeInfo>(); // this.availableAzureBrokers is being updated by the timer, // so create a tmp list reference here refering to the previous list. List <string> tmp = this.availableAzureBrokers; int count = tmp.Count; if (count >= 1) { int step = 0; if (incrementIndex) { step = Interlocked.Increment(ref this.seedNonHA); TraceHelper.TraceInfo(0, "[BrokerNodesMananger] .GetAvailableAzureBrokerEPRs: Incremented SeedNonHA = {0}", step); } for (int i = 0; i < count; i++) { string broker = tmp[(i + step) % count]; eprs.Add(SoaHelper.GetBrokerLauncherAddress(broker)); // no need to use the FQDN for Azure broker. nodesInfo.Add(GenerateNodeInfo(broker)); } } TraceHelper.TraceEvent( TraceEventType.Verbose, "[BrokerNodesManager] .GetAvailableAzureBrokerEPRs: {0} azure brokers are available.", eprs.Count); return(eprs.ToArray()); }
/// <summary> /// Open the broker launcher service /// </summary> public void OpenService() { Trace.TraceInformation("Open service."); //TODO: SF: remove the singleton implementation //SingletonRegistry.Initialize(SingletonRegistry.RegistryMode.WindowsNonHA); // for debug attach //Thread.Sleep(60 * 1000); bool isOnAzure = SoaHelper.IsOnAzure(); if (isOnAzure) { this.StartNodeMappingCacheService(); } try { // richci: if this is a console application we are running in MSCS in production. Make // sure only one instance of the console app is running at a time. if (!isOnAzure && IsConsoleApplication && !AcquireSingleProcessLock()) { // If another instance already created the mutex, release this handle ReleaseSingleProcessLock(); throw new InvalidOperationException("Only one instance of the process can be run a time"); } if (false) //!isOnAzure && !IsConsoleApplication && Win32API.IsFailoverBrokerNode()) { // // If this is a brokerlauncher service running as service on a failover BN, dont // open WCF endpoints. In this configuration, the broker launcher windows service is // for mgmt operations only. All application traffic will go through brokerlaunchers // running as console apps in MSCS resource groups // // Otherwise this a HpcBroker windows service on FO BN handling mgmt operations only // this.launcherInstance = new BrokerLauncher(true, this.context); } else { Trace.TraceInformation("Open broker launcher service host."); this.launcherInstance = new BrokerLauncher(false, this.context); this.launcherHost = new ServiceHost(this.launcherInstance, new Uri(SoaHelper.GetBrokerLauncherAddress(HostName))); BindingHelper.ApplyDefaultThrottlingBehavior(this.launcherHost); this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, string.Empty); this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, "Internal"); this.launcherHost.AddServiceEndpoint(typeof(IBrokerLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, "AAD"); // this.launcherHost.Credentials.UseInternalAuthenticationAsync(true).GetAwaiter().GetResult(); string addFormat = SoaHelper.BrokerLauncherAadAddressFormat; ServiceAuthorizationBehavior myServiceBehavior = this.launcherHost.Description.Behaviors.Find <ServiceAuthorizationBehavior>(); myServiceBehavior.PrincipalPermissionMode = PrincipalPermissionMode.None; this.launcherHost.Open(); if (BrokerLauncherSettings.Default.EnableAzureStorageQueueEndpoint) { if (string.IsNullOrEmpty(BrokerLauncherSettings.Default.AzureStorageConnectionString)) { Trace.TraceError("AzureStorageConnectionString is null or empty while EnableAzureStorageQueueEndpoint is set to true"); } else { this.watcher = new BrokerLauncherCloudQueueWatcher(this.launcherInstance, BrokerLauncherSettings.Default.AzureStorageConnectionString); } } Trace.TraceInformation("Open broker launcher service succeeded."); TraceHelper.TraceEvent(TraceEventType.Information, "Open broker launcher service succeeded."); if (SoaHelper.IsSchedulerOnAzure()) { // Broker service is enabled on scheduler node for on-premise and scheduler on Azure cluster. // SoaDiagSvc is not expected to run on the Azure cluster. return; } ISchedulerHelper helper = SchedulerHelperFactory.GetSchedulerHelper(this.context); #if HPCPACK ThreadPool.QueueUserWorkItem( (object state) => { try { RetryHelper <object> .InvokeOperation( () => { this.soaDiagAuthenticator = new SoaDiagAuthenticator(); SoaDiagService diagServiceInstance = new SoaDiagService(helper.GetClusterInfoAsync, this.soaDiagAuthenticator); this.diagServiceHost = new ServiceHost( diagServiceInstance, #if DEBUG new Uri("http://localhost/SoaDiagService"), #endif new Uri(SoaHelper.GetDiagServiceAddress(HostName))); BindingHelper.ApplyDefaultThrottlingBehavior(this.diagServiceHost, SoaDiagSvcMaxConcurrentCalls); var endpoint = this.diagServiceHost.AddServiceEndpoint(typeof(ISoaDiagService), BindingHelper.HardCodedDiagServiceNetTcpBinding, string.Empty); endpoint.Behaviors.Add(new SoaDiagServiceErrorHandler()); #if DEBUG var httpEndpoint = this.diagServiceHost.AddServiceEndpoint(typeof(ISoaDiagService), new BasicHttpBinding(), string.Empty); httpEndpoint.Behaviors.Add(new SoaDiagServiceErrorHandler()); #endif this.diagServiceHost.Open(); TraceHelper.TraceEvent(TraceEventType.Information, "Open soa diag service succeeded."); this.cleanupService = new DiagCleanupService(helper.GetClusterInfoAsync); this.cleanupService.Start(); TraceHelper.TraceEvent(TraceEventType.Information, "Open soa diag cleanup service succeeded."); return(null); }, (ex, count) => { TraceHelper.TraceEvent(TraceEventType.Error, "Failed to open soa diag service: {0}. Retry Count = {1}", ex, count); this.CloseSoaDiagService(); Thread.Sleep(RetryPeriod); }); } catch (Exception e) { TraceHelper.TraceEvent(TraceEventType.Error, "Failed to open soa diag service after all retry: {0}", e); } }); #endif #if HPCPACK ThreadPool.QueueUserWorkItem((object state) => { try { RetryHelper <object> .InvokeOperation( () => { this.azureStorageCleaner = new AzureStorageCleaner(helper); this.azureStorageCleaner.Start(); TraceHelper.TraceEvent(TraceEventType.Information, "Open Azure storage cleanup service succeeded."); return(null); }, (ex, count) => { TraceHelper.TraceEvent( TraceEventType.Error, "Failed to open Azure storage cleanup service: {0}. Retry Count = {1}", ex, count); if (this.azureStorageCleaner != null) { this.azureStorageCleaner.Close(); } Thread.Sleep(RetryPeriod); }); } catch (Exception e) { TraceHelper.TraceEvent(TraceEventType.Error, "Failed to open Azure storage cleanup service after all retry: {0}", e); } }); #endif } } catch (Exception e) { TraceHelper.TraceEvent(TraceEventType.Critical, "Failed to open service: {0}", e); throw; } }
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath) { try { bool brokerPerfMode = true; // TODO: implement separated broker mode if (brokerPerfMode) { TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode"); } TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.", callId, endpointPrefix, durable); using (var batchClient = AzureBatchConfiguration.GetBatchClient()) { var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName); ODATADetailLevel detailLevel = new ODATADetailLevel(); detailLevel.SelectClause = "affinityId, ipAddress"; //detailLevel.FilterClause = @"state eq 'idle'"; var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync(); if (nodes.Count < 1) { throw new InvalidOperationException("Compute node count in selected pool is less then 1."); } sessionAllocateInfo.Id = string.Empty; // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken }; IList <EnvironmentSetting> ConstructEnvironmentVariable() { List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables) { env.Add(new EnvironmentSetting(entry.Name, entry.Value)); } // pass service serviceInitializationTimeout as job environment variables env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString())); if (startInfo.ServiceHostIdleTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString())); } if (startInfo.ServiceHangTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString())); } // pass MessageLevelPreemption switcher as job environment variables env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString())); // pass trace switcher to svchost if (!string.IsNullOrEmpty(traceSwitchValue)) { env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue)); } // pass taskcancelgraceperiod as environment variable to svchosts env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString())); // pass service config file name to services env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName)); // pass maxMessageSize to service hosts int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize; env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString())); // pass service operation timeout to service hosts int?serviceOperationTimeout = null; if (startInfo.ServiceOperationTimeout.HasValue) { serviceOperationTimeout = startInfo.ServiceOperationTimeout; } else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null) { serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout; } if (serviceOperationTimeout.HasValue) { env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString())); } if (startInfo.Environments != null) { foreach (KeyValuePair <string, string> entry in startInfo.Environments) { env.Add(new EnvironmentSetting(entry.Key, entry.Value)); } } // Each SOA job is assigned a GUID "secret", which is used // to identify soa job owner. When a job running in Azure // tries to access common data, it sends this "secret" together // with a data request to data service. Data service trusts // the data request only if the job id and job "secret" // match. env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString())); // Set CCP_SERVICE_SESSIONPOOL env var of the job if (startInfo.UseSessionPool) { env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString)); } void SetBrokerNodeAuthenticationInfo() { // TODO: set the information needed by compute node to authenticate broker node return; } SetBrokerNodeAuthenticationInfo(); env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString())); env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString())); TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.", callId, BrokerSettingsConstants.Secure, startInfo.Secure, BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme); env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName())); env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE")); //Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir)); return(env); } var environment = ConstructEnvironmentVariable(); ResourceFile GetResourceFileReference(string containerName, string blobPrefix) { var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read); ResourceFile rf; if (string.IsNullOrEmpty(blobPrefix)) { rf = ResourceFile.FromStorageContainerUrl(sasToken); } else { rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix); } return(rf); } async Task <string> CreateJobAsync() { //TODO: need a function to test if all parameters are legal. if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0) { throw new ArgumentException("Maxunit value is invalid."); } string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId()); Debug.Assert(batchClient != null, nameof(batchClient) + " != null"); var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation() { PoolId = AzureBatchConfiguration.BatchPoolName }); job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine); job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); job.JobPreparationTask.ResourceFiles = new List <ResourceFile>() { GetResourceFileReference(ServiceRegistrationContainer, null), GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder), GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower()) }; job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine); job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // List<ResourceFile> resourceFiles = new List<ResourceFile>(); // resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); // resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null)); // // job.JobManagerTask = new JobManagerTask("Broker", // // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"); // job.JobManagerTask = new JobManagerTask("List", // $@"cmd /c dir & set"); // job.JobManagerTask.ResourceFiles = resourceFiles; // job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // Set Meta Data if (job.Metadata == null) { job.Metadata = new List <MetadataItem>(); } Dictionary <string, string> jobMetadata = new Dictionary <string, string>() { { BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() }, { BrokerSettingsConstants.Secure, startInfo.Secure.ToString() }, { BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() }, { BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() }, }; if (startInfo.ServiceVersion != null) { jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString()); } if (startInfo.MaxUnits != null) { jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString()); } Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>() { { BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout }, { BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout }, { BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold }, { BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold }, { BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout }, { BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize }, { BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout }, { BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink } }; job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value))) .Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList(); job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service"; await job.CommitAsync(); return(job.Id); } var jobId = await CreateJobAsync(); string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId); if (!sessionId.Equals("-1")) { sessionAllocateInfo.Id = sessionId; } else { TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId); } Task AddTasksAsync() { int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count; var comparer = new EnvironmentSettingComparer(); CloudTask CreateTask(string taskId) { CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone"); cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } CloudTask CreateBrokerTask(bool direct) { List <ResourceFile> resourceFiles = new List <ResourceFile>(); resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); string cmd; if (direct) { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } else { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } CloudTask cloudTask = new CloudTask("Broker", cmd); cloudTask.ResourceFiles = resourceFiles; cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } //TODO: task id type should be changed from int to string var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray(); if (!brokerPerfMode) { tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray(); } else { tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray(); } return(batchClient.JobOperations.AddTaskAsync(jobId, tasks)); } await AddTasksAsync(); async Task WaitBatchBrokerLauncher() { var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker"); TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor(); await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout); await brokerTask.RefreshAsync(); var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress; sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) }; } if (brokerPerfMode) { //If broker node and session launcher node is not the same node, this line should be modified. sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) }; } else { await WaitBatchBrokerLauncher(); } return(sessionAllocateInfo); } } catch (Exception ex) { TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}"); throw; } }