Ejemplo n.º 1
0
        public override async Task TerminateAsync(string sessionId)
        {
            using (var batchClient = AzureBatchConfiguration.GetBatchClient())
            {
                try
                {
                    var batchJob = await batchClient.JobOperations.GetJobAsync(AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(sessionId));

                    //Conflict error occurs when terminate a job which is not in the state of Enabling or Active
                    if (batchJob.State == Azure.Batch.Common.JobState.Active || batchJob.State == Azure.Batch.Common.JobState.Enabling || batchJob.State == Azure.Batch.Common.JobState.Disabling)
                    {
                        await batchJob.TerminateAsync();
                    }
                }
                catch (BatchException e)
                {
                    if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode != null)
                    {
                        if (e.RequestInformation.HttpStatusCode == HttpStatusCode.NotFound)
                        {
                            TraceHelper.TraceEvent(sessionId, TraceEventType.Warning, "[AzureBatchSessionLauncher] .TerminateAsync: The specified job can't be found, maybe has deleted.");
                            return;
                        }
                    }
                    TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[AzureBatchSessionLauncher] .TerminateAsync: Job was failed to terminate : {0}.", e);
                }
                catch (Exception e)
                {
                    TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[AzureBatchSessionLauncher] .TerminateAsync: Job was failed to terminate : {0}.", e);
                }
            }
        }
Ejemplo n.º 2
0
        public override async Task <SessionInfoContract> GetInfoAsync(string endpointPrefix, string sessionId)
        {
            SessionInfoContract sessionInfo = null;

            this.CheckAccess();

            ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix");

            TraceHelper.TraceEvent(
                sessionId,
                TraceEventType.Information,
                "[SessionLauncher] .GetInfo: headnode={0}, endpointPrefix={1}, sessionId={2}",
                Environment.MachineName,
                endpointPrefix,
                sessionId);

            if (!IsEndpointPrefixSupported(endpointPrefix))
            {
                TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[SessionLauncher] .GetInfo: {0} is not a supported endpoint prefix.", endpointPrefix);

                ThrowHelper.ThrowSessionFault(SOAFaultCode.InvalidArgument, SR.SessionLauncher_EndpointNotSupported, endpointPrefix);
            }

            try
            {
                using (var batchClient = AzureBatchConfiguration.GetBatchClient())
                {
                    var jobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(sessionId);

                    var sessionJob = await batchClient.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);

                    if (sessionJob == null)
                    {
                        throw new InvalidOperationException($"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.GetInfoAsync)} Failed to get batch job for session {sessionId}");
                    }

                    TraceHelper.TraceEvent(
                        sessionId,
                        TraceEventType.Information,
                        $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.GetInfoAsync)}: try to get the job properties(Secure, TransportScheme, BrokerEpr, BrokerNode, ControllerEpr, ResponseEpr) for the job, jobid={sessionId}.");

                    sessionInfo    = new SessionInfoContract();
                    sessionInfo.Id = AzureBatchSessionJobIdConverter.ConvertToSessionId(sessionJob.Id);

                    // TODO: sessionInfo.JobState
                    var metadata = sessionJob.Metadata;
                    if (metadata != null)
                    {
                        string brokerNodeString = null;

                        foreach (var pair in metadata)
                        {
                            if (pair.Name.Equals(BrokerSettingsConstants.Secure, StringComparison.OrdinalIgnoreCase))
                            {
                                string secureString = pair.Value;
                                Debug.Assert(secureString != null, "BrokerSettingsConstants.Secure value should be a string.");
                                bool secure;
                                if (bool.TryParse(secureString, out secure))
                                {
                                    sessionInfo.Secure = secure;
                                    TraceHelper.TraceEvent(sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job secure property, Secure={0}.", secure);
                                }
                                else
                                {
                                    TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.Secure + " property", secureString);
                                }
                            }
                            else if (pair.Name.Equals(BrokerSettingsConstants.TransportScheme, StringComparison.OrdinalIgnoreCase))
                            {
                                string schemeString = pair.Value;
                                Debug.Assert(schemeString != null, "BrokerSettingsConstants.TransportScheme value should be a string.");

                                int scheme;
                                if (int.TryParse(schemeString, out scheme))
                                {
                                    sessionInfo.TransportScheme = (TransportScheme)scheme;
                                    TraceHelper.TraceEvent(
                                        sessionId,
                                        TraceEventType.Information,
                                        "[SessionLauncher] .GetInfo: get the job TransportScheme property, TransportScheme={0}.",
                                        sessionInfo.TransportScheme);
                                }
                                else
                                {
                                    TraceHelper.TraceEvent(
                                        sessionId,
                                        TraceEventType.Error,
                                        "Illegal transport scheme value[{0}] for job's " + BrokerSettingsConstants.TransportScheme + " property",
                                        schemeString);
                                }
                            }
                            else if (pair.Name.Equals(BrokerSettingsConstants.BrokerNode, StringComparison.OrdinalIgnoreCase))
                            {
                                brokerNodeString = pair.Value;
                                Debug.Assert(brokerNodeString != null, "BrokerSettingsConstants.BrokerNode value should be a string.");

                                TraceHelper.TraceEvent(
                                    sessionId,
                                    TraceEventType.Information,
                                    "[SessionLauncher] .GetInfo: get the job BrokerLauncherEpr property, BrokerLauncherEpr={0}.",
                                    sessionInfo.BrokerLauncherEpr);
                            }
                            else if (pair.Name.Equals(BrokerSettingsConstants.Durable, StringComparison.OrdinalIgnoreCase))
                            {
                                string durableString = pair.Value;
                                Debug.Assert(durableString != null, "BrokerSettingsConstants.Durable value should be a string.");

                                bool durable;
                                if (bool.TryParse(durableString, out durable))
                                {
                                    sessionInfo.Durable = durable;
                                    TraceHelper.TraceEvent(sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job Durable property, Durable={0}.", sessionInfo.Durable);
                                }
                                else
                                {
                                    TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.Durable + " property", durableString);
                                }
                            }
                            else if (pair.Name.Equals(BrokerSettingsConstants.ServiceVersion, StringComparison.OrdinalIgnoreCase))
                            {
                                if (pair.Value != null)
                                {
                                    try
                                    {
                                        sessionInfo.ServiceVersion = new Version(pair.Value);
                                        TraceHelper.TraceEvent(
                                            sessionId,
                                            TraceEventType.Information,
                                            "[SessionLauncher] .GetInfo: get the job ServiceVersion property, ServiceVersion={0}.",
                                            (sessionInfo.ServiceVersion != null) ? sessionInfo.ServiceVersion.ToString() : string.Empty);
                                    }
                                    catch (Exception e)
                                    {
                                        TraceHelper.TraceEvent(
                                            sessionId,
                                            TraceEventType.Error,
                                            "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.ServiceVersion + " property. Exception = {1}",
                                            pair.Value,
                                            e);
                                    }
                                }
                            }
                        }

                        if (!string.IsNullOrEmpty(brokerNodeString))
                        {
                            if (brokerNodeString != Constant.InprocessBrokerNode)
                            {
                                sessionInfo.BrokerLauncherEpr = BrokerNodesManager.GenerateBrokerLauncherEpr(endpointPrefix, brokerNodeString, sessionInfo.TransportScheme);
                            }
                        }
                        else
                        {
                            sessionInfo.UseInprocessBroker = true;
                        }

                        TraceHelper.TraceEvent(
                            sessionId,
                            TraceEventType.Information,
                            "[SessionLauncher] .GetInfo: get the job BrokerLauncherEpr property, BrokerLauncherEpr={0}.",
                            sessionInfo.BrokerLauncherEpr);
                    }
                }
            }
            catch (Exception e)
            {
                TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[SessionLauncher] .GetInfo: Failed to get all properties from job[{0}], Exception:{1}", sessionId, e);

                ThrowHelper.ThrowSessionFault(SOAFaultCode.GetJobPropertyFailure, SR.SessionLauncher_FailToGetJobProperty, e.ToString());
            }

            if (sessionInfo.SessionOwner == null)
            {
                sessionInfo.SessionOwner = "Everyone";
            }

            if (sessionInfo.SessionACL == null)
            {
                sessionInfo.SessionACL = new string[0];
            }

            if (sessionInfo.JobState == 0)
            {
                // TODO: apply job state converting
                sessionInfo.JobState = JobState.Running;
            }

            TraceHelper.TraceEvent(
                sessionId,
                TraceEventType.Information,
                "[SessionLauncher] .GetInfo: return the sessionInfo, BrokerEpr={0}, BrokerLauncherEpr={1}, ControllerEpr={2}, Id={3}, JobState={4}, ResponseEpr={5}, Secure={6}, TransportScheme={7}, sessionOwner={8}, sessionACL={9}",
                sessionInfo.BrokerEpr,
                sessionInfo.BrokerLauncherEpr,
                sessionInfo.ControllerEpr,
                sessionInfo.Id,
                sessionInfo.JobState,
                sessionInfo.ResponseEpr,
                sessionInfo.Secure,
                sessionInfo.TransportScheme,
                sessionInfo.SessionOwner ?? "null",
                sessionInfo.SessionACL?.Length.ToString() ?? "null");
            return(sessionInfo);
        }
Ejemplo n.º 3
0
        protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
            SessionStartInfoContract startInfo,
            string endpointPrefix,
            bool durable,
            string callId,
            SecureString securePassword,
            ServiceRegistration registration,
            SessionAllocateInfoContract sessionAllocateInfo,
            string traceSwitchValue,
            string serviceName,
            BrokerConfigurations brokerConfigurations,
            string hostpath)
        {
            try
            {
                bool brokerPerfMode = true; // TODO: implement separated broker mode
                if (brokerPerfMode)
                {
                    TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode");
                }

                TraceHelper.TraceEvent(
                    TraceEventType.Information,
                    "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.",
                    callId,
                    endpointPrefix,
                    durable);
                using (var batchClient = AzureBatchConfiguration.GetBatchClient())
                {
                    var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName);

                    ODATADetailLevel detailLevel = new ODATADetailLevel();
                    detailLevel.SelectClause = "affinityId, ipAddress";
                    //detailLevel.FilterClause = @"state eq 'idle'";
                    var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync();

                    if (nodes.Count < 1)
                    {
                        throw new InvalidOperationException("Compute node count in selected pool is less then 1.");
                    }

                    sessionAllocateInfo.Id = string.Empty;

                    // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
                    IList <EnvironmentSetting> ConstructEnvironmentVariable()
                    {
                        List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite

                        foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables)
                        {
                            env.Add(new EnvironmentSetting(entry.Name, entry.Value));
                        }

                        // pass service serviceInitializationTimeout as job environment variables
                        env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString()));

                        if (startInfo.ServiceHostIdleTimeout == null)
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString()));
                        }
                        else
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString()));
                        }

                        if (startInfo.ServiceHangTimeout == null)
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString()));
                        }
                        else
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString()));
                        }

                        // pass MessageLevelPreemption switcher as job environment variables
                        env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString()));

                        // pass trace switcher to svchost
                        if (!string.IsNullOrEmpty(traceSwitchValue))
                        {
                            env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue));
                        }

                        // pass taskcancelgraceperiod as environment variable to svchosts
                        env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString()));

                        // pass service config file name to services
                        env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName));

                        // pass maxMessageSize to service hosts
                        int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize;

                        env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString()));

                        // pass service operation timeout to service hosts
                        int?serviceOperationTimeout = null;

                        if (startInfo.ServiceOperationTimeout.HasValue)
                        {
                            serviceOperationTimeout = startInfo.ServiceOperationTimeout;
                        }
                        else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null)
                        {
                            serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout;
                        }

                        if (serviceOperationTimeout.HasValue)
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString()));
                        }

                        if (startInfo.Environments != null)
                        {
                            foreach (KeyValuePair <string, string> entry in startInfo.Environments)
                            {
                                env.Add(new EnvironmentSetting(entry.Key, entry.Value));
                            }
                        }

                        // Each SOA job is assigned a GUID "secret", which is used
                        // to identify soa job owner. When a job running in Azure
                        // tries to access common data, it sends this "secret" together
                        // with a data request to data service.  Data service trusts
                        // the data request only if the job id and job "secret"
                        // match.
                        env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString()));

                        // Set CCP_SERVICE_SESSIONPOOL env var of the job
                        if (startInfo.UseSessionPool)
                        {
                            env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString));
                        }

                        void SetBrokerNodeAuthenticationInfo()
                        {
                            // TODO: set the information needed by compute node to authenticate broker node
                            return;
                        }

                        SetBrokerNodeAuthenticationInfo();

                        env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString()));
                        env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString()));

                        TraceHelper.TraceEvent(
                            TraceEventType.Information,
                            "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.",
                            callId,
                            BrokerSettingsConstants.Secure,
                            startInfo.Secure,
                            BrokerSettingsConstants.TransportScheme,
                            startInfo.TransportScheme);

                        env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName()));
                        env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE"));

                        //Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR
                        env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir));
                        return(env);
                    }
                    var environment = ConstructEnvironmentVariable();

                    ResourceFile GetResourceFileReference(string containerName, string blobPrefix)
                    {
                        var          sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read);
                        ResourceFile rf;

                        if (string.IsNullOrEmpty(blobPrefix))
                        {
                            rf = ResourceFile.FromStorageContainerUrl(sasToken);
                        }
                        else
                        {
                            rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix);
                        }

                        return(rf);
                    }

                    async Task <string> CreateJobAsync()
                    {
                        //TODO: need a function to test if all parameters are legal.
                        if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0)
                        {
                            throw new ArgumentException("Maxunit value is invalid.");
                        }
                        string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId());

                        Debug.Assert(batchClient != null, nameof(batchClient) + " != null");
                        var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation()
                        {
                            PoolId = AzureBatchConfiguration.BatchPoolName
                        });

                        job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine);
                        job.JobPreparationTask.UserIdentity  = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
                        job.JobPreparationTask.ResourceFiles = new List <ResourceFile>()
                        {
                            GetResourceFileReference(ServiceRegistrationContainer, null),
                            GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder),
                            GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower())
                        };

                        job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine);
                        job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));

                        // List<ResourceFile> resourceFiles = new List<ResourceFile>();
                        // resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
                        // resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null));

                        // // job.JobManagerTask = new JobManagerTask("Broker",
                        // // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}");
                        // job.JobManagerTask = new JobManagerTask("List",
                        // $@"cmd /c dir & set");
                        // job.JobManagerTask.ResourceFiles = resourceFiles;
                        // job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));

                        // Set Meta Data
                        if (job.Metadata == null)
                        {
                            job.Metadata = new List <MetadataItem>();
                        }

                        Dictionary <string, string> jobMetadata = new Dictionary <string, string>()
                        {
                            { BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() },
                            { BrokerSettingsConstants.Secure, startInfo.Secure.ToString() },
                            { BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() },
                            { BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() },
                        };

                        if (startInfo.ServiceVersion != null)
                        {
                            jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString());
                        }

                        if (startInfo.MaxUnits != null)
                        {
                            jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString());
                        }

                        Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>()
                        {
                            { BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout },
                            { BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout },
                            { BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold },
                            { BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold },
                            { BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout },
                            { BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize },
                            { BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout },
                            { BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink }
                        };

                        job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value)))
                                       .Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList();

                        job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service";
                        await job.CommitAsync();

                        return(job.Id);
                    }

                    var jobId = await CreateJobAsync();

                    string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId);
                    if (!sessionId.Equals("-1"))
                    {
                        sessionAllocateInfo.Id = sessionId;
                    }
                    else
                    {
                        TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId);
                    }

                    Task AddTasksAsync()
                    {
                        int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count;

                        var comparer = new EnvironmentSettingComparer();

                        CloudTask CreateTask(string taskId)
                        {
                            CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone");

                            cloudTask.UserIdentity        = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
                            cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
                            return(cloudTask);
                        }

                        CloudTask CreateBrokerTask(bool direct)
                        {
                            List <ResourceFile> resourceFiles = new List <ResourceFile>();

                            resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));

                            string cmd;

                            if (direct)
                            {
                                cmd =
                                    $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
                            }
                            else
                            {
                                cmd =
                                    $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
                            }

                            CloudTask cloudTask = new CloudTask("Broker", cmd);

                            cloudTask.ResourceFiles       = resourceFiles;
                            cloudTask.UserIdentity        = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
                            cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
                            return(cloudTask);
                        }

                        //TODO: task id type should be changed from int to string
                        var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray();

                        if (!brokerPerfMode)
                        {
                            tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray();
                        }
                        else
                        {
                            tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray();
                        }

                        return(batchClient.JobOperations.AddTaskAsync(jobId, tasks));
                    }

                    await AddTasksAsync();

                    async Task WaitBatchBrokerLauncher()
                    {
                        var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker");

                        TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor();
                        await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout);

                        await brokerTask.RefreshAsync();

                        var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress;

                        sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) };
                    }

                    if (brokerPerfMode)
                    {
                        //If broker node and session launcher node is not the same node, this line should be modified.
                        sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) };
                    }
                    else
                    {
                        await WaitBatchBrokerLauncher();
                    }

                    return(sessionAllocateInfo);
                }
            }
            catch (Exception ex)
            {
                TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}");
                throw;
            }
        }