public override async Task TerminateAsync(string sessionId) { using (var batchClient = AzureBatchConfiguration.GetBatchClient()) { try { var batchJob = await batchClient.JobOperations.GetJobAsync(AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(sessionId)); //Conflict error occurs when terminate a job which is not in the state of Enabling or Active if (batchJob.State == Azure.Batch.Common.JobState.Active || batchJob.State == Azure.Batch.Common.JobState.Enabling || batchJob.State == Azure.Batch.Common.JobState.Disabling) { await batchJob.TerminateAsync(); } } catch (BatchException e) { if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode != null) { if (e.RequestInformation.HttpStatusCode == HttpStatusCode.NotFound) { TraceHelper.TraceEvent(sessionId, TraceEventType.Warning, "[AzureBatchSessionLauncher] .TerminateAsync: The specified job can't be found, maybe has deleted."); return; } } TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[AzureBatchSessionLauncher] .TerminateAsync: Job was failed to terminate : {0}.", e); } catch (Exception e) { TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[AzureBatchSessionLauncher] .TerminateAsync: Job was failed to terminate : {0}.", e); } } }
public override async Task <SessionInfoContract> GetInfoAsync(string endpointPrefix, string sessionId) { SessionInfoContract sessionInfo = null; this.CheckAccess(); ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix"); TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: headnode={0}, endpointPrefix={1}, sessionId={2}", Environment.MachineName, endpointPrefix, sessionId); if (!IsEndpointPrefixSupported(endpointPrefix)) { TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[SessionLauncher] .GetInfo: {0} is not a supported endpoint prefix.", endpointPrefix); ThrowHelper.ThrowSessionFault(SOAFaultCode.InvalidArgument, SR.SessionLauncher_EndpointNotSupported, endpointPrefix); } try { using (var batchClient = AzureBatchConfiguration.GetBatchClient()) { var jobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(sessionId); var sessionJob = await batchClient.JobOperations.GetJobAsync(jobId).ConfigureAwait(false); if (sessionJob == null) { throw new InvalidOperationException($"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.GetInfoAsync)} Failed to get batch job for session {sessionId}"); } TraceHelper.TraceEvent( sessionId, TraceEventType.Information, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.GetInfoAsync)}: try to get the job properties(Secure, TransportScheme, BrokerEpr, BrokerNode, ControllerEpr, ResponseEpr) for the job, jobid={sessionId}."); sessionInfo = new SessionInfoContract(); sessionInfo.Id = AzureBatchSessionJobIdConverter.ConvertToSessionId(sessionJob.Id); // TODO: sessionInfo.JobState var metadata = sessionJob.Metadata; if (metadata != null) { string brokerNodeString = null; foreach (var pair in metadata) { if (pair.Name.Equals(BrokerSettingsConstants.Secure, StringComparison.OrdinalIgnoreCase)) { string secureString = pair.Value; Debug.Assert(secureString != null, "BrokerSettingsConstants.Secure value should be a string."); bool secure; if (bool.TryParse(secureString, out secure)) { sessionInfo.Secure = secure; TraceHelper.TraceEvent(sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job secure property, Secure={0}.", secure); } else { TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.Secure + " property", secureString); } } else if (pair.Name.Equals(BrokerSettingsConstants.TransportScheme, StringComparison.OrdinalIgnoreCase)) { string schemeString = pair.Value; Debug.Assert(schemeString != null, "BrokerSettingsConstants.TransportScheme value should be a string."); int scheme; if (int.TryParse(schemeString, out scheme)) { sessionInfo.TransportScheme = (TransportScheme)scheme; TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job TransportScheme property, TransportScheme={0}.", sessionInfo.TransportScheme); } else { TraceHelper.TraceEvent( sessionId, TraceEventType.Error, "Illegal transport scheme value[{0}] for job's " + BrokerSettingsConstants.TransportScheme + " property", schemeString); } } else if (pair.Name.Equals(BrokerSettingsConstants.BrokerNode, StringComparison.OrdinalIgnoreCase)) { brokerNodeString = pair.Value; Debug.Assert(brokerNodeString != null, "BrokerSettingsConstants.BrokerNode value should be a string."); TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job BrokerLauncherEpr property, BrokerLauncherEpr={0}.", sessionInfo.BrokerLauncherEpr); } else if (pair.Name.Equals(BrokerSettingsConstants.Durable, StringComparison.OrdinalIgnoreCase)) { string durableString = pair.Value; Debug.Assert(durableString != null, "BrokerSettingsConstants.Durable value should be a string."); bool durable; if (bool.TryParse(durableString, out durable)) { sessionInfo.Durable = durable; TraceHelper.TraceEvent(sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job Durable property, Durable={0}.", sessionInfo.Durable); } else { TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.Durable + " property", durableString); } } else if (pair.Name.Equals(BrokerSettingsConstants.ServiceVersion, StringComparison.OrdinalIgnoreCase)) { if (pair.Value != null) { try { sessionInfo.ServiceVersion = new Version(pair.Value); TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job ServiceVersion property, ServiceVersion={0}.", (sessionInfo.ServiceVersion != null) ? sessionInfo.ServiceVersion.ToString() : string.Empty); } catch (Exception e) { TraceHelper.TraceEvent( sessionId, TraceEventType.Error, "Illegal secure value[{0}] for job's " + BrokerSettingsConstants.ServiceVersion + " property. Exception = {1}", pair.Value, e); } } } } if (!string.IsNullOrEmpty(brokerNodeString)) { if (brokerNodeString != Constant.InprocessBrokerNode) { sessionInfo.BrokerLauncherEpr = BrokerNodesManager.GenerateBrokerLauncherEpr(endpointPrefix, brokerNodeString, sessionInfo.TransportScheme); } } else { sessionInfo.UseInprocessBroker = true; } TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: get the job BrokerLauncherEpr property, BrokerLauncherEpr={0}.", sessionInfo.BrokerLauncherEpr); } } } catch (Exception e) { TraceHelper.TraceEvent(sessionId, TraceEventType.Error, "[SessionLauncher] .GetInfo: Failed to get all properties from job[{0}], Exception:{1}", sessionId, e); ThrowHelper.ThrowSessionFault(SOAFaultCode.GetJobPropertyFailure, SR.SessionLauncher_FailToGetJobProperty, e.ToString()); } if (sessionInfo.SessionOwner == null) { sessionInfo.SessionOwner = "Everyone"; } if (sessionInfo.SessionACL == null) { sessionInfo.SessionACL = new string[0]; } if (sessionInfo.JobState == 0) { // TODO: apply job state converting sessionInfo.JobState = JobState.Running; } TraceHelper.TraceEvent( sessionId, TraceEventType.Information, "[SessionLauncher] .GetInfo: return the sessionInfo, BrokerEpr={0}, BrokerLauncherEpr={1}, ControllerEpr={2}, Id={3}, JobState={4}, ResponseEpr={5}, Secure={6}, TransportScheme={7}, sessionOwner={8}, sessionACL={9}", sessionInfo.BrokerEpr, sessionInfo.BrokerLauncherEpr, sessionInfo.ControllerEpr, sessionInfo.Id, sessionInfo.JobState, sessionInfo.ResponseEpr, sessionInfo.Secure, sessionInfo.TransportScheme, sessionInfo.SessionOwner ?? "null", sessionInfo.SessionACL?.Length.ToString() ?? "null"); return(sessionInfo); }
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath) { try { bool brokerPerfMode = true; // TODO: implement separated broker mode if (brokerPerfMode) { TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode"); } TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.", callId, endpointPrefix, durable); using (var batchClient = AzureBatchConfiguration.GetBatchClient()) { var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName); ODATADetailLevel detailLevel = new ODATADetailLevel(); detailLevel.SelectClause = "affinityId, ipAddress"; //detailLevel.FilterClause = @"state eq 'idle'"; var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync(); if (nodes.Count < 1) { throw new InvalidOperationException("Compute node count in selected pool is less then 1."); } sessionAllocateInfo.Id = string.Empty; // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken }; IList <EnvironmentSetting> ConstructEnvironmentVariable() { List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables) { env.Add(new EnvironmentSetting(entry.Name, entry.Value)); } // pass service serviceInitializationTimeout as job environment variables env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString())); if (startInfo.ServiceHostIdleTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString())); } if (startInfo.ServiceHangTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString())); } // pass MessageLevelPreemption switcher as job environment variables env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString())); // pass trace switcher to svchost if (!string.IsNullOrEmpty(traceSwitchValue)) { env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue)); } // pass taskcancelgraceperiod as environment variable to svchosts env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString())); // pass service config file name to services env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName)); // pass maxMessageSize to service hosts int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize; env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString())); // pass service operation timeout to service hosts int?serviceOperationTimeout = null; if (startInfo.ServiceOperationTimeout.HasValue) { serviceOperationTimeout = startInfo.ServiceOperationTimeout; } else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null) { serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout; } if (serviceOperationTimeout.HasValue) { env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString())); } if (startInfo.Environments != null) { foreach (KeyValuePair <string, string> entry in startInfo.Environments) { env.Add(new EnvironmentSetting(entry.Key, entry.Value)); } } // Each SOA job is assigned a GUID "secret", which is used // to identify soa job owner. When a job running in Azure // tries to access common data, it sends this "secret" together // with a data request to data service. Data service trusts // the data request only if the job id and job "secret" // match. env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString())); // Set CCP_SERVICE_SESSIONPOOL env var of the job if (startInfo.UseSessionPool) { env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString)); } void SetBrokerNodeAuthenticationInfo() { // TODO: set the information needed by compute node to authenticate broker node return; } SetBrokerNodeAuthenticationInfo(); env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString())); env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString())); TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.", callId, BrokerSettingsConstants.Secure, startInfo.Secure, BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme); env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName())); env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE")); //Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir)); return(env); } var environment = ConstructEnvironmentVariable(); ResourceFile GetResourceFileReference(string containerName, string blobPrefix) { var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read); ResourceFile rf; if (string.IsNullOrEmpty(blobPrefix)) { rf = ResourceFile.FromStorageContainerUrl(sasToken); } else { rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix); } return(rf); } async Task <string> CreateJobAsync() { //TODO: need a function to test if all parameters are legal. if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0) { throw new ArgumentException("Maxunit value is invalid."); } string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId()); Debug.Assert(batchClient != null, nameof(batchClient) + " != null"); var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation() { PoolId = AzureBatchConfiguration.BatchPoolName }); job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine); job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); job.JobPreparationTask.ResourceFiles = new List <ResourceFile>() { GetResourceFileReference(ServiceRegistrationContainer, null), GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder), GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower()) }; job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine); job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // List<ResourceFile> resourceFiles = new List<ResourceFile>(); // resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); // resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null)); // // job.JobManagerTask = new JobManagerTask("Broker", // // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"); // job.JobManagerTask = new JobManagerTask("List", // $@"cmd /c dir & set"); // job.JobManagerTask.ResourceFiles = resourceFiles; // job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // Set Meta Data if (job.Metadata == null) { job.Metadata = new List <MetadataItem>(); } Dictionary <string, string> jobMetadata = new Dictionary <string, string>() { { BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() }, { BrokerSettingsConstants.Secure, startInfo.Secure.ToString() }, { BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() }, { BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() }, }; if (startInfo.ServiceVersion != null) { jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString()); } if (startInfo.MaxUnits != null) { jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString()); } Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>() { { BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout }, { BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout }, { BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold }, { BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold }, { BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout }, { BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize }, { BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout }, { BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink } }; job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value))) .Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList(); job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service"; await job.CommitAsync(); return(job.Id); } var jobId = await CreateJobAsync(); string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId); if (!sessionId.Equals("-1")) { sessionAllocateInfo.Id = sessionId; } else { TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId); } Task AddTasksAsync() { int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count; var comparer = new EnvironmentSettingComparer(); CloudTask CreateTask(string taskId) { CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone"); cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } CloudTask CreateBrokerTask(bool direct) { List <ResourceFile> resourceFiles = new List <ResourceFile>(); resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); string cmd; if (direct) { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } else { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } CloudTask cloudTask = new CloudTask("Broker", cmd); cloudTask.ResourceFiles = resourceFiles; cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } //TODO: task id type should be changed from int to string var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray(); if (!brokerPerfMode) { tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray(); } else { tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray(); } return(batchClient.JobOperations.AddTaskAsync(jobId, tasks)); } await AddTasksAsync(); async Task WaitBatchBrokerLauncher() { var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker"); TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor(); await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout); await brokerTask.RefreshAsync(); var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress; sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) }; } if (brokerPerfMode) { //If broker node and session launcher node is not the same node, this line should be modified. sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) }; } else { await WaitBatchBrokerLauncher(); } return(sessionAllocateInfo); } } catch (Exception ex) { TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}"); throw; } }