protected abstract bool TryGetSessionAllocateInfoFromPooled(
string endpointPrefix,
bool durable,
SessionAllocateInfoContract sessionAllocateInfo,
string serviceConfigFile,
ServiceRegistration registration,
out SessionAllocateInfoContract allocateInternal);
protected override bool TryGetSessionAllocateInfoFromPooled(
string endpointPrefix,
bool durable,
SessionAllocateInfoContract sessionAllocateInfo,
string serviceConfigFile,
ServiceRegistration registration,
out SessionAllocateInfoContract allocateInternal)
{
throw new NotSupportedException("Currently Session Launcher does not support session pool on Azure Batch.");
}
protected override bool TryGetSessionAllocateInfoFromPooled(
string endpointPrefix,
bool durable,
SessionAllocateInfoContract sessionAllocateInfo,
string serviceConfigFile,
ServiceRegistration registration,
out SessionAllocateInfoContract allocateInternal)
{
throw new NotImplementedException();
}
protected abstract Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath);
/// <summary>
/// Allocate resource for service job and provide broker epr
/// </summary>
/// <param name="startInfo">indicating session start information</param>
/// <param name="durable">indicating whether it is a durable session</param>
/// <param name="timeout">indicating the timeout</param>
/// <param name="eprs">output string array of available broker epr list</param>
/// <param name="sessionInfo">output the session info</param>
/// <returns>returns unique session id</returns>
public Task <SessionAllocateInfoContract> AllocateResource(SessionStartInfo startInfo, bool durable, TimeSpan timeout)
{
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
sessionAllocateInfo.BrokerLauncherEpr = new string[0];
sessionAllocateInfo.SessionInfo = null;
sessionAllocateInfo.Id = InprocessSessions.GetInstance().AllocateSessionId(durable);
#if net40
return(TaskEx.FromResult(sessionAllocateInfo));
#else
return(Task.FromResult(sessionAllocateInfo));
#endif
}
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
// string sessionId = LocalSessionConfiguration.GetNextSessionId();
string cmd;
if (true)
{
cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath}";
}
else
{
cmd =
$"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath} --AzureStorageConnectionString {LocalSessionConfiguration.BrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True";
}
string sessionId = SessionStartInfo.StandaloneSessionId;
this.brokerLauncherProcess = Process.Start(
LocalSessionConfiguration.BrokerLauncherExePath,
cmd);
this.svcHostProcess = Process.Start(LocalSessionConfiguration.ServiceHostExePath, "-standalone");
sessionAllocateInfo.Id = sessionId;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress("localhost") };
return(sessionAllocateInfo);
}
/// <summary>
/// Create session
/// </summary>
/// <param name="startInfo">indicating session start information</param>
/// <param name="durable">indicating a value whether a durable session is to be created</param>
/// <param name="timeoutMilliseconds">indicating the timeout</param>
/// <param name="binding">indicating the binding</param>
/// <returns>returns session instance</returns>
public override async Task <SessionBase> CreateSession(SessionStartInfo startInfo, bool durable, int timeoutMilliseconds, Binding binding)
{
SessionBase.CheckSessionStartInfo(startInfo);
DateTime targetTimeout = DateTime.Now.AddMilliseconds(Constant.DefaultCreateSessionTimeout);
// if following env variable is set, we will launch the service host in an admin job.
string adminJobEnv = Environment.GetEnvironmentVariable(SessionBase.EnvVarNameForAdminJob, EnvironmentVariableTarget.Process);
if (!string.IsNullOrEmpty(adminJobEnv) && adminJobEnv.Equals("1", StringComparison.InvariantCultureIgnoreCase))
{
startInfo.AdminJobForHostInDiag = true;
}
SessionAllocateInfoContract sessionAllocateInfo = null;
CredType typeOfExpectedCred = CredType.None;
IResourceProvider resourceProvider = null;
IList <string> newDataClients = null;
try
{
int retry = 0;
bool askForCredential = false;
// allow users to try credential at most MaxRetryCount times
int askForCredentialTimes = 0;
while (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
{
retry++;
try
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:Unknown] Start to create session for on-premise cluster. IsDurable = {0}, RetryCount = {1}, AskForCredentialTimes = {2}, TimeCost = {3}", durable, retry, askForCredentialTimes, targetTimeout);
Stopwatch watch = new Stopwatch();
watch.Start();
if (startInfo.UseAad)
{
// Authentication handled by AADUtil
}
else if (SoaHelper.IsSchedulerOnAzure(startInfo.Headnode) || SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode) || (startInfo.TransportScheme & TransportScheme.Http) == TransportScheme.Http || (startInfo.TransportScheme & TransportScheme.NetHttp) == TransportScheme.NetHttp)
{
askForCredential = HpcSessionCredUtil.RetrieveCredentialOnAzure(startInfo);
if (askForCredential)
{
askForCredentialTimes++;
}
}
else
{
askForCredential = await HpcSessionCredUtil.RetrieveCredentialOnPremise(startInfo, typeOfExpectedCred, binding).ConfigureAwait(false);
if (askForCredential)
{
askForCredentialTimes++;
}
HpcSessionCredUtil.CheckCredential(startInfo);
}
resourceProvider = BuildResourceProvider(startInfo, durable, binding);
if (((newDataClients == null) || (newDataClients.Count == 0)) &&
(startInfo.DependFiles != null) && (startInfo.DependFiles.Count > 0))
{
// Upload the data files required for this session
newDataClients = this.UploadAzureFiles(startInfo);
}
watch.Stop();
// re-calculate the timeout to exclude the timespan for getting credential
targetTimeout = targetTimeout.AddMilliseconds(watch.ElapsedMilliseconds);
sessionAllocateInfo = await resourceProvider.AllocateResource(startInfo, durable, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false);
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource.", sessionAllocateInfo.Id);
if (sessionAllocateInfo.BrokerLauncherEpr != null)
{
if (SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode))
{
string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(startInfo.Headnode);
for (int i = 0; i < sessionAllocateInfo.BrokerLauncherEpr.Length; i++)
{
sessionAllocateInfo.BrokerLauncherEpr[i] = SoaHelper.UpdateEprWithCloudServiceName(sessionAllocateInfo.BrokerLauncherEpr[i], suffix);
}
}
HpcSessionCredUtil.SaveCrendential(startInfo, binding);
SessionBase.TraceSource.TraceInformation("Get the EPR list from headnode. number of eprs={0}", sessionAllocateInfo.BrokerLauncherEpr.Length);
break;
}
}
catch (EndpointNotFoundException e)
{
this.CleanUpDataClients(startInfo, newDataClients);
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:Unknown] EndpointNotFoundException occured while allocating resource: {0}", e);
SessionBase.HandleEndpointNotFoundException(startInfo.Headnode);
}
catch (AuthenticationException e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] AuthenticationException occured while allocating resource: {0}", e);
startInfo.ClearCredential();
SessionBase.PurgeCredential(startInfo);
if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
{
if (typeOfExpectedCred == CredType.None)
{
typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false);
}
if (resourceProvider is IDisposable)
{
((IDisposable)resourceProvider).Dispose();
}
continue;
}
else
{
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw;
}
}
catch (MessageSecurityException e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] MessageSecurityException occured while allocating resource: {0}", e);
startInfo.ClearCredential();
SessionBase.PurgeCredential(startInfo);
if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
{
if (typeOfExpectedCred == CredType.None)
{
typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false);
}
if (resourceProvider is IDisposable)
{
((IDisposable)resourceProvider).Dispose();
}
continue;
}
else
{
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw;
}
}
catch (FaultException <SessionFault> ex)
{
typeOfExpectedCred = Utility.CanRetry(retry, askForCredential, askForCredentialTimes) ?
CredUtil.GetCredTypeFromFaultCode(ex.Detail.Code) : CredType.None;
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Fault exception occured while allocating resource. FaultCode = {0}. Exception = {1}", ex.Detail.Code, ex.ToString());
if (typeOfExpectedCred == CredType.None)
{
this.CleanUpDataClients(startInfo, newDataClients);
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw Utility.TranslateFaultException(ex);
}
else
{
startInfo.ClearCredential();
if (resourceProvider is IDisposable)
{
((IDisposable)resourceProvider).Dispose();
}
continue;
}
}
catch (CommunicationException e)
{
this.CleanUpDataClients(startInfo, newDataClients);
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw new SessionException(SOAFaultCode.ConnectSessionLauncherFailure, SR.ConnectSessionLauncherFailure, e);
}
catch (TimeoutException e)
{
SessionBase.TraceSource.TraceInformation(e.ToString());
this.CleanUpDataClients(startInfo, newDataClients);
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw new TimeoutException(string.Format(SR.ConnectSessionLauncherTimeout, Constant.DefaultCreateSessionTimeout), e);
}
catch (Exception e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Exception occured while allocating resource: {0}", e);
this.CleanUpDataClients(startInfo, newDataClients);
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw;
}
if (sessionAllocateInfo.BrokerLauncherEpr == null && startInfo.UseSessionPool)
{
// If the session launcher picks a session from the pool, it returns the seesion id.
// eprs is null at this case. Try to attach to the session.
try
{
if (sessionAllocateInfo.SessionInfo == null)
{
SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to session {0} which is part of the session pool.", sessionAllocateInfo.Id);
return(await AttachSession(new SessionAttachInfo(startInfo.Headnode, sessionAllocateInfo.Id), durable, timeoutMilliseconds, binding).ConfigureAwait(false));
}
else
{
SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to broker of the session {0} which is part of the session pool.", sessionAllocateInfo.Id);
return(await AttachBroker(startInfo, sessionAllocateInfo.SessionInfo, durable, timeoutMilliseconds, binding).ConfigureAwait(false));
}
}
catch (Exception e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "Failed to attach to the session {0}. {1}", sessionAllocateInfo.Id, e);
if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
{
if (resourceProvider is IDisposable)
{
((IDisposable)resourceProvider).Dispose();
}
continue;
}
else
{
throw;
}
}
}
}
IBrokerFactory brokerFactory = BuildBrokerFactory(startInfo, durable);
try
{
return(await brokerFactory.CreateBroker(startInfo, sessionAllocateInfo.Id, targetTimeout, sessionAllocateInfo.BrokerLauncherEpr, binding).ConfigureAwait(false));
}
catch
{
// Free resource if failed to create broker or create session
this.CleanUpDataClients(startInfo, newDataClients);
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
throw;
}
finally
{
if (brokerFactory is IDisposable)
{
((IDisposable)brokerFactory).Dispose();
}
}
}
finally
{
if (resourceProvider != null)
{
if (resourceProvider is IDisposable)
{
((IDisposable)resourceProvider).Dispose();
}
}
}
}
/// <summary>
/// Allocate a new durable or non-durable session
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="durable">whether session should be durable</param>
/// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param>
/// <returns>the Broker Launcher EPRs, sorted by the preference.</returns>
protected virtual async Task <SessionAllocateInfoContract> AllocateInternalAsync(SessionStartInfoContract startInfo, string endpointPrefix, bool durable)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] Begin: AllocateInternalAsync");
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNullOrEmpty(startInfo.ServiceName, "startInfo.ServiceName");
ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix");
#if HPCPACK
// check client api version, 4.3 or older client is not supported by 4.4 server for the broken changes
if (startInfo.ClientVersion == null || startInfo.ClientVersion < new Version(4, 4))
{
TraceHelper.TraceEvent(TraceEventType.Error,
"[SessionLauncher] .AllocateInternalAsync: ClientVersion {0} does not match ServerVersion {1}.", startInfo.ClientVersion, ServerVersion);
ThrowHelper.ThrowSessionFault(SOAFaultCode.ClientServerVersionMismatch,
SR.SessionLauncher_ClientServerVersionMismatch,
startInfo.ClientVersion == null ? "NULL" : startInfo.ClientVersion.ToString(),
ServerVersion.ToString());
}
#endif
// Init service version to the service version passed in
if (startInfo.ServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = startInfo.ServiceVersion;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is {0}", sessionAllocateInfo.ServiceVersion);
}
else
{
sessionAllocateInfo.ServiceVersion = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is null.");
}
string callId = Guid.NewGuid().ToString();
this.CheckAccess();
SecureString securePassword = CreateSecureString(startInfo.Password);
startInfo.Password = null;
// BUG 4522 : Use CCP_SCHEDULER when referencing service registration file share so HA HN virtual name is used when needed
//var reliableRegistry = new ReliableRegistry(this.fabricClient.PropertyManager);
//string defaultServiceRegistrationServerName = await reliableRegistry.GetValueAsync<string>(HpcConstants.HpcFullKeyName, HpcConstants.FileShareServerRegVal, this.token);
//if (String.IsNullOrEmpty(defaultServiceRegistrationServerName))
//{
// defaultServiceRegistrationServerName = "localhost";
//}
// the reg repo path is from scheduler environments, defaultServiceRegistrationServerName is actually not used
string serviceConfigFile;
ServiceRegistrationRepo serviceRegistration = this.GetRegistrationRepo(callId);
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, startInfo.ServiceVersion);
// If the serviceConfigFile wasnt found and serviceversion isnt specified, try getitng the service config based on the service's latest version
if (string.IsNullOrEmpty(serviceConfigFile) && (startInfo.ServiceVersion == null))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Try to find out versioned service.");
// Get service version in ServiceRegistrationRepo
Version dynamicServiceVersion = serviceRegistration.GetServiceVersionInternal(startInfo.ServiceName, false);
if (dynamicServiceVersion != null)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Selected dynamicServiceVersion is {0}.", dynamicServiceVersion.ToString());
}
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, dynamicServiceVersion);
// If a config file is found, update the serviceVersion that is returned to client and stored in recovery info
if (!string.IsNullOrEmpty(serviceConfigFile))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: serviceConfigFile is {0}.", serviceConfigFile);
startInfo.ServiceVersion = dynamicServiceVersion;
if (dynamicServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = dynamicServiceVersion;
}
}
}
string serviceName = ServiceRegistrationRepo.GetServiceRegistrationFileName(startInfo.ServiceName, startInfo.ServiceVersion);
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Service name = {0}, Configuration file = {1}", serviceName, serviceConfigFile);
// If the service is not found and user code doesn't specify
// version, we will use the latest version.
if (string.IsNullOrEmpty(serviceConfigFile))
{
if (startInfo.ServiceVersion != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ServiceVersion_NotFound, SR.SessionLauncher_ServiceVersionNotFound, startInfo.ServiceName, startInfo.ServiceVersion.ToString());
}
else
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_NotFound, SR.SessionLauncher_ServiceNotFound, startInfo.ServiceName);
}
}
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = serviceConfigFile;
ServiceRegistration registration = null;
BrokerConfigurations brokerConfigurations = null;
string hostpath = null;
string traceSwitchValue = null;
try
{
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException).GetAwaiter().GetResult();
Debug.Assert(config != null, "Configuration is not opened properly.");
registration = ServiceRegistration.GetSectionGroup(config);
brokerConfigurations = BrokerConfigurations.GetSectionGroup(config);
if (registration != null && registration.Host != null && registration.Host.Path != null)
{
hostpath = registration.Host.Path;
}
else
{
// x86 or x64
hostpath = registration.Service.Architecture == ServiceArch.X86 ? TaskCommandLine32 : TaskCommandLine64;
}
traceSwitchValue = registration.Service.SoaDiagTraceLevel;
// TODO: should deprecate the previous settings
if (string.IsNullOrEmpty(traceSwitchValue))
{
traceSwitchValue = ConfigurationHelper.GetTraceSwitchValue(config);
}
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, SR.SessionLauncher_ConfigFileInvalid, e.ToString());
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, ex.ToString());
throw;
}
// after figuring out the service and version, and the session pool size, we check if the service pool already has the instance.
sessionAllocateInfo.Id = "0";
sessionAllocateInfo.SessionInfo = null;
if (startInfo.UseSessionPool)
{
if (this.TryGetSessionAllocateInfoFromPooled(endpointPrefix, durable, sessionAllocateInfo, serviceConfigFile, registration, out var allocateInternal))
{
return(allocateInternal);
}
}
// for sessions to add in session pool
try
{
var sessionAllocateInfoContract = await this.CreateAndSubmitSessionJob(
startInfo,
endpointPrefix,
durable,
callId,
securePassword,
registration,
sessionAllocateInfo,
traceSwitchValue,
serviceName,
brokerConfigurations,
hostpath);
if (sessionAllocateInfoContract != null)
{
return(sessionAllocateInfoContract);
}
}
finally
{
// Add the submitted job to the session pool.
if (startInfo.UseSessionPool)
{
this.AddSessionToPool(Path.GetFileNameWithoutExtension(serviceConfigFile), durable, sessionAllocateInfo.Id, registration.Service.MaxSessionPoolSize);
}
}
return(null);
}
public override async Task <SessionBase> CreateSession(SessionStartInfo startInfo, bool durable, int timeoutMilliseconds, Binding binding)
{
SessionBase.CheckSessionStartInfo(startInfo);
DateTime targetTimeout = DateTime.Now.AddMilliseconds(Constant.DefaultCreateSessionTimeout);
SessionAllocateInfoContract sessionAllocateInfo = null;
IResourceProvider resourceProvider = null;
try
{
try
{
resourceProvider = this.BuildResourceProvider(startInfo, binding);
// re-calculate the timeout to exclude the timespan for getting credential
sessionAllocateInfo = await resourceProvider.AllocateResource(startInfo, durable, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false);
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource.", sessionAllocateInfo.Id);
if (sessionAllocateInfo.BrokerLauncherEpr != null &&
sessionAllocateInfo.BrokerLauncherEpr.Count() == 1 &&
sessionAllocateInfo.BrokerLauncherEpr[0] == SessionInternalConstants.BrokerConnectionStringToken)
{
if (!startInfo.UseAzureStorage)
{
SessionBase.TraceSource.TraceEvent(
TraceEventType.Error,
0,
"[Session:{0}] Server side only supports communication via AzureStorageQueue while client doesn't specify UseAzureQueue property and not using AzureStorage scheme.",
sessionAllocateInfo.Id);
throw new InvalidOperationException("Server side only supports communication via AzureStorageQueue while client doesn't specify UseAzureQueue property.");
}
}
}
catch (Exception e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Exception occured while allocating resource: {0}", e);
if (sessionAllocateInfo != null)
{
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
}
throw;
}
IBrokerFactory brokerFactory = BuildBrokerFactory(startInfo, durable);
try
{
return(await brokerFactory.CreateBroker(startInfo, sessionAllocateInfo.Id, targetTimeout, sessionAllocateInfo.BrokerLauncherEpr, binding).ConfigureAwait(false));
}
catch
{
// Free resource if failed to create broker or create session
await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
throw;
}
finally
{
if (brokerFactory != null && brokerFactory is IDisposable disposable)
{
disposable.Dispose();
}
}
}
finally
{
if (resourceProvider != null && resourceProvider is IDisposable disposable)
{
disposable.Dispose();
}
}
}
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
try
{
bool brokerPerfMode = true; // TODO: implement separated broker mode
if (brokerPerfMode)
{
TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode");
}
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.",
callId,
endpointPrefix,
durable);
using (var batchClient = AzureBatchConfiguration.GetBatchClient())
{
var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName);
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "affinityId, ipAddress";
//detailLevel.FilterClause = @"state eq 'idle'";
var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync();
if (nodes.Count < 1)
{
throw new InvalidOperationException("Compute node count in selected pool is less then 1.");
}
sessionAllocateInfo.Id = string.Empty;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
IList <EnvironmentSetting> ConstructEnvironmentVariable()
{
List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite
foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables)
{
env.Add(new EnvironmentSetting(entry.Name, entry.Value));
}
// pass service serviceInitializationTimeout as job environment variables
env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString()));
if (startInfo.ServiceHostIdleTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString()));
}
if (startInfo.ServiceHangTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString()));
}
// pass MessageLevelPreemption switcher as job environment variables
env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString()));
// pass trace switcher to svchost
if (!string.IsNullOrEmpty(traceSwitchValue))
{
env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue));
}
// pass taskcancelgraceperiod as environment variable to svchosts
env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString()));
// pass service config file name to services
env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName));
// pass maxMessageSize to service hosts
int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize;
env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString()));
// pass service operation timeout to service hosts
int?serviceOperationTimeout = null;
if (startInfo.ServiceOperationTimeout.HasValue)
{
serviceOperationTimeout = startInfo.ServiceOperationTimeout;
}
else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null)
{
serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout;
}
if (serviceOperationTimeout.HasValue)
{
env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString()));
}
if (startInfo.Environments != null)
{
foreach (KeyValuePair <string, string> entry in startInfo.Environments)
{
env.Add(new EnvironmentSetting(entry.Key, entry.Value));
}
}
// Each SOA job is assigned a GUID "secret", which is used
// to identify soa job owner. When a job running in Azure
// tries to access common data, it sends this "secret" together
// with a data request to data service. Data service trusts
// the data request only if the job id and job "secret"
// match.
env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString()));
// Set CCP_SERVICE_SESSIONPOOL env var of the job
if (startInfo.UseSessionPool)
{
env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString));
}
void SetBrokerNodeAuthenticationInfo()
{
// TODO: set the information needed by compute node to authenticate broker node
return;
}
SetBrokerNodeAuthenticationInfo();
env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString()));
env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString()));
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.",
callId,
BrokerSettingsConstants.Secure,
startInfo.Secure,
BrokerSettingsConstants.TransportScheme,
startInfo.TransportScheme);
env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName()));
env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE"));
//Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR
env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir));
return(env);
}
var environment = ConstructEnvironmentVariable();
ResourceFile GetResourceFileReference(string containerName, string blobPrefix)
{
var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read);
ResourceFile rf;
if (string.IsNullOrEmpty(blobPrefix))
{
rf = ResourceFile.FromStorageContainerUrl(sasToken);
}
else
{
rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix);
}
return(rf);
}
async Task <string> CreateJobAsync()
{
//TODO: need a function to test if all parameters are legal.
if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0)
{
throw new ArgumentException("Maxunit value is invalid.");
}
string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId());
Debug.Assert(batchClient != null, nameof(batchClient) + " != null");
var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation()
{
PoolId = AzureBatchConfiguration.BatchPoolName
});
job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine);
job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
job.JobPreparationTask.ResourceFiles = new List <ResourceFile>()
{
GetResourceFileReference(ServiceRegistrationContainer, null),
GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder),
GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower())
};
job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine);
job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// List<ResourceFile> resourceFiles = new List<ResourceFile>();
// resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
// resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null));
// // job.JobManagerTask = new JobManagerTask("Broker",
// // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}");
// job.JobManagerTask = new JobManagerTask("List",
// $@"cmd /c dir & set");
// job.JobManagerTask.ResourceFiles = resourceFiles;
// job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// Set Meta Data
if (job.Metadata == null)
{
job.Metadata = new List <MetadataItem>();
}
Dictionary <string, string> jobMetadata = new Dictionary <string, string>()
{
{ BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() },
{ BrokerSettingsConstants.Secure, startInfo.Secure.ToString() },
{ BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() },
{ BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() },
};
if (startInfo.ServiceVersion != null)
{
jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString());
}
if (startInfo.MaxUnits != null)
{
jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString());
}
Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>()
{
{ BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout },
{ BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout },
{ BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold },
{ BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold },
{ BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout },
{ BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize },
{ BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout },
{ BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink }
};
job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value)))
.Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList();
job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service";
await job.CommitAsync();
return(job.Id);
}
var jobId = await CreateJobAsync();
string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId);
if (!sessionId.Equals("-1"))
{
sessionAllocateInfo.Id = sessionId;
}
else
{
TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId);
}
Task AddTasksAsync()
{
int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count;
var comparer = new EnvironmentSettingComparer();
CloudTask CreateTask(string taskId)
{
CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone");
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
CloudTask CreateBrokerTask(bool direct)
{
List <ResourceFile> resourceFiles = new List <ResourceFile>();
resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
string cmd;
if (direct)
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
else
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
CloudTask cloudTask = new CloudTask("Broker", cmd);
cloudTask.ResourceFiles = resourceFiles;
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
//TODO: task id type should be changed from int to string
var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray();
if (!brokerPerfMode)
{
tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray();
}
else
{
tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray();
}
return(batchClient.JobOperations.AddTaskAsync(jobId, tasks));
}
await AddTasksAsync();
async Task WaitBatchBrokerLauncher()
{
var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker");
TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor();
await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout);
await brokerTask.RefreshAsync();
var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress;
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) };
}
if (brokerPerfMode)
{
//If broker node and session launcher node is not the same node, this line should be modified.
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) };
}
else
{
await WaitBatchBrokerLauncher();
}
return(sessionAllocateInfo);
}
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}");
throw;
}
}
public async Task <SessionAllocateInfoContract> AllocateResource(SessionStartInfo startInfo, bool durable, TimeSpan timeout)
{
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
this.client.InnerChannel.OperationTimeout = timeout;
RetryManager retry = SoaHelper.GetDefaultExponentialRetryManager();
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:Unknown] Allocating resource... IsDurable = {0}, is LocalUser = {1}", durable, startInfo.LocalUser);
DateTime startTime = DateTime.Now;
if (durable)
{
sessionAllocateInfo = await RetryHelper <SessionAllocateInfoContract> .InvokeOperationAsync(
async() => await this.client.AllocateDurableAsync(startInfo.Data, this.endpointPrefix).ConfigureAwait(false),
(e, r) =>
{
var remainingTime = GetRemainingTime(timeout, startTime);
if ((e is EndpointNotFoundException || (e is CommunicationException && !(e is FaultException <SessionFault>))) && remainingTime > TimeSpan.Zero)
{
Utility.SafeCloseCommunicateObject(this.client);
this.client = new SessionLauncherClient(startInfo, this.binding);
this.client.InnerChannel.OperationTimeout = remainingTime;
}
else
{
r.MaxRetryCount = 0;
}
return(Task.CompletedTask);
},
retry).ConfigureAwait(false);
}
else
{
sessionAllocateInfo = await RetryHelper <SessionAllocateInfoContract> .InvokeOperationAsync(
async() => await this.client.AllocateAsync(startInfo.Data, this.endpointPrefix).ConfigureAwait(false),
(e, r) =>
{
var remainingTime = GetRemainingTime(timeout, startTime);
if ((e is EndpointNotFoundException || (e is CommunicationException && !(e is FaultException <SessionFault>))) && remainingTime > TimeSpan.Zero)
{
Utility.SafeCloseCommunicateObject(this.client);
this.client = new SessionLauncherClient(startInfo, this.binding);
this.client.InnerChannel.OperationTimeout = remainingTime;
}
else
{
r.MaxRetryCount = 0;
}
return(Task.CompletedTask);
},
retry).ConfigureAwait(false);
}
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource. Eprs = {1}", sessionAllocateInfo.Id, sessionAllocateInfo.BrokerLauncherEpr == null ? string.Empty : string.Join(",", sessionAllocateInfo.BrokerLauncherEpr));
if (sessionAllocateInfo.ServiceVersion != null)
{
try
{
startInfo.Data.ServiceVersion = sessionAllocateInfo.ServiceVersion;
}
catch
{
throw new SessionException(SR.InvalidServiceVersionReturned);
}
}
if (startInfo.UseSessionPool)
{
return(sessionAllocateInfo);
}
else
{
if (!startInfo.UseInprocessBroker && (sessionAllocateInfo.BrokerLauncherEpr == null || sessionAllocateInfo.BrokerLauncherEpr.Length == 0))
{
throw new SessionException(SR.NoBrokerNodeFound);
}
return(sessionAllocateInfo);
}
}