public void BuildFrontEndTest_Http()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
info.TransportScheme = TransportScheme.Http;
info.Secure = true;
BrokerStartInfo startInfo = new BrokerStartInfo();
MockBrokerAuthorization auth = new MockBrokerAuthorization();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = false;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList[1], @"https://.+/100/Http/Controller"));
Assert.AreEqual(null, result.GetResponseUriList.FirstOrDefault());
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList[1], @"https://.+/100/Http"));
}
/// <summary>
/// Initializes a new instance of the SharedData class
/// </summary>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="startInfo">indicating the start info</param>
/// <param name="config">indicating the broker configuration</param>
/// <param name="serviceCnfig">indicating the service configuration</param>
public SharedData(BrokerStartInfo brokerInfo, SessionStartInfoContract startInfo, BrokerConfigurations config, ServiceConfiguration serviceConfig)
{
this.brokerInfo = brokerInfo;
this.startInfo = startInfo;
this.config = config;
this.serviceConfig = serviceConfig;
}
public void BuildFrontEndTest_NetTcp()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
BrokerStartInfo startInfo = new BrokerStartInfo();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = true;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/Controller"));
Assert.IsTrue(Regex.IsMatch(result.GetResponseUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/GetResponse"));
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp"));
}
/// <summary>
/// Gets the custom broker registration from service registration file
/// </summary>
/// <param name="serviceRegistrationPath">indicating the path of the service registration file</param>
/// <returns>returns the instance of CustomBrokerRegistration class</returns>
private static CustomBrokerRegistration GetCustomBroker(string serviceRegistrationPath)
{
try
{
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = serviceRegistrationPath;
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException)
.GetAwaiter()
.GetResult();
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
if (brokerConfig == null)
{
return(null);
}
else
{
return(brokerConfig.CustomBroker);
}
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid,
"{0}",
e.ToString());
return(null);
}
}
static ConfigurationHelper()
{
defaultBrokerConfiguration = new BrokerConfigurations();
InitMonitorConfig();
InitServicesConfig();
InitLoadBalancingConfig();
}
protected abstract Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath);
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
// string sessionId = LocalSessionConfiguration.GetNextSessionId();
string cmd;
if (true)
{
cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath}";
}
else
{
cmd =
$"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath} --AzureStorageConnectionString {LocalSessionConfiguration.BrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True";
}
string sessionId = SessionStartInfo.StandaloneSessionId;
this.brokerLauncherProcess = Process.Start(
LocalSessionConfiguration.BrokerLauncherExePath,
cmd);
this.svcHostProcess = Process.Start(LocalSessionConfiguration.ServiceHostExePath, "-standalone");
sessionAllocateInfo.Id = sessionId;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress("localhost") };
return(sessionAllocateInfo);
}
/// <summary>
/// Allocate a new durable or non-durable session
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="durable">whether session should be durable</param>
/// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param>
/// <returns>the Broker Launcher EPRs, sorted by the preference.</returns>
protected virtual async Task <SessionAllocateInfoContract> AllocateInternalAsync(SessionStartInfoContract startInfo, string endpointPrefix, bool durable)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] Begin: AllocateInternalAsync");
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNullOrEmpty(startInfo.ServiceName, "startInfo.ServiceName");
ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix");
#if HPCPACK
// check client api version, 4.3 or older client is not supported by 4.4 server for the broken changes
if (startInfo.ClientVersion == null || startInfo.ClientVersion < new Version(4, 4))
{
TraceHelper.TraceEvent(TraceEventType.Error,
"[SessionLauncher] .AllocateInternalAsync: ClientVersion {0} does not match ServerVersion {1}.", startInfo.ClientVersion, ServerVersion);
ThrowHelper.ThrowSessionFault(SOAFaultCode.ClientServerVersionMismatch,
SR.SessionLauncher_ClientServerVersionMismatch,
startInfo.ClientVersion == null ? "NULL" : startInfo.ClientVersion.ToString(),
ServerVersion.ToString());
}
#endif
// Init service version to the service version passed in
if (startInfo.ServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = startInfo.ServiceVersion;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is {0}", sessionAllocateInfo.ServiceVersion);
}
else
{
sessionAllocateInfo.ServiceVersion = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is null.");
}
string callId = Guid.NewGuid().ToString();
this.CheckAccess();
SecureString securePassword = CreateSecureString(startInfo.Password);
startInfo.Password = null;
// BUG 4522 : Use CCP_SCHEDULER when referencing service registration file share so HA HN virtual name is used when needed
//var reliableRegistry = new ReliableRegistry(this.fabricClient.PropertyManager);
//string defaultServiceRegistrationServerName = await reliableRegistry.GetValueAsync<string>(HpcConstants.HpcFullKeyName, HpcConstants.FileShareServerRegVal, this.token);
//if (String.IsNullOrEmpty(defaultServiceRegistrationServerName))
//{
// defaultServiceRegistrationServerName = "localhost";
//}
// the reg repo path is from scheduler environments, defaultServiceRegistrationServerName is actually not used
string serviceConfigFile;
ServiceRegistrationRepo serviceRegistration = this.GetRegistrationRepo(callId);
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, startInfo.ServiceVersion);
// If the serviceConfigFile wasnt found and serviceversion isnt specified, try getitng the service config based on the service's latest version
if (string.IsNullOrEmpty(serviceConfigFile) && (startInfo.ServiceVersion == null))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Try to find out versioned service.");
// Get service version in ServiceRegistrationRepo
Version dynamicServiceVersion = serviceRegistration.GetServiceVersionInternal(startInfo.ServiceName, false);
if (dynamicServiceVersion != null)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Selected dynamicServiceVersion is {0}.", dynamicServiceVersion.ToString());
}
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, dynamicServiceVersion);
// If a config file is found, update the serviceVersion that is returned to client and stored in recovery info
if (!string.IsNullOrEmpty(serviceConfigFile))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: serviceConfigFile is {0}.", serviceConfigFile);
startInfo.ServiceVersion = dynamicServiceVersion;
if (dynamicServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = dynamicServiceVersion;
}
}
}
string serviceName = ServiceRegistrationRepo.GetServiceRegistrationFileName(startInfo.ServiceName, startInfo.ServiceVersion);
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Service name = {0}, Configuration file = {1}", serviceName, serviceConfigFile);
// If the service is not found and user code doesn't specify
// version, we will use the latest version.
if (string.IsNullOrEmpty(serviceConfigFile))
{
if (startInfo.ServiceVersion != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ServiceVersion_NotFound, SR.SessionLauncher_ServiceVersionNotFound, startInfo.ServiceName, startInfo.ServiceVersion.ToString());
}
else
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_NotFound, SR.SessionLauncher_ServiceNotFound, startInfo.ServiceName);
}
}
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = serviceConfigFile;
ServiceRegistration registration = null;
BrokerConfigurations brokerConfigurations = null;
string hostpath = null;
string traceSwitchValue = null;
try
{
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException).GetAwaiter().GetResult();
Debug.Assert(config != null, "Configuration is not opened properly.");
registration = ServiceRegistration.GetSectionGroup(config);
brokerConfigurations = BrokerConfigurations.GetSectionGroup(config);
if (registration != null && registration.Host != null && registration.Host.Path != null)
{
hostpath = registration.Host.Path;
}
else
{
// x86 or x64
hostpath = registration.Service.Architecture == ServiceArch.X86 ? TaskCommandLine32 : TaskCommandLine64;
}
traceSwitchValue = registration.Service.SoaDiagTraceLevel;
// TODO: should deprecate the previous settings
if (string.IsNullOrEmpty(traceSwitchValue))
{
traceSwitchValue = ConfigurationHelper.GetTraceSwitchValue(config);
}
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, SR.SessionLauncher_ConfigFileInvalid, e.ToString());
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, ex.ToString());
throw;
}
// after figuring out the service and version, and the session pool size, we check if the service pool already has the instance.
sessionAllocateInfo.Id = "0";
sessionAllocateInfo.SessionInfo = null;
if (startInfo.UseSessionPool)
{
if (this.TryGetSessionAllocateInfoFromPooled(endpointPrefix, durable, sessionAllocateInfo, serviceConfigFile, registration, out var allocateInternal))
{
return(allocateInternal);
}
}
// for sessions to add in session pool
try
{
var sessionAllocateInfoContract = await this.CreateAndSubmitSessionJob(
startInfo,
endpointPrefix,
durable,
callId,
securePassword,
registration,
sessionAllocateInfo,
traceSwitchValue,
serviceName,
brokerConfigurations,
hostpath);
if (sessionAllocateInfoContract != null)
{
return(sessionAllocateInfoContract);
}
}
finally
{
// Add the submitted job to the session pool.
if (startInfo.UseSessionPool)
{
this.AddSessionToPool(Path.GetFileNameWithoutExtension(serviceConfigFile), durable, sessionAllocateInfo.Id, registration.Service.MaxSessionPoolSize);
}
}
return(null);
}
/// <summary>
/// Load the configuration from the configuration file
/// If no configuration is found or some part is missing, we will fill the blank with default value
/// </summary>
/// <param name="brokerSettings">indicate the broker settings from the session start info, this settings will override the settings load from the configuration file</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="brokerConfig">out the broker configurations</param>
/// <param name="serviceConfig">out the service configurations</param>
/// <param name="bindings">output the bindings</param>
public static void LoadConfiguration(SessionStartInfoContract brokerSettings, BrokerStartInfo brokerInfo, out BrokerConfigurations brokerConfig, out ServiceConfiguration serviceConfig, out BindingsSection bindings)
{
// Init config file
string filename = brokerInfo.ConfigurationFile;
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 1: Load configuration file name: {0}", filename);
brokerConfig = null;
serviceConfig = null;
bindings = null;
try
{
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException)
.GetAwaiter()
.GetResult();
brokerConfig = BrokerConfigurations.GetSectionGroup(config);
serviceConfig = ServiceRegistration.GetSectionGroup(config).Service;
bindings = (BindingsSection)config.GetSection(BindingsSectionName);
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid,
"{0}",
e.ToString());
}
if (brokerConfig == null)
{
// Set the default value
brokerConfig = defaultBrokerConfiguration;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the broker config from the configuration file, use the default configuration");
}
else
{
// Set the default configuration if the very section is not found
if (brokerConfig.Monitor == null)
{
brokerConfig.Monitor = defaultBrokerConfiguration.Monitor;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the monitor config from the configuration file, use the default configuration");
}
if (brokerConfig.Services == null)
{
brokerConfig.Services = defaultBrokerConfiguration.Services;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the services config from the configuration file, use the default configuration");
}
if (brokerConfig.LoadBalancing == null)
{
brokerConfig.LoadBalancing = defaultBrokerConfiguration.LoadBalancing;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the load balancing config from the configuration file, use the default configuration");
}
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 2: Load broker config and service config succeeded.");
if (brokerConfig.Monitor.AllocationAdjustInterval < MinAllocationAdjustTime && brokerConfig.Monitor.AllocationAdjustInterval != System.Threading.Timeout.Infinite)
{
brokerConfig.Monitor.AllocationAdjustInterval = MinAllocationAdjustTime;
}
// Update the broker config using the session start info
if (brokerSettings.ClientIdleTimeout.HasValue)
{
brokerConfig.Monitor.ClientIdleTimeout = brokerSettings.ClientIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientIdleTimeout to {0}", brokerConfig.Monitor.ClientIdleTimeout);
}
if (brokerSettings.ClientConnectionTimeout.HasValue)
{
brokerConfig.Monitor.ClientConnectionTimeout = brokerSettings.ClientConnectionTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientConnectionTimeout to {0}", brokerConfig.Monitor.ClientConnectionTimeout);
}
if (brokerSettings.SessionIdleTimeout.HasValue)
{
brokerConfig.Monitor.SessionIdleTimeout = brokerSettings.SessionIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default SessionIdleTimeout to {0}", brokerConfig.Monitor.SessionIdleTimeout);
}
if (brokerSettings.MessagesThrottleStartThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStartThreshold = brokerSettings.MessagesThrottleStartThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStartThreshold to {0}", brokerConfig.Monitor.MessageThrottleStartThreshold);
}
if (brokerSettings.MessagesThrottleStopThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStopThreshold = brokerSettings.MessagesThrottleStopThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStopThreshold to {0}", brokerConfig.Monitor.MessageThrottleStopThreshold);
}
if (brokerSettings.ClientBrokerHeartbeatRetryCount.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount = brokerSettings.ClientBrokerHeartbeatRetryCount.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatRetryCount to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount);
}
if (brokerSettings.ClientBrokerHeartbeatInterval.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatInterval = brokerSettings.ClientBrokerHeartbeatInterval.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatInterval to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatInterval);
}
if (brokerSettings.ServiceOperationTimeout.HasValue)
{
brokerConfig.LoadBalancing.ServiceOperationTimeout = brokerSettings.ServiceOperationTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ServiceOperationTimeout to {0}", brokerConfig.LoadBalancing.ServiceOperationTimeout);
}
if (brokerSettings.DispatcherCapacityInGrowShrink.HasValue)
{
brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink = brokerSettings.DispatcherCapacityInGrowShrink.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default DispatcherCapacityInGrowShrink to {0}", brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink);
}
if (brokerSettings.MaxMessageSize.HasValue)
{
serviceConfig.MaxMessageSize = brokerSettings.MaxMessageSize.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MaxMessageSize to {0}", serviceConfig.MaxMessageSize);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 3: Override broker settings using session start info succeeded.");
// Validate the config section
string configError;
bool validateSucceeded;
try
{
validateSucceeded = brokerConfig.Validate(out configError);
}
catch (ConfigurationErrorsException e)
{
validateSucceeded = false;
configError = e.Message;
}
if (!validateSucceeded)
{
BrokerTracing.TraceEvent(TraceEventType.Error, 0, "[ConfigurationHelper] Invalid broker configuration section. Error {0}", configError);
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_InvalidConfiguration, configError);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 4: Validate broker configuration succeeded.");
StringBuilder sb = new StringBuilder();
sb.AppendLine("[Monitor]");
BrokerTracing.WriteProperties(sb, brokerConfig.Monitor, 3, typeof(int), typeof(string));
sb.AppendLine("[BaseAddress]");
sb.AppendFormat(" Http = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("http"));
sb.AppendFormat(" Https = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("https"));
sb.AppendFormat(" NetTcp = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("net.tcp"));
sb.AppendLine("[LoadBalancing]");
BrokerTracing.WriteProperties(sb, brokerConfig.LoadBalancing, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] BrokerConfiguration: \n{0}", sb.ToString());
sb = new StringBuilder();
sb.AppendLine("[Service]");
BrokerTracing.WriteProperties(sb, serviceConfig, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] ServiceConfiguration: \n{0}", sb.ToString());
}
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
try
{
bool brokerPerfMode = true; // TODO: implement separated broker mode
if (brokerPerfMode)
{
TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode");
}
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.",
callId,
endpointPrefix,
durable);
using (var batchClient = AzureBatchConfiguration.GetBatchClient())
{
var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName);
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "affinityId, ipAddress";
//detailLevel.FilterClause = @"state eq 'idle'";
var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync();
if (nodes.Count < 1)
{
throw new InvalidOperationException("Compute node count in selected pool is less then 1.");
}
sessionAllocateInfo.Id = string.Empty;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
IList <EnvironmentSetting> ConstructEnvironmentVariable()
{
List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite
foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables)
{
env.Add(new EnvironmentSetting(entry.Name, entry.Value));
}
// pass service serviceInitializationTimeout as job environment variables
env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString()));
if (startInfo.ServiceHostIdleTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString()));
}
if (startInfo.ServiceHangTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString()));
}
// pass MessageLevelPreemption switcher as job environment variables
env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString()));
// pass trace switcher to svchost
if (!string.IsNullOrEmpty(traceSwitchValue))
{
env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue));
}
// pass taskcancelgraceperiod as environment variable to svchosts
env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString()));
// pass service config file name to services
env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName));
// pass maxMessageSize to service hosts
int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize;
env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString()));
// pass service operation timeout to service hosts
int?serviceOperationTimeout = null;
if (startInfo.ServiceOperationTimeout.HasValue)
{
serviceOperationTimeout = startInfo.ServiceOperationTimeout;
}
else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null)
{
serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout;
}
if (serviceOperationTimeout.HasValue)
{
env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString()));
}
if (startInfo.Environments != null)
{
foreach (KeyValuePair <string, string> entry in startInfo.Environments)
{
env.Add(new EnvironmentSetting(entry.Key, entry.Value));
}
}
// Each SOA job is assigned a GUID "secret", which is used
// to identify soa job owner. When a job running in Azure
// tries to access common data, it sends this "secret" together
// with a data request to data service. Data service trusts
// the data request only if the job id and job "secret"
// match.
env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString()));
// Set CCP_SERVICE_SESSIONPOOL env var of the job
if (startInfo.UseSessionPool)
{
env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString));
}
void SetBrokerNodeAuthenticationInfo()
{
// TODO: set the information needed by compute node to authenticate broker node
return;
}
SetBrokerNodeAuthenticationInfo();
env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString()));
env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString()));
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.",
callId,
BrokerSettingsConstants.Secure,
startInfo.Secure,
BrokerSettingsConstants.TransportScheme,
startInfo.TransportScheme);
env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName()));
env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE"));
//Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR
env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir));
return(env);
}
var environment = ConstructEnvironmentVariable();
ResourceFile GetResourceFileReference(string containerName, string blobPrefix)
{
var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read);
ResourceFile rf;
if (string.IsNullOrEmpty(blobPrefix))
{
rf = ResourceFile.FromStorageContainerUrl(sasToken);
}
else
{
rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix);
}
return(rf);
}
async Task <string> CreateJobAsync()
{
//TODO: need a function to test if all parameters are legal.
if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0)
{
throw new ArgumentException("Maxunit value is invalid.");
}
string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId());
Debug.Assert(batchClient != null, nameof(batchClient) + " != null");
var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation()
{
PoolId = AzureBatchConfiguration.BatchPoolName
});
job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine);
job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
job.JobPreparationTask.ResourceFiles = new List <ResourceFile>()
{
GetResourceFileReference(ServiceRegistrationContainer, null),
GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder),
GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower())
};
job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine);
job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// List<ResourceFile> resourceFiles = new List<ResourceFile>();
// resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
// resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null));
// // job.JobManagerTask = new JobManagerTask("Broker",
// // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}");
// job.JobManagerTask = new JobManagerTask("List",
// $@"cmd /c dir & set");
// job.JobManagerTask.ResourceFiles = resourceFiles;
// job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// Set Meta Data
if (job.Metadata == null)
{
job.Metadata = new List <MetadataItem>();
}
Dictionary <string, string> jobMetadata = new Dictionary <string, string>()
{
{ BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() },
{ BrokerSettingsConstants.Secure, startInfo.Secure.ToString() },
{ BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() },
{ BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() },
};
if (startInfo.ServiceVersion != null)
{
jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString());
}
if (startInfo.MaxUnits != null)
{
jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString());
}
Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>()
{
{ BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout },
{ BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout },
{ BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold },
{ BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold },
{ BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout },
{ BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize },
{ BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout },
{ BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink }
};
job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value)))
.Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList();
job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service";
await job.CommitAsync();
return(job.Id);
}
var jobId = await CreateJobAsync();
string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId);
if (!sessionId.Equals("-1"))
{
sessionAllocateInfo.Id = sessionId;
}
else
{
TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId);
}
Task AddTasksAsync()
{
int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count;
var comparer = new EnvironmentSettingComparer();
CloudTask CreateTask(string taskId)
{
CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone");
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
CloudTask CreateBrokerTask(bool direct)
{
List <ResourceFile> resourceFiles = new List <ResourceFile>();
resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
string cmd;
if (direct)
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
else
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
CloudTask cloudTask = new CloudTask("Broker", cmd);
cloudTask.ResourceFiles = resourceFiles;
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
//TODO: task id type should be changed from int to string
var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray();
if (!brokerPerfMode)
{
tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray();
}
else
{
tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray();
}
return(batchClient.JobOperations.AddTaskAsync(jobId, tasks));
}
await AddTasksAsync();
async Task WaitBatchBrokerLauncher()
{
var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker");
TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor();
await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout);
await brokerTask.RefreshAsync();
var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress;
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) };
}
if (brokerPerfMode)
{
//If broker node and session launcher node is not the same node, this line should be modified.
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) };
}
else
{
await WaitBatchBrokerLauncher();
}
return(sessionAllocateInfo);
}
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}");
throw;
}
}
