public void BuildFrontEndTest_Http()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
info.TransportScheme = TransportScheme.Http;
info.Secure = true;
BrokerStartInfo startInfo = new BrokerStartInfo();
MockBrokerAuthorization auth = new MockBrokerAuthorization();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = false;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList[1], @"https://.+/100/Http/Controller"));
Assert.AreEqual(null, result.GetResponseUriList.FirstOrDefault());
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList[1], @"https://.+/100/Http"));
}
/// <summary>
/// Initializes a new instance of the SharedData class
/// </summary>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="startInfo">indicating the start info</param>
/// <param name="config">indicating the broker configuration</param>
/// <param name="serviceCnfig">indicating the service configuration</param>
public SharedData(BrokerStartInfo brokerInfo, SessionStartInfoContract startInfo, BrokerConfigurations config, ServiceConfiguration serviceConfig)
{
this.brokerInfo = brokerInfo;
this.startInfo = startInfo;
this.config = config;
this.serviceConfig = serviceConfig;
}
public void BuildFrontEndTest_NetTcp()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
BrokerStartInfo startInfo = new BrokerStartInfo();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = true;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/Controller"));
Assert.IsTrue(Regex.IsMatch(result.GetResponseUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/GetResponse"));
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp"));
}
private void BuildStandaloneBrokerStartInfo(string serviceName, Version serviceVersion, string sessionId, bool durable, bool attached)
{
this.brokerInfo = new BrokerStartInfo();
this.brokerInfo.Headnode = this.startInfo.Headnode;
// Secure will be set to false and the following two settings are ignored
this.brokerInfo.JobOwnerSID = null;
this.brokerInfo.JobTemplateACL = null;
this.brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
this.brokerInfo.SessionId = sessionId;
this.brokerInfo.Attached = attached;
this.brokerInfo.Durable = durable;
this.brokerInfo.NetworkPrefix = Constant.EnterpriseNetwork;
//rewrite the method building configuration file since regpath gotten from input.
string regPath = this.startInfo.RegPath;
ServiceRegistrationRepo serviceRegistration = new ServiceRegistrationRepo(regPath, null);
this.brokerInfo.ConfigurationFile = serviceRegistration.GetServiceRegistrationPath(serviceName, serviceVersion);
if (string.IsNullOrEmpty(this.brokerInfo.ConfigurationFile) && serviceVersion == null)
{
Version version = serviceRegistration.GetServiceVersionInternal(serviceName, false);
if (version != null)
{
string serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(serviceName, version);
if (!string.IsNullOrEmpty(serviceConfigFile))
{
this.brokerInfo.ConfigurationFile = serviceConfigFile;
this.startInfoContract.ServiceVersion = version;
}
}
}
}
/// <summary>
/// Initializes a new instance of the BrokerInfo class from broker recover info
/// </summary>
/// <param name="recoverInfo">indicating the broker recover info</param>
/// <param name="brokerInfo">indicating the broker start info</param>
/// <param name="auth">indicating the broker auth</param>
/// <param name="customBroker">indicating the custom broker configuration</param>
/// <param name="pool">indicating the broker process pool</param>
public BrokerInfo(BrokerRecoverInfo recoverInfo, BrokerStartInfo brokerInfo, BrokerAuthorization auth, CustomBrokerRegistration customBroker, BrokerProcessPool pool)
{
this.callbackToCloseBroker = new ThreadHelper <IAsyncResult>(new AsyncCallback(this.OnCloseBroker)).CallbackRoot;
this.durable = recoverInfo.Durable;
this.brokerInfo = brokerInfo;
this.sessionId = recoverInfo.SessionId;
this.sessionStartInfo = recoverInfo.StartInfo;
this.auth = auth;
this.customBroker = customBroker;
this.pool = pool;
this.sessionStartInfo.IpAddress = BrokerLauncherSettings.Default.SvcHostList.Cast <string>().ToArray();
this.sessionStartInfo.RegPath = BrokerLauncherSettings.Default.CCP_SERVICEREGISTRATION_PATH;
}
/// <summary>
/// Update the broker info
/// </summary>
/// <param name="info">broker info</param>
private async Task UpdateBrokerInfo(BrokerStartInfo info)
{
int retry = 3;
Dictionary <string, object> properties = new Dictionary <string, object>();
properties.Add(BrokerSettingsConstants.BrokerNode, Constant.InprocessBrokerNode);
properties.Add(BrokerSettingsConstants.Suspended, info.Durable);
properties.Add(BrokerSettingsConstants.Durable, info.Durable);
properties.Add(BrokerSettingsConstants.PersistVersion, info.PersistVersion);
//SchedulerAdapterInternalClient client = new SchedulerAdapterInternalClient(await this.startInfo.ResolveHeadnodeMachineAsync().ConfigureAwait(false));
ISchedulerAdapter client = SessionServiceContainer.SchedulerAdapterInstance;
try
{
while (retry > 0)
{
try
{
if (await client.UpdateBrokerInfoAsync(info.SessionId, properties).ConfigureAwait(false))
{
return;
}
}
catch (Exception ex)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[BrokerLauncher.SchedulerHelper] UpdateBrokerInfo failed: Exception = {0}", ex);
}
retry--;
}
}
finally
{
var communicateObj = client as ICommunicationObject;
if (communicateObj != null)
{
Utility.SafeCloseCommunicateObject(communicateObj);
}
}
throw new InvalidOperationException("Can not update the properties in the scheduler database for EPRs");
}
/// <summary>
/// Initializes a new instance of the BrokerEntry class
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance)
{
if (startInfo.TransportScheme != TransportScheme.NetTcp)
{
throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp.");
}
if (brokerInfo.Durable)
{
throw new NotSupportedException("Sample broker does not support durable session.");
}
this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance);
Frontend frontend = new Frontend(this.clientManager);
WebHttpBinding binding = new WebHttpBinding();
binding.MaxBufferPoolSize = 5000000;
binding.MaxBufferSize = 5000000;
binding.MaxReceivedMessageSize = 5000000;
binding.ReaderQuotas.MaxArrayLength = 5000000;
binding.ReaderQuotas.MaxBytesPerRead = 5000000;
binding.ReaderQuotas.MaxDepth = 5000000;
binding.ReaderQuotas.MaxNameTableCharCount = 5000000;
binding.ReaderQuotas.MaxStringContentLength = 5000000;
this.frontendServiceHost = new WebServiceHost(frontend, new Uri(String.Format("http://{0}:8081/", Environment.MachineName)));
ServiceEndpoint endpoint = this.frontendServiceHost.AddServiceEndpoint(typeof(IWebHttpFrontendService), binding, String.Empty);
endpoint.Behaviors.Add(new WebHttpBehavior());
string listenUri = endpoint.ListenUri.AbsoluteUri;
this.frontendServiceHost.Open();
this.result = new BrokerInitializationResult();
this.result.BrokerEpr = new string[3] {
listenUri, null, null
};
this.result.ControllerEpr = new string[3];
this.result.ResponseEpr = new string[3];
}
/// <summary>
/// Initializes a new instance of the BrokerEntry class
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance)
{
if (startInfo.TransportScheme != TransportScheme.NetTcp)
{
throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp.");
}
if (brokerInfo.Durable)
{
throw new NotSupportedException("Sample broker does not support durable session.");
}
this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance);
Frontend frontend = new Frontend(this.clientManager);
// Choose different binding configuration by start info
NetTcpBinding frontendBinding;
if (startInfo.Secure)
{
frontendBinding = new NetTcpBinding();
}
else
{
frontendBinding = new NetTcpBinding(SecurityMode.None);
}
frontendBinding.PortSharingEnabled = true;
this.frontendServiceHost = new ServiceHost(frontend, new Uri(String.Format("net.tcp://{0}:9091/SampleBroker", Environment.MachineName)));
string listenUri = this.frontendServiceHost.AddServiceEndpoint(typeof(IDuplexService), frontendBinding, String.Empty).ListenUri.AbsoluteUri;
this.frontendServiceHost.Open();
this.result = new BrokerInitializationResult();
this.result.BrokerEpr = new string[3] {
listenUri, null, null
};
this.result.ControllerEpr = new string[3];
this.result.ResponseEpr = new string[3];
}
/// <summary>
/// Create a broker appdomain
/// </summary>
/// <param name="recoverInfo">broker recover info</param>
/// <param name="sessionid">session id</param>
/// <param name="durable">indicate if the session is durable</param>
/// <param name="attached">indicate if it is attaching</param>
/// <returns>returns the initialization result</returns>
private async Task <BrokerInitializationResult> CreateBrokerAndRun(BrokerRecoverInfo recoverInfo, bool attached, ClusterInfoContract clusterInfo)
{
// Check the brokerDic to see if the session Id already exists
lock (this.brokerDic)
{
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
}
//TODO: SF: make sure the clusterInfo.NetworkTopology string can be converted to ClusterTopology enum
//ClusterTopology topo = ClusterTopology.Public;
// ClusterTopology topo;
// Enum.TryParse<ClusterTopology>(clusterInfo.NetworkTopology, out topo);
//get soa configurations
Dictionary <string, string> soaConfig = new Dictionary <string, string>();
List <string> keys = new List <string>()
{
Constant.RegistryPathEnv, Constant.AutomaticShrinkEnabled, Constant.NettcpOver443, Constant.NetworkPrefixEnv, Constant.EnableFqdnEnv
};
soaConfig = await this.schedulerHelper.GetSOAConfigurations(keys);
ServiceRegistrationRepo serviceRegistration = await this.GetRegistrationRepo(soaConfig[Constant.RegistryPathEnv]);
string serviceRegistrationPath = serviceRegistration.GetServiceRegistrationPath(recoverInfo.StartInfo.ServiceName, recoverInfo.StartInfo.ServiceVersion);
if (serviceRegistrationPath == null)
{
throw new FileNotFoundException("Registration file is not found", recoverInfo.StartInfo.ServiceName);
}
CustomBrokerRegistration customBroker = GetCustomBroker(serviceRegistrationPath);
// Build the broker start info
BrokerStartInfo brokerInfo = new BrokerStartInfo();
brokerInfo.SessionId = recoverInfo.SessionId;
#if HPCPACK
brokerInfo.JobOwnerSID = await this.schedulerHelper.GetJobOwnerSID(brokerInfo.SessionId);
#endif
brokerInfo.Durable = recoverInfo.Durable;
brokerInfo.Attached = attached;
//this is scheduler node or cluster connection string
brokerInfo.Headnode = this.headnode;
brokerInfo.PurgedFailed = recoverInfo.PurgedFailed;
brokerInfo.PurgedProcessed = recoverInfo.PurgedProcessed;
brokerInfo.PurgedTotal = recoverInfo.PurgedTotal;
brokerInfo.ConfigurationFile = serviceRegistrationPath;
brokerInfo.NetworkTopology = 0; // ClusterTopology.Public
brokerInfo.ClusterName = clusterInfo.ClusterName;
brokerInfo.ClusterId = clusterInfo.ClusterId;
brokerInfo.AzureStorageConnectionString = clusterInfo.AzureStorageConnectionString;
brokerInfo.Standalone = BrokerLauncherEnvironment.Standalone;
brokerInfo.UseAad = recoverInfo.StartInfo.UseAad;
brokerInfo.AadUserSid = recoverInfo.AadUserSid;
brokerInfo.AadUserName = recoverInfo.AadUserName;
if (soaConfig.TryGetValue(Constant.AutomaticShrinkEnabled, out var v))
{
brokerInfo.AutomaticShrinkEnabled = Convert.ToBoolean(v);
}
else
{
brokerInfo.AutomaticShrinkEnabled = false;
}
if (SoaHelper.IsOnAzure())
{
brokerInfo.EnableDiagTrace = true;
}
else
{
brokerInfo.EnableDiagTrace = SoaDiagTraceHelper.IsDiagTraceEnabled(recoverInfo.SessionId);
}
if (!SoaHelper.IsSchedulerOnAzure())
{
// default value is true
bool nettcpOver443 = true;
string value = soaConfig[Constant.NettcpOver443];
if (!string.IsNullOrEmpty(value))
{
if (!bool.TryParse(value, out nettcpOver443))
{
nettcpOver443 = true;
}
}
brokerInfo.HttpsBurst = !nettcpOver443;
}
if (SoaHelper.IsSchedulerOnAzure())
{
// do not need network prefix for the Azure nodes
brokerInfo.NetworkPrefix = string.Empty;
}
else
{
brokerInfo.NetworkPrefix = soaConfig[Constant.NetworkPrefixEnv];
}
// get enableFQDN setting from the cluster env var
bool enableFQDN = false;
string enableFqdnStr = soaConfig[Constant.EnableFqdnEnv];
if (!string.IsNullOrEmpty(enableFqdnStr))
{
if (bool.TryParse(enableFqdnStr, out enableFQDN))
{
brokerInfo.EnableFQDN = enableFQDN;
BrokerTracing.TraceVerbose(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting in cluster env var is {0}",
enableFQDN);
}
else
{
BrokerTracing.TraceError(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting \"{0}\" in cluster env var is not a valid bool value.",
enableFqdnStr);
}
}
// set persist version.
if (!brokerInfo.Attached)
{
//if creating a new session, set persist version to BrokerVersion.PersistVersion
brokerInfo.PersistVersion = BrokerVersion.PersistVersion;
}
else
{
//if attaching an existing session, get PersistVersion from recoverInfo
if (recoverInfo.PersistVersion.HasValue)
{
brokerInfo.PersistVersion = recoverInfo.PersistVersion.Value;
}
else
{
// if recover info doesn't have PersistVersion info, default to DefaultPersistVersion
brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
}
// if version is not supported, throw UnsupportedVersion exception
if (!BrokerVersion.IsSupportedPersistVersion(brokerInfo.PersistVersion))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_UnsupportedVersion, SR.UnsupportedVersion, brokerInfo.PersistVersion.ToString(), BrokerVersion.PersistVersion.ToString());
}
}
BrokerAuthorization auth = null;
if (recoverInfo.StartInfo.Secure)
{
if (recoverInfo.StartInfo.ShareSession)
{
#if HPCPACK
brokerInfo.JobTemplateACL = await this.schedulerHelper.GetJobTemplateACL(recoverInfo.StartInfo.JobTemplate);
auth = new BrokerAuthorization(brokerInfo.JobTemplateACL, (int)JobTemplateRights.SubmitJob, (int)JobTemplateRights.Generic_Read, (int)JobTemplateRights.Generic_Write, (int)JobTemplateRights.Generic_Execute, (int)JobTemplateRights.Generic_All);
#endif
// TODO: support share session
throw new NotImplementedException();
}
else
{
auth = new BrokerAuthorization(new SecurityIdentifier(brokerInfo.JobOwnerSID));
}
}
BrokerInfo info = new BrokerInfo(recoverInfo, brokerInfo, auth, customBroker, this.pool);
try
{
info.BrokerExited += new EventHandler(this.BrokerInfo_BrokerExited); // if the broker exit quickly due to short timeouts, the broker info could remain in the brokerDic, because it is added later.
info.StartBroker();
lock (this.brokerDic)
{
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
this.brokerDic.Add(recoverInfo.SessionId, info);
}
// Update broker info into job property
await this.schedulerHelper.UpdateBrokerInfo(info);
}
catch (Exception e)
{
// Some exception happens during the call, do some clean up
TraceHelper.TraceEvent(recoverInfo.SessionId, System.Diagnostics.TraceEventType.Error, "[BrokerManager] CreateBrokerDomainAndRun: Failed : {0}\nRevert change...", e);
// Bug 5378: If the broker is raised because of attaching (failover), revert it to suspend but not finished state
RevertCreateDomainAndRun(info, attached);
throw;
}
return(info.InitializationResult);
}
/// <summary>
/// Load the configuration from the configuration file
/// If no configuration is found or some part is missing, we will fill the blank with default value
/// </summary>
/// <param name="brokerSettings">indicate the broker settings from the session start info, this settings will override the settings load from the configuration file</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="brokerConfig">out the broker configurations</param>
/// <param name="serviceConfig">out the service configurations</param>
/// <param name="bindings">output the bindings</param>
public static void LoadConfiguration(SessionStartInfoContract brokerSettings, BrokerStartInfo brokerInfo, out BrokerConfigurations brokerConfig, out ServiceConfiguration serviceConfig, out BindingsSection bindings)
{
// Init config file
string filename = brokerInfo.ConfigurationFile;
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 1: Load configuration file name: {0}", filename);
brokerConfig = null;
serviceConfig = null;
bindings = null;
try
{
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException)
.GetAwaiter()
.GetResult();
brokerConfig = BrokerConfigurations.GetSectionGroup(config);
serviceConfig = ServiceRegistration.GetSectionGroup(config).Service;
bindings = (BindingsSection)config.GetSection(BindingsSectionName);
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid,
"{0}",
e.ToString());
}
if (brokerConfig == null)
{
// Set the default value
brokerConfig = defaultBrokerConfiguration;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the broker config from the configuration file, use the default configuration");
}
else
{
// Set the default configuration if the very section is not found
if (brokerConfig.Monitor == null)
{
brokerConfig.Monitor = defaultBrokerConfiguration.Monitor;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the monitor config from the configuration file, use the default configuration");
}
if (brokerConfig.Services == null)
{
brokerConfig.Services = defaultBrokerConfiguration.Services;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the services config from the configuration file, use the default configuration");
}
if (brokerConfig.LoadBalancing == null)
{
brokerConfig.LoadBalancing = defaultBrokerConfiguration.LoadBalancing;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the load balancing config from the configuration file, use the default configuration");
}
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 2: Load broker config and service config succeeded.");
if (brokerConfig.Monitor.AllocationAdjustInterval < MinAllocationAdjustTime && brokerConfig.Monitor.AllocationAdjustInterval != System.Threading.Timeout.Infinite)
{
brokerConfig.Monitor.AllocationAdjustInterval = MinAllocationAdjustTime;
}
// Update the broker config using the session start info
if (brokerSettings.ClientIdleTimeout.HasValue)
{
brokerConfig.Monitor.ClientIdleTimeout = brokerSettings.ClientIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientIdleTimeout to {0}", brokerConfig.Monitor.ClientIdleTimeout);
}
if (brokerSettings.ClientConnectionTimeout.HasValue)
{
brokerConfig.Monitor.ClientConnectionTimeout = brokerSettings.ClientConnectionTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientConnectionTimeout to {0}", brokerConfig.Monitor.ClientConnectionTimeout);
}
if (brokerSettings.SessionIdleTimeout.HasValue)
{
brokerConfig.Monitor.SessionIdleTimeout = brokerSettings.SessionIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default SessionIdleTimeout to {0}", brokerConfig.Monitor.SessionIdleTimeout);
}
if (brokerSettings.MessagesThrottleStartThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStartThreshold = brokerSettings.MessagesThrottleStartThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStartThreshold to {0}", brokerConfig.Monitor.MessageThrottleStartThreshold);
}
if (brokerSettings.MessagesThrottleStopThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStopThreshold = brokerSettings.MessagesThrottleStopThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStopThreshold to {0}", brokerConfig.Monitor.MessageThrottleStopThreshold);
}
if (brokerSettings.ClientBrokerHeartbeatRetryCount.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount = brokerSettings.ClientBrokerHeartbeatRetryCount.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatRetryCount to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount);
}
if (brokerSettings.ClientBrokerHeartbeatInterval.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatInterval = brokerSettings.ClientBrokerHeartbeatInterval.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatInterval to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatInterval);
}
if (brokerSettings.ServiceOperationTimeout.HasValue)
{
brokerConfig.LoadBalancing.ServiceOperationTimeout = brokerSettings.ServiceOperationTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ServiceOperationTimeout to {0}", brokerConfig.LoadBalancing.ServiceOperationTimeout);
}
if (brokerSettings.DispatcherCapacityInGrowShrink.HasValue)
{
brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink = brokerSettings.DispatcherCapacityInGrowShrink.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default DispatcherCapacityInGrowShrink to {0}", brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink);
}
if (brokerSettings.MaxMessageSize.HasValue)
{
serviceConfig.MaxMessageSize = brokerSettings.MaxMessageSize.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MaxMessageSize to {0}", serviceConfig.MaxMessageSize);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 3: Override broker settings using session start info succeeded.");
// Validate the config section
string configError;
bool validateSucceeded;
try
{
validateSucceeded = brokerConfig.Validate(out configError);
}
catch (ConfigurationErrorsException e)
{
validateSucceeded = false;
configError = e.Message;
}
if (!validateSucceeded)
{
BrokerTracing.TraceEvent(TraceEventType.Error, 0, "[ConfigurationHelper] Invalid broker configuration section. Error {0}", configError);
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_InvalidConfiguration, configError);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 4: Validate broker configuration succeeded.");
StringBuilder sb = new StringBuilder();
sb.AppendLine("[Monitor]");
BrokerTracing.WriteProperties(sb, brokerConfig.Monitor, 3, typeof(int), typeof(string));
sb.AppendLine("[BaseAddress]");
sb.AppendFormat(" Http = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("http"));
sb.AppendFormat(" Https = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("https"));
sb.AppendFormat(" NetTcp = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("net.tcp"));
sb.AppendLine("[LoadBalancing]");
BrokerTracing.WriteProperties(sb, brokerConfig.LoadBalancing, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] BrokerConfiguration: \n{0}", sb.ToString());
sb = new StringBuilder();
sb.AppendLine("[Service]");
BrokerTracing.WriteProperties(sb, serviceConfig, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] ServiceConfiguration: \n{0}", sb.ToString());
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <returns>returns broker initialization result</returns>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
ThrowIfNull(startInfo, "startInfo");
ThrowIfNull(brokerInfo, "brokerInfo");
this.entry = new BrokerEntry(startInfo, brokerInfo, this);
return(this.entry.Result);
}
/// <summary>
/// Build broker start information
/// </summary>
/// <param name="serviceName">indicating service name</param>
/// <param name="serviceVersion">indicating service version</param>
/// <param name="sessionId">indicating session id</param>
/// <param name="durable">indicating whether the session is durable</param>
/// <param name="attached">indicating whether the session is raised up by attaching</param>
private async Task BuildBrokerStartInfo(string serviceName, Version serviceVersion, string sessionId, bool durable, bool attached)
{
this.brokerInfo = new BrokerStartInfo();
this.brokerInfo.Headnode = this.startInfo.Headnode;
// Secure will be set to false and the following two settings are ignored
this.brokerInfo.JobOwnerSID = null;
this.brokerInfo.JobTemplateACL = null;
this.brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
this.brokerInfo.SessionId = sessionId;
this.brokerInfo.Attached = attached;
this.brokerInfo.Durable = durable;
this.brokerInfo.NetworkPrefix = Constant.EnterpriseNetwork;
this.brokerInfo.ConfigurationFile = await this.FetchServiceRegistrationPath(serviceName, serviceVersion).ConfigureAwait(false);
// Bug 14892: Fetch AutoShrinkEnabled property from scheduler (via session launcher)
if (!this.isDebugModeEnabled)
{
SessionLauncherClient client = new SessionLauncherClient(await Utility.GetSessionLauncherAsync(this.startInfo, this.binding).ConfigureAwait(false), this.binding, this.startInfo.IsAadOrLocalUser);
try
{
this.brokerInfo.AutomaticShrinkEnabled = await RetryHelper <bool> .InvokeOperationAsync(
async() => Convert.ToBoolean(await client.GetSOAConfigurationAsync(Constant.AutomaticShrinkEnabled).ConfigureAwait(false)),
async (e, count) =>
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[InprocBrokerAdapter] Failed to get AutomaticShrinkEnabled property via session launcher service: {0}\nRetryCount = {1}", e, count);
Utility.SafeCloseCommunicateObject(client);
client = new SessionLauncherClient(await Utility.GetSessionLauncherAsync(this.startInfo, this.binding).ConfigureAwait(false), this.binding, this.startInfo.IsAadOrLocalUser);
},
SoaHelper.GetDefaultExponentialRetryManager()).ConfigureAwait(false);
}
catch (Exception e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[InprocBrokerAdapter] Failed to get AutomaticShrinkEnabled property via session launcher service: {0}", e);
}
finally
{
Utility.SafeCloseCommunicateObject(client);
}
this.brokerInfo.EnableDiagTrace = false; // TODO: retrieve this from session
// SchedulerAdapterInternalClient schedulerAdapterClient = new SchedulerAdapterInternalClient(await this.startInfo.ResolveHeadnodeMachineAsync().ConfigureAwait(false));
/*
* ISchedulerAdapter schedulerAdapterClient = SessionServiceContainer.SchedulerAdapterInstance;
* try
* {
* this.brokerInfo.EnableDiagTrace = await RetryHelper<bool>.InvokeOperationAsync(
* async () => await
#if net40
* TaskEx.Run(
#else
* Task.Run(
#endif
* () => schedulerAdapterClient.IsDiagTraceEnabled(sessionId)).ConfigureAwait(false),
* async (e, count) =>
* {
* SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[InprocBrokerAdapter] Failed to get IsDiagTraceEnabled property via session launcher service: {0}\nRetryCount = {1}", e, count);
* var communicateObj = schedulerAdapterClient as ICommunicationObject;
* if (communicateObj != null)
* {
* Utility.SafeCloseCommunicateObject(communicateObj);
* }
*
* //schedulerAdapterClient = new SchedulerAdapterInternalClient(await this.startInfo.ResolveHeadnodeMachineAsync().ConfigureAwait(false));
* schedulerAdapterClient = SessionServiceContainer.SchedulerAdapterInstance;
* },
* SoaHelper.GetDefaultExponentialRetryManager()).ConfigureAwait(false);
* }
* catch (Exception e)
* {
* SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[InprocBrokerAdapter] Failed to get IsDiagTraceEnabled property via session launcher service: {0}", e);
* }
* finally
* {
* var communicateObj = schedulerAdapterClient as ICommunicationObject;
* if (communicateObj != null)
* {
* Utility.SafeCloseCommunicateObject(communicateObj);
* }
*
* }
*/
}
}
/// <summary>
/// Run the broker
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="brokerInfo">indicate the broker start info</param>
/// <returns>initialization result</returns>
public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId);
try
{
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology);
// Step 1: Initialize configuration and shared data
ServiceConfiguration serviceConfig;
BrokerConfigurations brokerConfig;
BindingsSection bindings;
SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed.
ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings);
this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded.");
Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}");
// Step 2: Initialize broker queue
ClientInfo[] clientInfo;
this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded.");
// Step 3: Initialize observer
this.observer = new BrokerObserver(this.sharedData, clientInfo);
this.sharedData.Observer = this.observer;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded.");
// Step 4: Initialize state manager
this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0);
this.stateManager.UnloadBroker += this.UnloadBroker;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded.");
// Step 5: Initialize service job monitor
var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode);
if (SoaCommonConfig.WithoutSessionLayer)
{
this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
else
{
this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded.");
// Step 6: Initalize broker authorization
this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded.");
// Step 7: Initialize dispatcher manager
DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded.");
// Step 8: Start service job monitor
this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded.");
// Step 9: Initailize client manager
this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded.");
// if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist
string[] requestQueueUris = { };
string requestBlobUri = string.Empty;
string controllerRequestQueueUri = string.Empty;
string controllerResponseQueueUri = string.Empty;
if (startInfo.UseAzureStorage)
{
int clusterHash = 0;
if (!string.IsNullOrEmpty(brokerInfo.ClusterId))
{
string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant();
clusterHash = clusterIdString.GetHashCode();
}
else if (!string.IsNullOrEmpty(brokerInfo.ClusterName))
{
string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant();
clusterHash = clusterNameString.GetHashCode();
}
else
{
throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined.");
}
if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString))
{
this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString);
requestQueueUris = this.azureQueueProxy.RequestQueueUris;
requestBlobUri = this.azureQueueProxy.RequestBlobUri;
var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId);
var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId);
controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
requestQName,
CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult();
controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
responseQName,
CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult();
if (this.SessionId == SessionStartInfo.StandaloneSessionId)
{
CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName });
}
}
else
{
BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set.");
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable);
}
}
// Step 10: Initialize frontend
this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy);
////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize;
////this.readerQuotas = this.frontendResult.ReaderQuotas;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded.");
// Step 11: Start frontend, Initialization finished after this step
this.OpenFrontend();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded.");
// Step 12: Build initialization result and retrun to client
BrokerInitializationResult result = BrokerEntry.BuildInitializationResult(
this.frontendResult,
dispatcherManager,
this.sharedData.Config.LoadBalancing.ServiceOperationTimeout,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount,
requestQueueUris,
requestBlobUri,
controllerRequestQueueUri,
controllerResponseQueueUri,
startInfo.UseAzureStorage);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded.");
BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded.");
return(result);
}
catch (Exception ex)
{
BrokerTracing.TraceError(ex.ToString());
throw;
}
finally
{
if (this.sharedData != null)
{
this.sharedData.InitializationFinished();
}
}
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <returns>returns broker initialization result</returns>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNull(brokerInfo, "brokerInfo");
// Concurrent mode is set to single so no need to lock here
if (this.entry != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AlreadyInitialized, SR.AlreadyInitialized);
}
// Notice: We get the session Id here and expect to change the
// cosmos log file name to include session id. But cosmos log has a
// limitation that it is a static class without close method, and
// it can't be initialized twice. HpcTraceListener is just a wrapper
// for that static class. So creating a new HpcTraceListener can not
// workaround this issue.
TraceHelper.TraceInfo(
brokerInfo.SessionId,
"[BrokerManagementService].Initialize: Broker worker initializes for session {0}",
brokerInfo.SessionId);
// Set the configuration indicating enable/disable diag trace
SoaDiagTraceHelper.SetDiagTraceEnabledFlag(brokerInfo.SessionId, brokerInfo.EnableDiagTrace);
TraceHelper.IsDiagTraceEnabled = SoaDiagTraceHelper.IsDiagTraceEnabled;
Environment.SetEnvironmentVariable(SessionIdEnv, brokerInfo.SessionId.ToString(), EnvironmentVariableTarget.Process);
#if HPCPACK
// create the session id mapping file
foreach (TraceListener listener in TraceHelper.RuntimeTrace.CosmosTrace.Listeners)
{
if (listener is HpcTraceListener)
{
HpcTraceListener hpcListener = listener as HpcTraceListener;
if (hpcListener.IsPerSessionLogging)
{
string logIdFileName = string.Format("{0}_{1}", HpcTraceListener.LogFileBaseName, brokerInfo.SessionId);
string logIdFilePath = Path.Combine(HpcTraceListener.LogDir, logIdFileName);
try
{
using (FileStream file = File.Open(logIdFilePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.None))
{
}
}
catch (Exception e)
{
TraceHelper.TraceError(brokerInfo.SessionId,
"[BrokerManagementService].Initialize: Create log session Id match file failed with Exception {0}",
e.ToString());
}
}
}
}
#endif
this.entry = new BrokerEntry(brokerInfo.SessionId);
this.entry.BrokerFinished += new EventHandler(this.Entry_BrokerFinished);
return(this.entry.Run(startInfo, brokerInfo));
}
/// <summary>
/// Async version to ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="clusterEnvs">indicating the cluster envs</param>
/// <param name="callback">indicating the callback</param>
/// <param name="state">indicating the state</param>
public IAsyncResult BeginInitialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, AsyncCallback callback, object state)
{
return(this.Channel.BeginInitialize(startInfo, brokerInfo, callback, state));
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
// Call async version and block on completion in order to workaround System.Net.Socket bug #750028
IAsyncResult result = this.Channel.BeginInitialize(startInfo, brokerInfo, null, null);
return(this.Channel.EndInitialize(result));
}
