/// <summary>
/// Get the service info from registry (x64 boot) or info file (x86 boot)
/// </summary>
static string GetServiceInfo(string serviceConfigFileName)
{
string serviceConfigFile = string.Empty;
try
{
//TODO get headnode
string headnode;
if (ServiceHostRuntimeConfiguration.Standalone)
{
headnode = System.Net.Dns.GetHostName();
}
else
{
headnode = EndpointsConnectionString.LoadFromEnvVarsOrWindowsRegistry().ConnectionString;
}
Debug.WriteLine($"[{nameof(GetServiceInfo)}](Debug) headnode: {headnode}.");
Debug.Assert(!string.IsNullOrEmpty(headnode), "Head node connection string is null or empty.");
ServiceRegistrationRepo serviceRegistration = new ServiceRegistrationRepo(Environment.GetEnvironmentVariable(Constant.RegistryPathEnv), null);
// Get the path to the service config file name
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(serviceConfigFileName);
if (serviceConfigFile == null)
{
// Make a part for the error message
string CentrialPath = serviceRegistration.CentrialPath;
StringBuilder serviceRegDirsBuilder = new StringBuilder();
if (!string.IsNullOrEmpty(CentrialPath))
{
serviceRegDirsBuilder.Append("\n\t");
serviceRegDirsBuilder.Append(CentrialPath);
}
serviceRegDirsBuilder.Append("\n");
Console.Error.WriteLine(
string.Format(
CultureInfo.CurrentCulture,
StringTable.CantFindServiceRegistrationFileUnderFolders,
serviceRegDirsBuilder.ToString()));
}
}
catch (Exception e)
{
Console.Error.WriteLine(
string.Format(CultureInfo.CurrentCulture, StringTable.ExceptionInReadingRegistrationFile, serviceConfigFile, e.ToString()));
}
return(serviceConfigFile);
}
protected virtual ServiceRegistrationRepo GetRegistrationRepo(string callId)
{
ServiceRegistrationRepo repo = null;
string regPath = string.Empty;
try
{
if (!string.IsNullOrEmpty(SessionLauncherSettings.Default.ServiceRegistrationPath))
{
regPath = SessionLauncherSettings.Default.ServiceRegistrationPath + ";" + regPath;
}
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Error, "[SessionLauncher] .GetRegistrationRepo: callId={0}, Get the scheduler environment failed. exception = {1}", callId, e);
ThrowHelper.ThrowSessionFault(SOAFaultCode.GetClusterPropertyFailure, SR.SessionLauncher_FailToGetClusterProperty, e.ToString());
}
if (!string.IsNullOrEmpty(regPath))
{
repo = this.CreateServiceRegistrationRepo(regPath);
}
else
{
TraceHelper.TraceEvent(TraceEventType.Error, "[SessionLauncher] .GetRegistrationRepo: callId={0}, Get the scheduler environment for RegistryPathEnv is empty or null.", callId);
}
if (repo != null && repo.GetServiceRegistrationDirectories() != null)
{
return(repo);
}
else
{
TraceHelper.TraceEvent(TraceEventType.Error, "[SessionLauncher] .GetRegistrationRepo: No service registration directories are configured");
ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_RegistrationDirsMissing, SR.SessionLauncher_NoServiceRegistrationDirs);
return(null);
}
}
/// <summary>
/// Allocate a new durable or non-durable session
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="durable">whether session should be durable</param>
/// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param>
/// <returns>the Broker Launcher EPRs, sorted by the preference.</returns>
protected virtual async Task <SessionAllocateInfoContract> AllocateInternalAsync(SessionStartInfoContract startInfo, string endpointPrefix, bool durable)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] Begin: AllocateInternalAsync");
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNullOrEmpty(startInfo.ServiceName, "startInfo.ServiceName");
ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix");
#if HPCPACK
// check client api version, 4.3 or older client is not supported by 4.4 server for the broken changes
if (startInfo.ClientVersion == null || startInfo.ClientVersion < new Version(4, 4))
{
TraceHelper.TraceEvent(TraceEventType.Error,
"[SessionLauncher] .AllocateInternalAsync: ClientVersion {0} does not match ServerVersion {1}.", startInfo.ClientVersion, ServerVersion);
ThrowHelper.ThrowSessionFault(SOAFaultCode.ClientServerVersionMismatch,
SR.SessionLauncher_ClientServerVersionMismatch,
startInfo.ClientVersion == null ? "NULL" : startInfo.ClientVersion.ToString(),
ServerVersion.ToString());
}
#endif
// Init service version to the service version passed in
if (startInfo.ServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = startInfo.ServiceVersion;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is {0}", sessionAllocateInfo.ServiceVersion);
}
else
{
sessionAllocateInfo.ServiceVersion = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is null.");
}
string callId = Guid.NewGuid().ToString();
this.CheckAccess();
SecureString securePassword = CreateSecureString(startInfo.Password);
startInfo.Password = null;
// BUG 4522 : Use CCP_SCHEDULER when referencing service registration file share so HA HN virtual name is used when needed
//var reliableRegistry = new ReliableRegistry(this.fabricClient.PropertyManager);
//string defaultServiceRegistrationServerName = await reliableRegistry.GetValueAsync<string>(HpcConstants.HpcFullKeyName, HpcConstants.FileShareServerRegVal, this.token);
//if (String.IsNullOrEmpty(defaultServiceRegistrationServerName))
//{
// defaultServiceRegistrationServerName = "localhost";
//}
// the reg repo path is from scheduler environments, defaultServiceRegistrationServerName is actually not used
string serviceConfigFile;
ServiceRegistrationRepo serviceRegistration = this.GetRegistrationRepo(callId);
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, startInfo.ServiceVersion);
// If the serviceConfigFile wasnt found and serviceversion isnt specified, try getitng the service config based on the service's latest version
if (string.IsNullOrEmpty(serviceConfigFile) && (startInfo.ServiceVersion == null))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Try to find out versioned service.");
// Get service version in ServiceRegistrationRepo
Version dynamicServiceVersion = serviceRegistration.GetServiceVersionInternal(startInfo.ServiceName, false);
if (dynamicServiceVersion != null)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Selected dynamicServiceVersion is {0}.", dynamicServiceVersion.ToString());
}
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, dynamicServiceVersion);
// If a config file is found, update the serviceVersion that is returned to client and stored in recovery info
if (!string.IsNullOrEmpty(serviceConfigFile))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: serviceConfigFile is {0}.", serviceConfigFile);
startInfo.ServiceVersion = dynamicServiceVersion;
if (dynamicServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = dynamicServiceVersion;
}
}
}
string serviceName = ServiceRegistrationRepo.GetServiceRegistrationFileName(startInfo.ServiceName, startInfo.ServiceVersion);
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Service name = {0}, Configuration file = {1}", serviceName, serviceConfigFile);
// If the service is not found and user code doesn't specify
// version, we will use the latest version.
if (string.IsNullOrEmpty(serviceConfigFile))
{
if (startInfo.ServiceVersion != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ServiceVersion_NotFound, SR.SessionLauncher_ServiceVersionNotFound, startInfo.ServiceName, startInfo.ServiceVersion.ToString());
}
else
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_NotFound, SR.SessionLauncher_ServiceNotFound, startInfo.ServiceName);
}
}
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = serviceConfigFile;
ServiceRegistration registration = null;
BrokerConfigurations brokerConfigurations = null;
string hostpath = null;
string traceSwitchValue = null;
try
{
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException).GetAwaiter().GetResult();
Debug.Assert(config != null, "Configuration is not opened properly.");
registration = ServiceRegistration.GetSectionGroup(config);
brokerConfigurations = BrokerConfigurations.GetSectionGroup(config);
if (registration != null && registration.Host != null && registration.Host.Path != null)
{
hostpath = registration.Host.Path;
}
else
{
// x86 or x64
hostpath = registration.Service.Architecture == ServiceArch.X86 ? TaskCommandLine32 : TaskCommandLine64;
}
traceSwitchValue = registration.Service.SoaDiagTraceLevel;
// TODO: should deprecate the previous settings
if (string.IsNullOrEmpty(traceSwitchValue))
{
traceSwitchValue = ConfigurationHelper.GetTraceSwitchValue(config);
}
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, SR.SessionLauncher_ConfigFileInvalid, e.ToString());
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, ex.ToString());
throw;
}
// after figuring out the service and version, and the session pool size, we check if the service pool already has the instance.
sessionAllocateInfo.Id = "0";
sessionAllocateInfo.SessionInfo = null;
if (startInfo.UseSessionPool)
{
if (this.TryGetSessionAllocateInfoFromPooled(endpointPrefix, durable, sessionAllocateInfo, serviceConfigFile, registration, out var allocateInternal))
{
return(allocateInternal);
}
}
// for sessions to add in session pool
try
{
var sessionAllocateInfoContract = await this.CreateAndSubmitSessionJob(
startInfo,
endpointPrefix,
durable,
callId,
securePassword,
registration,
sessionAllocateInfo,
traceSwitchValue,
serviceName,
brokerConfigurations,
hostpath);
if (sessionAllocateInfoContract != null)
{
return(sessionAllocateInfoContract);
}
}
finally
{
// Add the submitted job to the session pool.
if (startInfo.UseSessionPool)
{
this.AddSessionToPool(Path.GetFileNameWithoutExtension(serviceConfigFile), durable, sessionAllocateInfo.Id, registration.Service.MaxSessionPoolSize);
}
}
return(null);
}
/// <summary>
/// Create a broker appdomain
/// </summary>
/// <param name="recoverInfo">broker recover info</param>
/// <param name="sessionid">session id</param>
/// <param name="durable">indicate if the session is durable</param>
/// <param name="attached">indicate if it is attaching</param>
/// <returns>returns the initialization result</returns>
private async Task <BrokerInitializationResult> CreateBrokerAndRun(BrokerRecoverInfo recoverInfo, bool attached, ClusterInfoContract clusterInfo)
{
// Check the brokerDic to see if the session Id already exists
lock (this.brokerDic)
{
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
}
//TODO: SF: make sure the clusterInfo.NetworkTopology string can be converted to ClusterTopology enum
//ClusterTopology topo = ClusterTopology.Public;
// ClusterTopology topo;
// Enum.TryParse<ClusterTopology>(clusterInfo.NetworkTopology, out topo);
//get soa configurations
Dictionary <string, string> soaConfig = new Dictionary <string, string>();
List <string> keys = new List <string>()
{
Constant.RegistryPathEnv, Constant.AutomaticShrinkEnabled, Constant.NettcpOver443, Constant.NetworkPrefixEnv, Constant.EnableFqdnEnv
};
soaConfig = await this.schedulerHelper.GetSOAConfigurations(keys);
ServiceRegistrationRepo serviceRegistration = await this.GetRegistrationRepo(soaConfig[Constant.RegistryPathEnv]);
string serviceRegistrationPath = serviceRegistration.GetServiceRegistrationPath(recoverInfo.StartInfo.ServiceName, recoverInfo.StartInfo.ServiceVersion);
if (serviceRegistrationPath == null)
{
throw new FileNotFoundException("Registration file is not found", recoverInfo.StartInfo.ServiceName);
}
CustomBrokerRegistration customBroker = GetCustomBroker(serviceRegistrationPath);
// Build the broker start info
BrokerStartInfo brokerInfo = new BrokerStartInfo();
brokerInfo.SessionId = recoverInfo.SessionId;
#if HPCPACK
brokerInfo.JobOwnerSID = await this.schedulerHelper.GetJobOwnerSID(brokerInfo.SessionId);
#endif
brokerInfo.Durable = recoverInfo.Durable;
brokerInfo.Attached = attached;
//this is scheduler node or cluster connection string
brokerInfo.Headnode = this.headnode;
brokerInfo.PurgedFailed = recoverInfo.PurgedFailed;
brokerInfo.PurgedProcessed = recoverInfo.PurgedProcessed;
brokerInfo.PurgedTotal = recoverInfo.PurgedTotal;
brokerInfo.ConfigurationFile = serviceRegistrationPath;
brokerInfo.NetworkTopology = 0; // ClusterTopology.Public
brokerInfo.ClusterName = clusterInfo.ClusterName;
brokerInfo.ClusterId = clusterInfo.ClusterId;
brokerInfo.AzureStorageConnectionString = clusterInfo.AzureStorageConnectionString;
brokerInfo.Standalone = BrokerLauncherEnvironment.Standalone;
brokerInfo.UseAad = recoverInfo.StartInfo.UseAad;
brokerInfo.AadUserSid = recoverInfo.AadUserSid;
brokerInfo.AadUserName = recoverInfo.AadUserName;
if (soaConfig.TryGetValue(Constant.AutomaticShrinkEnabled, out var v))
{
brokerInfo.AutomaticShrinkEnabled = Convert.ToBoolean(v);
}
else
{
brokerInfo.AutomaticShrinkEnabled = false;
}
if (SoaHelper.IsOnAzure())
{
brokerInfo.EnableDiagTrace = true;
}
else
{
brokerInfo.EnableDiagTrace = SoaDiagTraceHelper.IsDiagTraceEnabled(recoverInfo.SessionId);
}
if (!SoaHelper.IsSchedulerOnAzure())
{
// default value is true
bool nettcpOver443 = true;
string value = soaConfig[Constant.NettcpOver443];
if (!string.IsNullOrEmpty(value))
{
if (!bool.TryParse(value, out nettcpOver443))
{
nettcpOver443 = true;
}
}
brokerInfo.HttpsBurst = !nettcpOver443;
}
if (SoaHelper.IsSchedulerOnAzure())
{
// do not need network prefix for the Azure nodes
brokerInfo.NetworkPrefix = string.Empty;
}
else
{
brokerInfo.NetworkPrefix = soaConfig[Constant.NetworkPrefixEnv];
}
// get enableFQDN setting from the cluster env var
bool enableFQDN = false;
string enableFqdnStr = soaConfig[Constant.EnableFqdnEnv];
if (!string.IsNullOrEmpty(enableFqdnStr))
{
if (bool.TryParse(enableFqdnStr, out enableFQDN))
{
brokerInfo.EnableFQDN = enableFQDN;
BrokerTracing.TraceVerbose(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting in cluster env var is {0}",
enableFQDN);
}
else
{
BrokerTracing.TraceError(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting \"{0}\" in cluster env var is not a valid bool value.",
enableFqdnStr);
}
}
// set persist version.
if (!brokerInfo.Attached)
{
//if creating a new session, set persist version to BrokerVersion.PersistVersion
brokerInfo.PersistVersion = BrokerVersion.PersistVersion;
}
else
{
//if attaching an existing session, get PersistVersion from recoverInfo
if (recoverInfo.PersistVersion.HasValue)
{
brokerInfo.PersistVersion = recoverInfo.PersistVersion.Value;
}
else
{
// if recover info doesn't have PersistVersion info, default to DefaultPersistVersion
brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
}
// if version is not supported, throw UnsupportedVersion exception
if (!BrokerVersion.IsSupportedPersistVersion(brokerInfo.PersistVersion))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_UnsupportedVersion, SR.UnsupportedVersion, brokerInfo.PersistVersion.ToString(), BrokerVersion.PersistVersion.ToString());
}
}
BrokerAuthorization auth = null;
if (recoverInfo.StartInfo.Secure)
{
if (recoverInfo.StartInfo.ShareSession)
{
#if HPCPACK
brokerInfo.JobTemplateACL = await this.schedulerHelper.GetJobTemplateACL(recoverInfo.StartInfo.JobTemplate);
auth = new BrokerAuthorization(brokerInfo.JobTemplateACL, (int)JobTemplateRights.SubmitJob, (int)JobTemplateRights.Generic_Read, (int)JobTemplateRights.Generic_Write, (int)JobTemplateRights.Generic_Execute, (int)JobTemplateRights.Generic_All);
#endif
// TODO: support share session
throw new NotImplementedException();
}
else
{
auth = new BrokerAuthorization(new SecurityIdentifier(brokerInfo.JobOwnerSID));
}
}
BrokerInfo info = new BrokerInfo(recoverInfo, brokerInfo, auth, customBroker, this.pool);
try
{
info.BrokerExited += new EventHandler(this.BrokerInfo_BrokerExited); // if the broker exit quickly due to short timeouts, the broker info could remain in the brokerDic, because it is added later.
info.StartBroker();
lock (this.brokerDic)
{
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
this.brokerDic.Add(recoverInfo.SessionId, info);
}
// Update broker info into job property
await this.schedulerHelper.UpdateBrokerInfo(info);
}
catch (Exception e)
{
// Some exception happens during the call, do some clean up
TraceHelper.TraceEvent(recoverInfo.SessionId, System.Diagnostics.TraceEventType.Error, "[BrokerManager] CreateBrokerDomainAndRun: Failed : {0}\nRevert change...", e);
// Bug 5378: If the broker is raised because of attaching (failover), revert it to suspend but not finished state
RevertCreateDomainAndRun(info, attached);
throw;
}
return(info.InitializationResult);
}
