/// <summary>
/// Create custom broker process
/// </summary>
/// <param name="customBrokerRegistration">indicating the custom broker registration</param>
/// <returns>returns the broker process object</returns>
private static BrokerProcess CreateCustomBrokerProcess(CustomBrokerRegistration customBroker)
{
BrokerProcess process = new BrokerProcess(customBroker.Executive, customBroker.EnvironmentVariables);
process.Start();
process.WaitForReady();
return(process);
}
/// <summary>
/// Initializes a new instance of the BrokerInfo class from broker recover info
/// </summary>
/// <param name="recoverInfo">indicating the broker recover info</param>
/// <param name="brokerInfo">indicating the broker start info</param>
/// <param name="auth">indicating the broker auth</param>
/// <param name="customBroker">indicating the custom broker configuration</param>
/// <param name="pool">indicating the broker process pool</param>
public BrokerInfo(BrokerRecoverInfo recoverInfo, BrokerStartInfo brokerInfo, BrokerAuthorization auth, CustomBrokerRegistration customBroker, BrokerProcessPool pool)
{
this.callbackToCloseBroker = new ThreadHelper <IAsyncResult>(new AsyncCallback(this.OnCloseBroker)).CallbackRoot;
this.durable = recoverInfo.Durable;
this.brokerInfo = brokerInfo;
this.sessionId = recoverInfo.SessionId;
this.sessionStartInfo = recoverInfo.StartInfo;
this.auth = auth;
this.customBroker = customBroker;
this.pool = pool;
this.sessionStartInfo.IpAddress = BrokerLauncherSettings.Default.SvcHostList.Cast <string>().ToArray();
this.sessionStartInfo.RegPath = BrokerLauncherSettings.Default.CCP_SERVICEREGISTRATION_PATH;
}
/// <summary>
/// Create a broker appdomain
/// </summary>
/// <param name="recoverInfo">broker recover info</param>
/// <param name="sessionid">session id</param>
/// <param name="durable">indicate if the session is durable</param>
/// <param name="attached">indicate if it is attaching</param>
/// <returns>returns the initialization result</returns>
private async Task <BrokerInitializationResult> CreateBrokerAndRun(BrokerRecoverInfo recoverInfo, bool attached, ClusterInfoContract clusterInfo)
{
// Check the brokerDic to see if the session Id already exists
lock (this.brokerDic)
{
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
}
//TODO: SF: make sure the clusterInfo.NetworkTopology string can be converted to ClusterTopology enum
//ClusterTopology topo = ClusterTopology.Public;
// ClusterTopology topo;
// Enum.TryParse<ClusterTopology>(clusterInfo.NetworkTopology, out topo);
//get soa configurations
Dictionary <string, string> soaConfig = new Dictionary <string, string>();
List <string> keys = new List <string>()
{
Constant.RegistryPathEnv, Constant.AutomaticShrinkEnabled, Constant.NettcpOver443, Constant.NetworkPrefixEnv, Constant.EnableFqdnEnv
};
soaConfig = await this.schedulerHelper.GetSOAConfigurations(keys);
ServiceRegistrationRepo serviceRegistration = await this.GetRegistrationRepo(soaConfig[Constant.RegistryPathEnv]);
string serviceRegistrationPath = serviceRegistration.GetServiceRegistrationPath(recoverInfo.StartInfo.ServiceName, recoverInfo.StartInfo.ServiceVersion);
if (serviceRegistrationPath == null)
{
throw new FileNotFoundException("Registration file is not found", recoverInfo.StartInfo.ServiceName);
}
CustomBrokerRegistration customBroker = GetCustomBroker(serviceRegistrationPath);
// Build the broker start info
BrokerStartInfo brokerInfo = new BrokerStartInfo();
brokerInfo.SessionId = recoverInfo.SessionId;
#if HPCPACK
brokerInfo.JobOwnerSID = await this.schedulerHelper.GetJobOwnerSID(brokerInfo.SessionId);
#endif
brokerInfo.Durable = recoverInfo.Durable;
brokerInfo.Attached = attached;
//this is scheduler node or cluster connection string
brokerInfo.Headnode = this.headnode;
brokerInfo.PurgedFailed = recoverInfo.PurgedFailed;
brokerInfo.PurgedProcessed = recoverInfo.PurgedProcessed;
brokerInfo.PurgedTotal = recoverInfo.PurgedTotal;
brokerInfo.ConfigurationFile = serviceRegistrationPath;
brokerInfo.NetworkTopology = 0; // ClusterTopology.Public
brokerInfo.ClusterName = clusterInfo.ClusterName;
brokerInfo.ClusterId = clusterInfo.ClusterId;
brokerInfo.AzureStorageConnectionString = clusterInfo.AzureStorageConnectionString;
brokerInfo.Standalone = BrokerLauncherEnvironment.Standalone;
brokerInfo.UseAad = recoverInfo.StartInfo.UseAad;
brokerInfo.AadUserSid = recoverInfo.AadUserSid;
brokerInfo.AadUserName = recoverInfo.AadUserName;
if (soaConfig.TryGetValue(Constant.AutomaticShrinkEnabled, out var v))
{
brokerInfo.AutomaticShrinkEnabled = Convert.ToBoolean(v);
}
else
{
brokerInfo.AutomaticShrinkEnabled = false;
}
if (SoaHelper.IsOnAzure())
{
brokerInfo.EnableDiagTrace = true;
}
else
{
brokerInfo.EnableDiagTrace = SoaDiagTraceHelper.IsDiagTraceEnabled(recoverInfo.SessionId);
}
if (!SoaHelper.IsSchedulerOnAzure())
{
// default value is true
bool nettcpOver443 = true;
string value = soaConfig[Constant.NettcpOver443];
if (!string.IsNullOrEmpty(value))
{
if (!bool.TryParse(value, out nettcpOver443))
{
nettcpOver443 = true;
}
}
brokerInfo.HttpsBurst = !nettcpOver443;
}
if (SoaHelper.IsSchedulerOnAzure())
{
// do not need network prefix for the Azure nodes
brokerInfo.NetworkPrefix = string.Empty;
}
else
{
brokerInfo.NetworkPrefix = soaConfig[Constant.NetworkPrefixEnv];
}
// get enableFQDN setting from the cluster env var
bool enableFQDN = false;
string enableFqdnStr = soaConfig[Constant.EnableFqdnEnv];
if (!string.IsNullOrEmpty(enableFqdnStr))
{
if (bool.TryParse(enableFqdnStr, out enableFQDN))
{
brokerInfo.EnableFQDN = enableFQDN;
BrokerTracing.TraceVerbose(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting in cluster env var is {0}",
enableFQDN);
}
else
{
BrokerTracing.TraceError(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting \"{0}\" in cluster env var is not a valid bool value.",
enableFqdnStr);
}
}
// set persist version.
if (!brokerInfo.Attached)
{
//if creating a new session, set persist version to BrokerVersion.PersistVersion
brokerInfo.PersistVersion = BrokerVersion.PersistVersion;
}
else
{
//if attaching an existing session, get PersistVersion from recoverInfo
if (recoverInfo.PersistVersion.HasValue)
{
brokerInfo.PersistVersion = recoverInfo.PersistVersion.Value;
}
else
{
// if recover info doesn't have PersistVersion info, default to DefaultPersistVersion
brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
}
// if version is not supported, throw UnsupportedVersion exception
if (!BrokerVersion.IsSupportedPersistVersion(brokerInfo.PersistVersion))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_UnsupportedVersion, SR.UnsupportedVersion, brokerInfo.PersistVersion.ToString(), BrokerVersion.PersistVersion.ToString());
}
}
BrokerAuthorization auth = null;
if (recoverInfo.StartInfo.Secure)
{
if (recoverInfo.StartInfo.ShareSession)
{
#if HPCPACK
brokerInfo.JobTemplateACL = await this.schedulerHelper.GetJobTemplateACL(recoverInfo.StartInfo.JobTemplate);
auth = new BrokerAuthorization(brokerInfo.JobTemplateACL, (int)JobTemplateRights.SubmitJob, (int)JobTemplateRights.Generic_Read, (int)JobTemplateRights.Generic_Write, (int)JobTemplateRights.Generic_Execute, (int)JobTemplateRights.Generic_All);
#endif
// TODO: support share session
throw new NotImplementedException();
}
else
{
auth = new BrokerAuthorization(new SecurityIdentifier(brokerInfo.JobOwnerSID));
}
}
BrokerInfo info = new BrokerInfo(recoverInfo, brokerInfo, auth, customBroker, this.pool);
try
{
info.BrokerExited += new EventHandler(this.BrokerInfo_BrokerExited); // if the broker exit quickly due to short timeouts, the broker info could remain in the brokerDic, because it is added later.
info.StartBroker();
lock (this.brokerDic)
{
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
this.brokerDic.Add(recoverInfo.SessionId, info);
}
// Update broker info into job property
await this.schedulerHelper.UpdateBrokerInfo(info);
}
catch (Exception e)
{
// Some exception happens during the call, do some clean up
TraceHelper.TraceEvent(recoverInfo.SessionId, System.Diagnostics.TraceEventType.Error, "[BrokerManager] CreateBrokerDomainAndRun: Failed : {0}\nRevert change...", e);
// Bug 5378: If the broker is raised because of attaching (failover), revert it to suspend but not finished state
RevertCreateDomainAndRun(info, attached);
throw;
}
return(info.InitializationResult);
}
