/// <summary>
/// Revert the change done by CreateDomainAndRun
/// </summary>
/// <param name="info">indicate the broker info</param>
/// <param name="suspended">indicating whether revert to suspended or not</param>
private static void RevertCreateDomainAndRun(BrokerInfo info, bool suspended)
{
if (info == null)
{
return;
}
try
{
info.CloseBroker(suspended);
}
catch (Exception e)
{
TraceHelper.TraceEvent(info.SessionId, System.Diagnostics.TraceEventType.Error, "[BrokerManager] RevertCreateDomainAndRun: Failed to close the entry: {0}", e);
}
}
/// <summary>
/// Update the broker info
/// </summary>
/// <param name="info">broker info</param>
public async Task UpdateBrokerInfo(BrokerInfo info)
{
Dictionary <string, object> properties = new Dictionary <string, object>();
properties.Add("EndpointReference", string.Join(";", info.InitializationResult.BrokerEpr));
// use the broker role IP address for the broker node on Azure
properties.Add(BrokerSettingsConstants.BrokerNode, SoaHelper.IsOnAzure() ? AzureRoleHelper.GetLocalMachineAddress() : Environment.MachineName);
properties.Add(BrokerSettingsConstants.Suspended, info.Durable);
properties.Add(BrokerSettingsConstants.Durable, info.Durable);
properties.Add(BrokerSettingsConstants.PersistVersion, info.PersistVersion);
properties.Add(BrokerSettingsConstants.MessageDetailsAvailable, info.InitializationResult.SupportsMessageDetails);
await this.UpdateBrokerInfoInternalAsync(info.SessionId, properties);
}
public bool?IfSeesionCreatedByAadOrLocalUser(string sessionId)
{
BrokerInfo info = null;
lock (this.brokerDic)
{
this.brokerDic.TryGetValue(sessionId, out info);
}
if (info == null)
{
// Proberbly already disposed.
return(null);
}
else
{
return(info.IsAadOrLocalUser);
}
}
/// <summary>
/// Handle broker exit event
/// </summary>
/// <param name="sender">indicating the sender</param>
/// <param name="e">indicating the event args</param>
private void BrokerInfo_BrokerExited(object sender, EventArgs e)
{
BrokerInfo info = (BrokerInfo)sender;
int exitCode = info.GetExitCode();
TraceHelper.TraceEvent(info.SessionId, exitCode == 0 ? TraceEventType.Information : TraceEventType.Error, "[BrokerManager] Broker process exited, ExitCode = {0}", exitCode);
try
{
if (exitCode != 0 && info.RetryCount <= RecoverBrokerRetryLimit)
{
// ExitCode != 0 means the broker encountered an unexpected error, retry
if (info.Durable)
{
// Bug 8285: Only retry if it is a durable session
TraceHelper.TraceEvent(info.SessionId, TraceEventType.Information, "[BrokerManager] Retry to create broker.");
info.StartBroker();
return;
}
else
{
// Bug 8285: Fail service job for interactive session if broker worker dies
this.schedulerHelper.FailJob(info.SessionId, FailInteractiveServiceJobBecauseBrokerWorkerDied).GetAwaiter().GetResult();
}
}
}
catch (Exception ex)
{
TraceHelper.TraceError(info.SessionId, "[BrokerManager] Exception thrown when handling broker exited event: {0}", ex);
}
lock (this.brokerDic)
{
if (!this.brokerDic.Remove(info.SessionId))
{
TraceHelper.TraceError(info.SessionId,
"[BrokerManager] Failed to remove the session {0} from the brokerDic", info.SessionId);
}
}
SoaDiagTraceHelper.RemoveDiagTraceEnabledFlag(info.SessionId);
}
/// <summary>
/// Close the broker domain
/// </summary>
/// <param name="sessionId">indicating the session id</param>
public async Task CloseBrokerDomain(string sessionId)
{
TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerManager] Close broker {0}", sessionId);
BrokerInfo info = null;
lock (this.brokerDic)
{
this.brokerDic.TryGetValue(sessionId, out info);
}
if (info != null)
{
info.CheckAccess();
await this.CleanupAsync(sessionId, false);
}
else
{
TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerManager] Broker {0} didn't found. Maybe it has already been closed.", sessionId);
}
}
public async Task UpdateBrokerInfo(BrokerInfo info)
{
}
/// <summary>
/// Create a broker appdomain
/// </summary>
/// <param name="recoverInfo">broker recover info</param>
/// <param name="sessionid">session id</param>
/// <param name="durable">indicate if the session is durable</param>
/// <param name="attached">indicate if it is attaching</param>
/// <returns>returns the initialization result</returns>
private async Task <BrokerInitializationResult> CreateBrokerAndRun(BrokerRecoverInfo recoverInfo, bool attached, ClusterInfoContract clusterInfo)
{
// Check the brokerDic to see if the session Id already exists
lock (this.brokerDic)
{
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
}
//TODO: SF: make sure the clusterInfo.NetworkTopology string can be converted to ClusterTopology enum
//ClusterTopology topo = ClusterTopology.Public;
// ClusterTopology topo;
// Enum.TryParse<ClusterTopology>(clusterInfo.NetworkTopology, out topo);
//get soa configurations
Dictionary <string, string> soaConfig = new Dictionary <string, string>();
List <string> keys = new List <string>()
{
Constant.RegistryPathEnv, Constant.AutomaticShrinkEnabled, Constant.NettcpOver443, Constant.NetworkPrefixEnv, Constant.EnableFqdnEnv
};
soaConfig = await this.schedulerHelper.GetSOAConfigurations(keys);
ServiceRegistrationRepo serviceRegistration = await this.GetRegistrationRepo(soaConfig[Constant.RegistryPathEnv]);
string serviceRegistrationPath = serviceRegistration.GetServiceRegistrationPath(recoverInfo.StartInfo.ServiceName, recoverInfo.StartInfo.ServiceVersion);
if (serviceRegistrationPath == null)
{
throw new FileNotFoundException("Registration file is not found", recoverInfo.StartInfo.ServiceName);
}
CustomBrokerRegistration customBroker = GetCustomBroker(serviceRegistrationPath);
// Build the broker start info
BrokerStartInfo brokerInfo = new BrokerStartInfo();
brokerInfo.SessionId = recoverInfo.SessionId;
#if HPCPACK
brokerInfo.JobOwnerSID = await this.schedulerHelper.GetJobOwnerSID(brokerInfo.SessionId);
#endif
brokerInfo.Durable = recoverInfo.Durable;
brokerInfo.Attached = attached;
//this is scheduler node or cluster connection string
brokerInfo.Headnode = this.headnode;
brokerInfo.PurgedFailed = recoverInfo.PurgedFailed;
brokerInfo.PurgedProcessed = recoverInfo.PurgedProcessed;
brokerInfo.PurgedTotal = recoverInfo.PurgedTotal;
brokerInfo.ConfigurationFile = serviceRegistrationPath;
brokerInfo.NetworkTopology = 0; // ClusterTopology.Public
brokerInfo.ClusterName = clusterInfo.ClusterName;
brokerInfo.ClusterId = clusterInfo.ClusterId;
brokerInfo.AzureStorageConnectionString = clusterInfo.AzureStorageConnectionString;
brokerInfo.Standalone = BrokerLauncherEnvironment.Standalone;
brokerInfo.UseAad = recoverInfo.StartInfo.UseAad;
brokerInfo.AadUserSid = recoverInfo.AadUserSid;
brokerInfo.AadUserName = recoverInfo.AadUserName;
if (soaConfig.TryGetValue(Constant.AutomaticShrinkEnabled, out var v))
{
brokerInfo.AutomaticShrinkEnabled = Convert.ToBoolean(v);
}
else
{
brokerInfo.AutomaticShrinkEnabled = false;
}
if (SoaHelper.IsOnAzure())
{
brokerInfo.EnableDiagTrace = true;
}
else
{
brokerInfo.EnableDiagTrace = SoaDiagTraceHelper.IsDiagTraceEnabled(recoverInfo.SessionId);
}
if (!SoaHelper.IsSchedulerOnAzure())
{
// default value is true
bool nettcpOver443 = true;
string value = soaConfig[Constant.NettcpOver443];
if (!string.IsNullOrEmpty(value))
{
if (!bool.TryParse(value, out nettcpOver443))
{
nettcpOver443 = true;
}
}
brokerInfo.HttpsBurst = !nettcpOver443;
}
if (SoaHelper.IsSchedulerOnAzure())
{
// do not need network prefix for the Azure nodes
brokerInfo.NetworkPrefix = string.Empty;
}
else
{
brokerInfo.NetworkPrefix = soaConfig[Constant.NetworkPrefixEnv];
}
// get enableFQDN setting from the cluster env var
bool enableFQDN = false;
string enableFqdnStr = soaConfig[Constant.EnableFqdnEnv];
if (!string.IsNullOrEmpty(enableFqdnStr))
{
if (bool.TryParse(enableFqdnStr, out enableFQDN))
{
brokerInfo.EnableFQDN = enableFQDN;
BrokerTracing.TraceVerbose(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting in cluster env var is {0}",
enableFQDN);
}
else
{
BrokerTracing.TraceError(
"[BrokerManager].CreateBrokerAndRun: The enableFQDN setting \"{0}\" in cluster env var is not a valid bool value.",
enableFqdnStr);
}
}
// set persist version.
if (!brokerInfo.Attached)
{
//if creating a new session, set persist version to BrokerVersion.PersistVersion
brokerInfo.PersistVersion = BrokerVersion.PersistVersion;
}
else
{
//if attaching an existing session, get PersistVersion from recoverInfo
if (recoverInfo.PersistVersion.HasValue)
{
brokerInfo.PersistVersion = recoverInfo.PersistVersion.Value;
}
else
{
// if recover info doesn't have PersistVersion info, default to DefaultPersistVersion
brokerInfo.PersistVersion = BrokerVersion.DefaultPersistVersion;
}
// if version is not supported, throw UnsupportedVersion exception
if (!BrokerVersion.IsSupportedPersistVersion(brokerInfo.PersistVersion))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_UnsupportedVersion, SR.UnsupportedVersion, brokerInfo.PersistVersion.ToString(), BrokerVersion.PersistVersion.ToString());
}
}
BrokerAuthorization auth = null;
if (recoverInfo.StartInfo.Secure)
{
if (recoverInfo.StartInfo.ShareSession)
{
#if HPCPACK
brokerInfo.JobTemplateACL = await this.schedulerHelper.GetJobTemplateACL(recoverInfo.StartInfo.JobTemplate);
auth = new BrokerAuthorization(brokerInfo.JobTemplateACL, (int)JobTemplateRights.SubmitJob, (int)JobTemplateRights.Generic_Read, (int)JobTemplateRights.Generic_Write, (int)JobTemplateRights.Generic_Execute, (int)JobTemplateRights.Generic_All);
#endif
// TODO: support share session
throw new NotImplementedException();
}
else
{
auth = new BrokerAuthorization(new SecurityIdentifier(brokerInfo.JobOwnerSID));
}
}
BrokerInfo info = new BrokerInfo(recoverInfo, brokerInfo, auth, customBroker, this.pool);
try
{
info.BrokerExited += new EventHandler(this.BrokerInfo_BrokerExited); // if the broker exit quickly due to short timeouts, the broker info could remain in the brokerDic, because it is added later.
info.StartBroker();
lock (this.brokerDic)
{
if (BrokerLauncherSettings.Default.MaxConcurrentSession > 0 && this.brokerDic.Count >= BrokerLauncherSettings.Default.MaxConcurrentSession)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_TooManyBrokerRunning, SR.TooManyBrokerRunning, BrokerLauncherSettings.Default.MaxConcurrentSession.ToString());
}
if (this.brokerDic.ContainsKey(recoverInfo.SessionId))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_SessionIdAlreadyExists, SR.SessionIdAlreadyExists, recoverInfo.SessionId.ToString());
}
this.brokerDic.Add(recoverInfo.SessionId, info);
}
// Update broker info into job property
await this.schedulerHelper.UpdateBrokerInfo(info);
}
catch (Exception e)
{
// Some exception happens during the call, do some clean up
TraceHelper.TraceEvent(recoverInfo.SessionId, System.Diagnostics.TraceEventType.Error, "[BrokerManager] CreateBrokerDomainAndRun: Failed : {0}\nRevert change...", e);
// Bug 5378: If the broker is raised because of attaching (failover), revert it to suspend but not finished state
RevertCreateDomainAndRun(info, attached);
throw;
}
return(info.InitializationResult);
}