/// <summary>
/// Initializes a new instance of the SharedData class
/// </summary>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="startInfo">indicating the start info</param>
/// <param name="config">indicating the broker configuration</param>
/// <param name="serviceCnfig">indicating the service configuration</param>
public SharedData(BrokerStartInfo brokerInfo, SessionStartInfoContract startInfo, BrokerConfigurations config, ServiceConfiguration serviceConfig)
{
this.brokerInfo = brokerInfo;
this.startInfo = startInfo;
this.config = config;
this.serviceConfig = serviceConfig;
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <returns>returns broker initialization result</returns>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNull(brokerInfo, "brokerInfo");
// Concurrent mode is set to single so no need to lock here
if (this.entry != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AlreadyInitialized, SR.AlreadyInitialized);
}
// Notice: We get the session Id here and expect to change the
// cosmos log file name to include session id. But cosmos log has a
// limitation that it is a static class without close method, and
// it can't be initialized twice. HpcTraceListener is just a wrapper
// for that static class. So creating a new HpcTraceListener can not
// workaround this issue.
TraceHelper.TraceInfo(
brokerInfo.SessionId,
"[BrokerManagementService].Initialize: Broker worker initializes for session {0}",
brokerInfo.SessionId);
// Set the configuration indicating enable/disable diag trace
SoaDiagTraceHelper.SetDiagTraceEnabledFlag(brokerInfo.SessionId, brokerInfo.EnableDiagTrace);
TraceHelper.IsDiagTraceEnabled = SoaDiagTraceHelper.IsDiagTraceEnabled;
Environment.SetEnvironmentVariable(SessionIdEnv, brokerInfo.SessionId.ToString(), EnvironmentVariableTarget.Process);
#if HPCPACK
// create the session id mapping file
foreach (TraceListener listener in TraceHelper.RuntimeTrace.CosmosTrace.Listeners)
{
if (listener is HpcTraceListener)
{
HpcTraceListener hpcListener = listener as HpcTraceListener;
if (hpcListener.IsPerSessionLogging)
{
string logIdFileName = string.Format("{0}_{1}", HpcTraceListener.LogFileBaseName, brokerInfo.SessionId);
string logIdFilePath = Path.Combine(HpcTraceListener.LogDir, logIdFileName);
try
{
using (FileStream file = File.Open(logIdFilePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.None))
{
}
}
catch (Exception e)
{
TraceHelper.TraceError(brokerInfo.SessionId,
"[BrokerManagementService].Initialize: Create log session Id match file failed with Exception {0}",
e.ToString());
}
}
}
}
#endif
this.entry = new BrokerEntry(brokerInfo.SessionId);
this.entry.BrokerFinished += new EventHandler(this.Entry_BrokerFinished);
return(this.entry.Run(startInfo, brokerInfo));
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
// Call async version and block on completion in order to workaround System.Net.Socket bug #750028
IAsyncResult result = this.Channel.BeginInitialize(startInfo, brokerInfo, null, null);
return(this.Channel.EndInitialize(result));
}
public void BuildFrontEndTest_NetTcp()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
BrokerStartInfo startInfo = new BrokerStartInfo();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = true;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/Controller"));
Assert.IsTrue(Regex.IsMatch(result.GetResponseUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/GetResponse"));
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp"));
}
/// <summary>
/// Create a session with the specific sessionid
/// </summary>
/// <param name="info">Session Start Info</param>
/// <param name="sessionid">the session id</param>
/// <returns>the session info</returns>
public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionid)
{
// Call async version and block on completion in order to workaround System.Net.Socket bug #750028
IAsyncResult result = this.Channel.BeginCreate(info, sessionid, null, null);
return(this.Channel.EndCreate(result));
}
public void BuildFrontEndTest_Http()
{
SessionStartInfoContract info = new SessionStartInfoContract();
info.ServiceName = "CcpEchoSvc";
info.TransportScheme = TransportScheme.Http;
info.Secure = true;
BrokerStartInfo startInfo = new BrokerStartInfo();
MockBrokerAuthorization auth = new MockBrokerAuthorization();
string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None);
BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config);
startInfo.Durable = false;
startInfo.SessionId = "100";
startInfo.ConfigurationFile = "CcpEchoSvc.config";
ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings);
var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig);
var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null);
Assert.IsTrue(Regex.IsMatch(result.ControllerUriList[1], @"https://.+/100/Http/Controller"));
Assert.AreEqual(null, result.GetResponseUriList.FirstOrDefault());
Assert.IsTrue(Regex.IsMatch(result.FrontendUriList[1], @"https://.+/100/Http"));
}
/// <summary>
/// Ask broker to initialize
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <returns>returns broker initialization result</returns>
public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
ThrowIfNull(startInfo, "startInfo");
ThrowIfNull(brokerInfo, "brokerInfo");
this.entry = new BrokerEntry(startInfo, brokerInfo, this);
return(this.entry.Result);
}
/// <summary>
/// Create a new broker which working under Durable manner.
/// </summary>
/// <param name="info">session start info</param>
/// <param name="sessionid">the session id which is also service job id</param>
/// <returns>The created session info</returns>
public BrokerInitializationResult CreateDurable(SessionStartInfoContract info, string sessionId)
{
this.ThrowIfInitialized();
this.startInfoContract = info;
this.BuildBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, true, this.attached).GetAwaiter().GetResult();
this.CreateInternal(sessionId).GetAwaiter().GetResult();
return(this.result);
}
/// <summary>
/// Allocate a session and get a list of brokerlauncher EPR
/// </summary>
/// <param name="info">Session start info</param>
/// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param>
/// <param name="sessionid">the sessionid returns</param>
/// <param name="serviceVersion">the service version</param>
/// <param name="sessionInfo">the session info</param>
/// <returns>The EPRs of the broker launchers</returns>
public async Task <SessionAllocateInfoContract> AllocateAsync(SessionStartInfoContract info, string endpointPrefix)
{
return(await this.Channel.AllocateAsync(info, endpointPrefix).ConfigureAwait(false));
// Call async version and block on completion in order to workaround System.Net.Socket bug #750028
//IAsyncResult result = this.Channel.BeginAllocate(info, endpointPrefix, null, null);
//return this.Channel.EndAllocate(result);
}
/// <summary>
/// Create a session with the specific sessionid
/// </summary>
/// <param name="info">Session Start Info</param>
/// <param name="sessionid">the session id</param>
/// <param name="callback">The async callback</param>
/// <param name="state">async state object</param>
/// <returns>the async result</returns>
public IAsyncResult BeginCreate(
SessionStartInfoContract info,
string sessionid,
AsyncCallback callback,
object state)
{
return(this.Channel.BeginCreate(info, sessionid, callback, state));
}
public static async Task OpenSvcHostsAsync(string sessionId, SessionStartInfoContract startInfo, Func <List <TaskInfo>, Task> taskStateChangedCallBack)
{
for (int i = 0; i < startInfo.IpAddress.Length; i++)
{
OpenSvcHostWithRetryAsync(sessionId, i, startInfo.IpAddress[i], startInfo.RegPath, startInfo.ServiceName, startInfo.ServiceVersion, startInfo.Environments, startInfo.DependFilesStorageInfo)
.ContinueWith(t => taskStateChangedCallBack(new List <TaskInfo> {
t.Result
}));
}
}
protected abstract Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath);
/// <summary>
/// Initializes a new instance of the BrokerInfo class from broker recover info
/// </summary>
/// <param name="recoverInfo">indicating the broker recover info</param>
/// <param name="brokerInfo">indicating the broker start info</param>
/// <param name="auth">indicating the broker auth</param>
/// <param name="customBroker">indicating the custom broker configuration</param>
/// <param name="pool">indicating the broker process pool</param>
public BrokerInfo(BrokerRecoverInfo recoverInfo, BrokerStartInfo brokerInfo, BrokerAuthorization auth, CustomBrokerRegistration customBroker, BrokerProcessPool pool)
{
this.callbackToCloseBroker = new ThreadHelper <IAsyncResult>(new AsyncCallback(this.OnCloseBroker)).CallbackRoot;
this.durable = recoverInfo.Durable;
this.brokerInfo = brokerInfo;
this.sessionId = recoverInfo.SessionId;
this.sessionStartInfo = recoverInfo.StartInfo;
this.auth = auth;
this.customBroker = customBroker;
this.pool = pool;
this.sessionStartInfo.IpAddress = BrokerLauncherSettings.Default.SvcHostList.Cast <string>().ToArray();
this.sessionStartInfo.RegPath = BrokerLauncherSettings.Default.CCP_SERVICEREGISTRATION_PATH;
}
/// <summary>
/// Create a new broker
/// </summary>
/// <param name="info">session start info</param>
/// <param name="sessionid">the session id which is also service job id</param>
/// <returns>The session Info</returns>
public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionId)
{
// TODO: SF: change to async
this.ThrowIfInitialized();
this.startInfoContract = info;
if (info.IsNoSession)
{
this.BuildStandaloneBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, false, this.attached);
}
else
{
this.BuildBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, false, this.attached).GetAwaiter().GetResult();
}
this.CreateInternal(sessionId).GetAwaiter().GetResult();
return(this.result);
}
/// <summary>
/// Initializes a new instance of the BrokerEntry class
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance)
{
if (startInfo.TransportScheme != TransportScheme.NetTcp)
{
throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp.");
}
if (brokerInfo.Durable)
{
throw new NotSupportedException("Sample broker does not support durable session.");
}
this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance);
Frontend frontend = new Frontend(this.clientManager);
WebHttpBinding binding = new WebHttpBinding();
binding.MaxBufferPoolSize = 5000000;
binding.MaxBufferSize = 5000000;
binding.MaxReceivedMessageSize = 5000000;
binding.ReaderQuotas.MaxArrayLength = 5000000;
binding.ReaderQuotas.MaxBytesPerRead = 5000000;
binding.ReaderQuotas.MaxDepth = 5000000;
binding.ReaderQuotas.MaxNameTableCharCount = 5000000;
binding.ReaderQuotas.MaxStringContentLength = 5000000;
this.frontendServiceHost = new WebServiceHost(frontend, new Uri(String.Format("http://{0}:8081/", Environment.MachineName)));
ServiceEndpoint endpoint = this.frontendServiceHost.AddServiceEndpoint(typeof(IWebHttpFrontendService), binding, String.Empty);
endpoint.Behaviors.Add(new WebHttpBehavior());
string listenUri = endpoint.ListenUri.AbsoluteUri;
this.frontendServiceHost.Open();
this.result = new BrokerInitializationResult();
this.result.BrokerEpr = new string[3] {
listenUri, null, null
};
this.result.ControllerEpr = new string[3];
this.result.ResponseEpr = new string[3];
}
/// <summary>
/// Create a session
/// </summary>
/// <param name="info">session start info</param>
/// <param name="sessionId">session id</param>
/// <returns>returns broker initialization result</returns>
public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionId)
{
if ((!this.AllowNewSession) || (!this.IsOnline && String.IsNullOrEmpty(info.DiagnosticBrokerNode)))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerIsOffline, SR.BrokerIsOffline);
}
// Handle invalid input parameters
try
{
ParamCheckUtility.ThrowIfNull(info, "info");
if (!BrokerLauncherEnvironment.Standalone)
{
this.CheckAccess(sessionId);
}
TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerLauncher] Create: SessionId = {0}", sessionId);
//TODO: make it async
BrokerInitializationResult returnValue = this.brokerManager.CreateNewBrokerDomain(info, sessionId, false).GetAwaiter().GetResult();
#region Debug Failure Test
SimulateFailure.FailOperation(1);
#endregion
TraceHelper.RuntimeTrace.LogSessionCreated(sessionId);
TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerLauncher] Create Broker {0} Succeeded.", sessionId);
return(returnValue);
}
catch (Exception e)
{
TraceHelper.RuntimeTrace.LogFailedToCreateSession(sessionId);
// Bug 10614: Throw a proper exception when the broker node is being taken offline
if ((!this.AllowNewSession) || (!this.IsOnline && String.IsNullOrEmpty(info.DiagnosticBrokerNode)))
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerIsOffline, SR.BrokerIsOffline);
}
TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Error, "[BrokerLauncher] Create Broker {0} failed: {1}", sessionId, e.ToString());
throw ExceptionHelper.ConvertExceptionToFaultException(e);
}
}
/// <summary>
/// Initializes a new instance of the BrokerEntry class
/// </summary>
/// <param name="startInfo">indicating the start info</param>
/// <param name="brokerInfo">indicating the broker info</param>
public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance)
{
if (startInfo.TransportScheme != TransportScheme.NetTcp)
{
throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp.");
}
if (brokerInfo.Durable)
{
throw new NotSupportedException("Sample broker does not support durable session.");
}
this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance);
Frontend frontend = new Frontend(this.clientManager);
// Choose different binding configuration by start info
NetTcpBinding frontendBinding;
if (startInfo.Secure)
{
frontendBinding = new NetTcpBinding();
}
else
{
frontendBinding = new NetTcpBinding(SecurityMode.None);
}
frontendBinding.PortSharingEnabled = true;
this.frontendServiceHost = new ServiceHost(frontend, new Uri(String.Format("net.tcp://{0}:9091/SampleBroker", Environment.MachineName)));
string listenUri = this.frontendServiceHost.AddServiceEndpoint(typeof(IDuplexService), frontendBinding, String.Empty).ListenUri.AbsoluteUri;
this.frontendServiceHost.Open();
this.result = new BrokerInitializationResult();
this.result.BrokerEpr = new string[3] {
listenUri, null, null
};
this.result.ControllerEpr = new string[3];
this.result.ResponseEpr = new string[3];
}
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
// string sessionId = LocalSessionConfiguration.GetNextSessionId();
string cmd;
if (true)
{
cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath}";
}
else
{
cmd =
$"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath} --AzureStorageConnectionString {LocalSessionConfiguration.BrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True";
}
string sessionId = SessionStartInfo.StandaloneSessionId;
this.brokerLauncherProcess = Process.Start(
LocalSessionConfiguration.BrokerLauncherExePath,
cmd);
this.svcHostProcess = Process.Start(LocalSessionConfiguration.ServiceHostExePath, "-standalone");
sessionAllocateInfo.Id = sessionId;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress("localhost") };
return(sessionAllocateInfo);
}
/// <summary>
/// Override Start method and start dummy service job monitor
/// </summary>
/// <param name="startInfo"></param>
/// <param name="dispatcherManager"></param>
/// <param name="observer"></param>
/// <returns></returns>
public override async Task Start(SessionStartInfoContract startInfo, DispatcherManager dispatcherManager, BrokerObserver observer)
{
BrokerTracing.TraceVerbose("[ServiceJobMonitor].Start: Enter");
this.dispatcherManager = dispatcherManager;
this.observer = observer;
this.gracefulPreemptionHandler = new GracefulPreemptionHandler(this.dispatcherManager, this.sharedData.BrokerInfo.SessionId, taskId => this.FinishTask(taskId, isRunAwayTask: true));
this.schedulerAdapterClientFactory = new SchedulerAdapterClientFactory(this.sharedData, this, dispatcherManager, this.context);
if (this.sharedData.Config.Monitor.AllocationAdjustInterval != Timeout.Infinite && this.sharedData.StartInfo.EprList == null)
{
BrokerTracing.TraceVerbose("[ServiceJobMonitor].Start: start the allocation adjust thread, interval={0}", this.sharedData.Config.Monitor.AllocationAdjustInterval);
this.allocationAdjustThread.Start();
}
// await SvcHostRestModule.OpenSvcHostsAsync(this.sharedData.BrokerInfo.SessionId, startInfo, ((ISchedulerNotify)this).TaskStateChanged);
await((ISchedulerNotify)this).TaskStateChanged(SvcHostRestModule.CreateDummyTaskInfos(startInfo.IpAddress));
}
/// <summary>
/// Create a new broker application domain
/// </summary>
/// <param name="info">session start info</param>
/// <param name="sessionid">session id</param>
/// <param name="durable">indicate if the session is durable</param>
/// <returns>returns broker initialization result</returns>
public async Task <BrokerInitializationResult> CreateNewBrokerDomain(SessionStartInfoContract info, string sessionid, bool durable)
{
string userName =
(OperationContext.Current != null && OperationContext.Current.ServiceSecurityContext != null && OperationContext.Current.ServiceSecurityContext.WindowsIdentity != null) ?
OperationContext.Current.ServiceSecurityContext.WindowsIdentity.Name :
String.Empty;
TraceHelper.RuntimeTrace.LogSessionCreating(sessionid, userName);
TraceHelper.TraceEvent(sessionid, System.Diagnostics.TraceEventType.Information, "[BrokerManager] Create new broker domain: {0}", sessionid);
BrokerRecoverInfo recoverInfo = new BrokerRecoverInfo();
recoverInfo.StartInfo = info;
recoverInfo.SessionId = sessionid;
recoverInfo.Durable = durable;
if (this.schedulerHelper == null)
{
this.schedulerHelper = SchedulerHelperFactory.GetSchedulerHelper(this.context);
}
ClusterInfoContract clusterInfo = await this.schedulerHelper.GetClusterInfoAsync();
return(await this.CreateBrokerAndRun(recoverInfo, false, clusterInfo));
}
IAsyncResult IBrokerLauncher.BeginCreateDurable(SessionStartInfoContract info, string sessionId, AsyncCallback callback, object state)
{
throw new NotSupportedException();
}
public virtual async Task <SessionAllocateInfoContract> AllocateAsync(SessionStartInfoContract info, string endpointPrefix)
{
return(await this.AllocateInternalAsync(info, endpointPrefix, false));
}
/// <summary>
/// Allocate a new durable or non-durable session
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="durable">whether session should be durable</param>
/// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param>
/// <returns>the Broker Launcher EPRs, sorted by the preference.</returns>
protected virtual async Task <SessionAllocateInfoContract> AllocateInternalAsync(SessionStartInfoContract startInfo, string endpointPrefix, bool durable)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] Begin: AllocateInternalAsync");
SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract();
ParamCheckUtility.ThrowIfNull(startInfo, "startInfo");
ParamCheckUtility.ThrowIfNullOrEmpty(startInfo.ServiceName, "startInfo.ServiceName");
ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix");
#if HPCPACK
// check client api version, 4.3 or older client is not supported by 4.4 server for the broken changes
if (startInfo.ClientVersion == null || startInfo.ClientVersion < new Version(4, 4))
{
TraceHelper.TraceEvent(TraceEventType.Error,
"[SessionLauncher] .AllocateInternalAsync: ClientVersion {0} does not match ServerVersion {1}.", startInfo.ClientVersion, ServerVersion);
ThrowHelper.ThrowSessionFault(SOAFaultCode.ClientServerVersionMismatch,
SR.SessionLauncher_ClientServerVersionMismatch,
startInfo.ClientVersion == null ? "NULL" : startInfo.ClientVersion.ToString(),
ServerVersion.ToString());
}
#endif
// Init service version to the service version passed in
if (startInfo.ServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = startInfo.ServiceVersion;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is {0}", sessionAllocateInfo.ServiceVersion);
}
else
{
sessionAllocateInfo.ServiceVersion = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is null.");
}
string callId = Guid.NewGuid().ToString();
this.CheckAccess();
SecureString securePassword = CreateSecureString(startInfo.Password);
startInfo.Password = null;
// BUG 4522 : Use CCP_SCHEDULER when referencing service registration file share so HA HN virtual name is used when needed
//var reliableRegistry = new ReliableRegistry(this.fabricClient.PropertyManager);
//string defaultServiceRegistrationServerName = await reliableRegistry.GetValueAsync<string>(HpcConstants.HpcFullKeyName, HpcConstants.FileShareServerRegVal, this.token);
//if (String.IsNullOrEmpty(defaultServiceRegistrationServerName))
//{
// defaultServiceRegistrationServerName = "localhost";
//}
// the reg repo path is from scheduler environments, defaultServiceRegistrationServerName is actually not used
string serviceConfigFile;
ServiceRegistrationRepo serviceRegistration = this.GetRegistrationRepo(callId);
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, startInfo.ServiceVersion);
// If the serviceConfigFile wasnt found and serviceversion isnt specified, try getitng the service config based on the service's latest version
if (string.IsNullOrEmpty(serviceConfigFile) && (startInfo.ServiceVersion == null))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Try to find out versioned service.");
// Get service version in ServiceRegistrationRepo
Version dynamicServiceVersion = serviceRegistration.GetServiceVersionInternal(startInfo.ServiceName, false);
if (dynamicServiceVersion != null)
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Selected dynamicServiceVersion is {0}.", dynamicServiceVersion.ToString());
}
serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, dynamicServiceVersion);
// If a config file is found, update the serviceVersion that is returned to client and stored in recovery info
if (!string.IsNullOrEmpty(serviceConfigFile))
{
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: serviceConfigFile is {0}.", serviceConfigFile);
startInfo.ServiceVersion = dynamicServiceVersion;
if (dynamicServiceVersion != null)
{
sessionAllocateInfo.ServiceVersion = dynamicServiceVersion;
}
}
}
string serviceName = ServiceRegistrationRepo.GetServiceRegistrationFileName(startInfo.ServiceName, startInfo.ServiceVersion);
TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Service name = {0}, Configuration file = {1}", serviceName, serviceConfigFile);
// If the service is not found and user code doesn't specify
// version, we will use the latest version.
if (string.IsNullOrEmpty(serviceConfigFile))
{
if (startInfo.ServiceVersion != null)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ServiceVersion_NotFound, SR.SessionLauncher_ServiceVersionNotFound, startInfo.ServiceName, startInfo.ServiceVersion.ToString());
}
else
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_NotFound, SR.SessionLauncher_ServiceNotFound, startInfo.ServiceName);
}
}
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = serviceConfigFile;
ServiceRegistration registration = null;
BrokerConfigurations brokerConfigurations = null;
string hostpath = null;
string traceSwitchValue = null;
try
{
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException).GetAwaiter().GetResult();
Debug.Assert(config != null, "Configuration is not opened properly.");
registration = ServiceRegistration.GetSectionGroup(config);
brokerConfigurations = BrokerConfigurations.GetSectionGroup(config);
if (registration != null && registration.Host != null && registration.Host.Path != null)
{
hostpath = registration.Host.Path;
}
else
{
// x86 or x64
hostpath = registration.Service.Architecture == ServiceArch.X86 ? TaskCommandLine32 : TaskCommandLine64;
}
traceSwitchValue = registration.Service.SoaDiagTraceLevel;
// TODO: should deprecate the previous settings
if (string.IsNullOrEmpty(traceSwitchValue))
{
traceSwitchValue = ConfigurationHelper.GetTraceSwitchValue(config);
}
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, SR.SessionLauncher_ConfigFileInvalid, e.ToString());
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, ex.ToString());
throw;
}
// after figuring out the service and version, and the session pool size, we check if the service pool already has the instance.
sessionAllocateInfo.Id = "0";
sessionAllocateInfo.SessionInfo = null;
if (startInfo.UseSessionPool)
{
if (this.TryGetSessionAllocateInfoFromPooled(endpointPrefix, durable, sessionAllocateInfo, serviceConfigFile, registration, out var allocateInternal))
{
return(allocateInternal);
}
}
// for sessions to add in session pool
try
{
var sessionAllocateInfoContract = await this.CreateAndSubmitSessionJob(
startInfo,
endpointPrefix,
durable,
callId,
securePassword,
registration,
sessionAllocateInfo,
traceSwitchValue,
serviceName,
brokerConfigurations,
hostpath);
if (sessionAllocateInfoContract != null)
{
return(sessionAllocateInfoContract);
}
}
finally
{
// Add the submitted job to the session pool.
if (startInfo.UseSessionPool)
{
this.AddSessionToPool(Path.GetFileNameWithoutExtension(serviceConfigFile), durable, sessionAllocateInfo.Id, registration.Service.MaxSessionPoolSize);
}
}
return(null);
}
/// <summary>
/// Run the broker
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="brokerInfo">indicate the broker start info</param>
/// <returns>initialization result</returns>
public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId);
try
{
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology);
// Step 1: Initialize configuration and shared data
ServiceConfiguration serviceConfig;
BrokerConfigurations brokerConfig;
BindingsSection bindings;
SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed.
ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings);
this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded.");
Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}");
// Step 2: Initialize broker queue
ClientInfo[] clientInfo;
this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded.");
// Step 3: Initialize observer
this.observer = new BrokerObserver(this.sharedData, clientInfo);
this.sharedData.Observer = this.observer;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded.");
// Step 4: Initialize state manager
this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0);
this.stateManager.UnloadBroker += this.UnloadBroker;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded.");
// Step 5: Initialize service job monitor
var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode);
if (SoaCommonConfig.WithoutSessionLayer)
{
this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
else
{
this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded.");
// Step 6: Initalize broker authorization
this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded.");
// Step 7: Initialize dispatcher manager
DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded.");
// Step 8: Start service job monitor
this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded.");
// Step 9: Initailize client manager
this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded.");
// if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist
string[] requestQueueUris = { };
string requestBlobUri = string.Empty;
string controllerRequestQueueUri = string.Empty;
string controllerResponseQueueUri = string.Empty;
if (startInfo.UseAzureStorage)
{
int clusterHash = 0;
if (!string.IsNullOrEmpty(brokerInfo.ClusterId))
{
string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant();
clusterHash = clusterIdString.GetHashCode();
}
else if (!string.IsNullOrEmpty(brokerInfo.ClusterName))
{
string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant();
clusterHash = clusterNameString.GetHashCode();
}
else
{
throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined.");
}
if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString))
{
this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString);
requestQueueUris = this.azureQueueProxy.RequestQueueUris;
requestBlobUri = this.azureQueueProxy.RequestBlobUri;
var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId);
var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId);
controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
requestQName,
CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult();
controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
responseQName,
CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult();
if (this.SessionId == SessionStartInfo.StandaloneSessionId)
{
CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName });
}
}
else
{
BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set.");
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable);
}
}
// Step 10: Initialize frontend
this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy);
////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize;
////this.readerQuotas = this.frontendResult.ReaderQuotas;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded.");
// Step 11: Start frontend, Initialization finished after this step
this.OpenFrontend();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded.");
// Step 12: Build initialization result and retrun to client
BrokerInitializationResult result = BrokerEntry.BuildInitializationResult(
this.frontendResult,
dispatcherManager,
this.sharedData.Config.LoadBalancing.ServiceOperationTimeout,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount,
requestQueueUris,
requestBlobUri,
controllerRequestQueueUri,
controllerResponseQueueUri,
startInfo.UseAzureStorage);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded.");
BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded.");
return(result);
}
catch (Exception ex)
{
BrokerTracing.TraceError(ex.ToString());
throw;
}
finally
{
if (this.sharedData != null)
{
this.sharedData.InitializationFinished();
}
}
}
/// <summary>
/// override the Start Method in internal service job monitor
/// </summary>
/// <param name="startInfo"></param>
/// <param name="dispatcherManager"></param>
/// <param name="observer"></param>
/// <returns></returns>
public override async Task Start(SessionStartInfoContract startInfo, DispatcherManager dispatcherManager, BrokerObserver observer)
{
await this.Start(dispatcherManager, observer);
}
internal static async Task <ServiceTaskDispatcherInfo> OpenSvcHostsAsync(string sessionId, SessionStartInfoContract startInfo, ServiceTaskDispatcherInfo taskDispatcherInfo)
{
return(await OpenSvcHostWithRetryAsync(sessionId, taskDispatcherInfo, startInfo.RegPath, startInfo.ServiceName, startInfo.ServiceVersion, startInfo.Environments, startInfo.DependFilesStorageInfo));
}
/// <summary>
/// This method will only be internally called by session launcher,
/// before the job is submitted. It combines the user's job properties and the
/// items set in the service start info. The job properties will override the start info items.
/// </summary>
/// <param name="startInfo">the session start info</param>
/// <param name="schedulerJob">service job</param>
/// <param name="traceLevel">diag trace level</param>
internal static void MakeJobProperties(SessionStartInfoContract startInfo, ISchedulerJob schedulerJob, string traceLevel)
{
Debug.Assert(startInfo != null,
"The session startInfo cannot be null.");
Debug.Assert(!string.IsNullOrEmpty(startInfo.ServiceName),
"The service name in the session start info cannnot be null or empty.");
schedulerJob.ServiceName = startInfo.ServiceName;
schedulerJob.SetEnvironmentVariable("CCP_SERVICENAME", startInfo.ServiceName);
if (startInfo.CanPreempt.HasValue)
{
schedulerJob.CanPreempt = startInfo.CanPreempt.Value;
}
if (!string.IsNullOrEmpty(startInfo.JobTemplate))
{
schedulerJob.SetJobTemplate(startInfo.JobTemplate);
}
if (startInfo.Priority.HasValue)
{
schedulerJob.Priority = (JobPriority)startInfo.Priority.Value;
}
if (startInfo.ExtendedPriority.HasValue)
{
schedulerJob.ExpandedPriority = startInfo.ExtendedPriority.Value;
}
schedulerJob.Progress = 0;
// For max units
if (startInfo.MaxUnits != null)
{
schedulerJob.MaximumNumberOfCores =
schedulerJob.MaximumNumberOfSockets =
schedulerJob.MaximumNumberOfNodes = startInfo.MaxUnits.Value;
schedulerJob.AutoCalculateMax = false;
}
// For min units
if (startInfo.MinUnits != null)
{
schedulerJob.MinimumNumberOfCores =
schedulerJob.MinimumNumberOfSockets =
schedulerJob.MinimumNumberOfNodes = startInfo.MinUnits.Value;
schedulerJob.AutoCalculateMin = false;
}
// Should set UnitType after above resource count update
if (startInfo.ResourceUnitType != null)
{
schedulerJob.UnitType = (JobUnitType)startInfo.ResourceUnitType.Value;
}
schedulerJob.Name = string.IsNullOrEmpty(startInfo.ServiceJobName) ?
string.Format("{0} - WCF service", startInfo.ServiceName) :
startInfo.ServiceJobName;
if (!string.IsNullOrEmpty(startInfo.ServiceJobProject))
{
schedulerJob.Project = startInfo.ServiceJobProject;
}
if (!string.IsNullOrEmpty(startInfo.NodeGroupsStr))
{
string[] nodes = startInfo.NodeGroupsStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string node in nodes)
{
schedulerJob.NodeGroups.Add(node);
}
}
if (!string.IsNullOrEmpty(startInfo.RequestedNodesStr))
{
schedulerJob.RequestedNodes = new StringCollection(startInfo.RequestedNodesStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries));
}
if (startInfo.Runtime >= 0)
{
schedulerJob.Runtime = startInfo.Runtime;
}
// start adding the broker settings.
schedulerJob.SetCustomProperty(BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString());
schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAad, startInfo.UseAad.ToString());
schedulerJob.SetCustomProperty(BrokerSettingsConstants.Secure, startInfo.Secure.ToString());
schedulerJob.SetCustomProperty(BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString());
schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString());
schedulerJob.SetCustomProperty(BrokerSettingsConstants.LocalUser, (startInfo.LocalUser == true).ToString());
var context = HpcContext.Get();
var principal = Thread.CurrentPrincipal;
if (principal.IsHpcAadPrincipal(context))
{
string identity = principal.GenerateSecurityIdentifierFromAadPrincipal(context).Value + ";" + principal.Identity.Name;
schedulerJob.SetCustomProperty(BrokerSettingsConstants.AadUserIdentity, identity);
}
// Save ServiceVersion if set
if (startInfo.ServiceVersion != null)
{
schedulerJob.SetCustomProperty(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion.ToString());
}
string[] customPropNames = new string[]
{
BrokerSettingsConstants.AllocationGrowLoadRatioThreshold,
BrokerSettingsConstants.AllocationShrinkLoadRatioThreshold,
BrokerSettingsConstants.ClientIdleTimeout,
BrokerSettingsConstants.SessionIdleTimeout,
BrokerSettingsConstants.MessagesThrottleStartThreshold,
BrokerSettingsConstants.MessagesThrottleStopThreshold,
BrokerSettingsConstants.ClientConnectionTimeout,
BrokerSettingsConstants.ServiceConfigMaxMessageSize,
BrokerSettingsConstants.ServiceConfigOperationTimeout,
BrokerSettingsConstants.DispatcherCapacityInGrowShrink
};
int?[] intNullableValues = new int?[]
{
startInfo.AllocationGrowLoadRatioThreshold,
startInfo.AllocationShrinkLoadRatioThreshold,
startInfo.ClientIdleTimeout,
startInfo.SessionIdleTimeout,
startInfo.MessagesThrottleStartThreshold,
startInfo.MessagesThrottleStopThreshold,
startInfo.ClientConnectionTimeout,
startInfo.MaxMessageSize,
startInfo.ServiceOperationTimeout,
startInfo.DispatcherCapacityInGrowShrink
};
Debug.Assert(intNullableValues.Length == customPropNames.Length);
for (int i = 0; i < customPropNames.Length; i++)
{
if (intNullableValues[i].HasValue)
{
schedulerJob.SetCustomProperty(customPropNames[i], intNullableValues[i].Value.ToString());
}
}
// add soa diag settings
schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceLevel, traceLevel);
schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceCleanup, Boolean.FalseString);
}
/// <summary>
/// Load the configuration from the configuration file
/// If no configuration is found or some part is missing, we will fill the blank with default value
/// </summary>
/// <param name="brokerSettings">indicate the broker settings from the session start info, this settings will override the settings load from the configuration file</param>
/// <param name="brokerInfo">indicating the broker info</param>
/// <param name="brokerConfig">out the broker configurations</param>
/// <param name="serviceConfig">out the service configurations</param>
/// <param name="bindings">output the bindings</param>
public static void LoadConfiguration(SessionStartInfoContract brokerSettings, BrokerStartInfo brokerInfo, out BrokerConfigurations brokerConfig, out ServiceConfiguration serviceConfig, out BindingsSection bindings)
{
// Init config file
string filename = brokerInfo.ConfigurationFile;
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 1: Load configuration file name: {0}", filename);
brokerConfig = null;
serviceConfig = null;
bindings = null;
try
{
ExeConfigurationFileMap map = new ExeConfigurationFileMap();
map.ExeConfigFilename = filename;
Configuration config = null;
RetryManager.RetryOnceAsync(
() => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None),
TimeSpan.FromSeconds(1),
ex => ex is ConfigurationErrorsException)
.GetAwaiter()
.GetResult();
brokerConfig = BrokerConfigurations.GetSectionGroup(config);
serviceConfig = ServiceRegistration.GetSectionGroup(config).Service;
bindings = (BindingsSection)config.GetSection(BindingsSectionName);
}
catch (ConfigurationErrorsException e)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid,
"{0}",
e.ToString());
}
if (brokerConfig == null)
{
// Set the default value
brokerConfig = defaultBrokerConfiguration;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the broker config from the configuration file, use the default configuration");
}
else
{
// Set the default configuration if the very section is not found
if (brokerConfig.Monitor == null)
{
brokerConfig.Monitor = defaultBrokerConfiguration.Monitor;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the monitor config from the configuration file, use the default configuration");
}
if (brokerConfig.Services == null)
{
brokerConfig.Services = defaultBrokerConfiguration.Services;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the services config from the configuration file, use the default configuration");
}
if (brokerConfig.LoadBalancing == null)
{
brokerConfig.LoadBalancing = defaultBrokerConfiguration.LoadBalancing;
BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the load balancing config from the configuration file, use the default configuration");
}
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 2: Load broker config and service config succeeded.");
if (brokerConfig.Monitor.AllocationAdjustInterval < MinAllocationAdjustTime && brokerConfig.Monitor.AllocationAdjustInterval != System.Threading.Timeout.Infinite)
{
brokerConfig.Monitor.AllocationAdjustInterval = MinAllocationAdjustTime;
}
// Update the broker config using the session start info
if (brokerSettings.ClientIdleTimeout.HasValue)
{
brokerConfig.Monitor.ClientIdleTimeout = brokerSettings.ClientIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientIdleTimeout to {0}", brokerConfig.Monitor.ClientIdleTimeout);
}
if (brokerSettings.ClientConnectionTimeout.HasValue)
{
brokerConfig.Monitor.ClientConnectionTimeout = brokerSettings.ClientConnectionTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientConnectionTimeout to {0}", brokerConfig.Monitor.ClientConnectionTimeout);
}
if (brokerSettings.SessionIdleTimeout.HasValue)
{
brokerConfig.Monitor.SessionIdleTimeout = brokerSettings.SessionIdleTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default SessionIdleTimeout to {0}", brokerConfig.Monitor.SessionIdleTimeout);
}
if (brokerSettings.MessagesThrottleStartThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStartThreshold = brokerSettings.MessagesThrottleStartThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStartThreshold to {0}", brokerConfig.Monitor.MessageThrottleStartThreshold);
}
if (brokerSettings.MessagesThrottleStopThreshold.HasValue)
{
brokerConfig.Monitor.MessageThrottleStopThreshold = brokerSettings.MessagesThrottleStopThreshold.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStopThreshold to {0}", brokerConfig.Monitor.MessageThrottleStopThreshold);
}
if (brokerSettings.ClientBrokerHeartbeatRetryCount.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount = brokerSettings.ClientBrokerHeartbeatRetryCount.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatRetryCount to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount);
}
if (brokerSettings.ClientBrokerHeartbeatInterval.HasValue)
{
brokerConfig.Monitor.ClientBrokerHeartbeatInterval = brokerSettings.ClientBrokerHeartbeatInterval.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatInterval to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatInterval);
}
if (brokerSettings.ServiceOperationTimeout.HasValue)
{
brokerConfig.LoadBalancing.ServiceOperationTimeout = brokerSettings.ServiceOperationTimeout.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ServiceOperationTimeout to {0}", brokerConfig.LoadBalancing.ServiceOperationTimeout);
}
if (brokerSettings.DispatcherCapacityInGrowShrink.HasValue)
{
brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink = brokerSettings.DispatcherCapacityInGrowShrink.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default DispatcherCapacityInGrowShrink to {0}", brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink);
}
if (brokerSettings.MaxMessageSize.HasValue)
{
serviceConfig.MaxMessageSize = brokerSettings.MaxMessageSize.Value;
BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MaxMessageSize to {0}", serviceConfig.MaxMessageSize);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 3: Override broker settings using session start info succeeded.");
// Validate the config section
string configError;
bool validateSucceeded;
try
{
validateSucceeded = brokerConfig.Validate(out configError);
}
catch (ConfigurationErrorsException e)
{
validateSucceeded = false;
configError = e.Message;
}
if (!validateSucceeded)
{
BrokerTracing.TraceEvent(TraceEventType.Error, 0, "[ConfigurationHelper] Invalid broker configuration section. Error {0}", configError);
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_InvalidConfiguration, configError);
}
BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 4: Validate broker configuration succeeded.");
StringBuilder sb = new StringBuilder();
sb.AppendLine("[Monitor]");
BrokerTracing.WriteProperties(sb, brokerConfig.Monitor, 3, typeof(int), typeof(string));
sb.AppendLine("[BaseAddress]");
sb.AppendFormat(" Http = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("http"));
sb.AppendFormat(" Https = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("https"));
sb.AppendFormat(" NetTcp = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("net.tcp"));
sb.AppendLine("[LoadBalancing]");
BrokerTracing.WriteProperties(sb, brokerConfig.LoadBalancing, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] BrokerConfiguration: \n{0}", sb.ToString());
sb = new StringBuilder();
sb.AppendLine("[Service]");
BrokerTracing.WriteProperties(sb, serviceConfig, 3, typeof(int), typeof(string));
BrokerTracing.TraceVerbose("[ConfigurationHelper] ServiceConfiguration: \n{0}", sb.ToString());
}
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob(
SessionStartInfoContract startInfo,
string endpointPrefix,
bool durable,
string callId,
SecureString securePassword,
ServiceRegistration registration,
SessionAllocateInfoContract sessionAllocateInfo,
string traceSwitchValue,
string serviceName,
BrokerConfigurations brokerConfigurations,
string hostpath)
{
try
{
bool brokerPerfMode = true; // TODO: implement separated broker mode
if (brokerPerfMode)
{
TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode");
}
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.",
callId,
endpointPrefix,
durable);
using (var batchClient = AzureBatchConfiguration.GetBatchClient())
{
var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName);
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "affinityId, ipAddress";
//detailLevel.FilterClause = @"state eq 'idle'";
var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync();
if (nodes.Count < 1)
{
throw new InvalidOperationException("Compute node count in selected pool is less then 1.");
}
sessionAllocateInfo.Id = string.Empty;
// sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken };
IList <EnvironmentSetting> ConstructEnvironmentVariable()
{
List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite
foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables)
{
env.Add(new EnvironmentSetting(entry.Name, entry.Value));
}
// pass service serviceInitializationTimeout as job environment variables
env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString()));
if (startInfo.ServiceHostIdleTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString()));
}
if (startInfo.ServiceHangTimeout == null)
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString()));
}
else
{
env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString()));
}
// pass MessageLevelPreemption switcher as job environment variables
env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString()));
// pass trace switcher to svchost
if (!string.IsNullOrEmpty(traceSwitchValue))
{
env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue));
}
// pass taskcancelgraceperiod as environment variable to svchosts
env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString()));
// pass service config file name to services
env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName));
// pass maxMessageSize to service hosts
int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize;
env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString()));
// pass service operation timeout to service hosts
int?serviceOperationTimeout = null;
if (startInfo.ServiceOperationTimeout.HasValue)
{
serviceOperationTimeout = startInfo.ServiceOperationTimeout;
}
else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null)
{
serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout;
}
if (serviceOperationTimeout.HasValue)
{
env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString()));
}
if (startInfo.Environments != null)
{
foreach (KeyValuePair <string, string> entry in startInfo.Environments)
{
env.Add(new EnvironmentSetting(entry.Key, entry.Value));
}
}
// Each SOA job is assigned a GUID "secret", which is used
// to identify soa job owner. When a job running in Azure
// tries to access common data, it sends this "secret" together
// with a data request to data service. Data service trusts
// the data request only if the job id and job "secret"
// match.
env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString()));
// Set CCP_SERVICE_SESSIONPOOL env var of the job
if (startInfo.UseSessionPool)
{
env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString));
}
void SetBrokerNodeAuthenticationInfo()
{
// TODO: set the information needed by compute node to authenticate broker node
return;
}
SetBrokerNodeAuthenticationInfo();
env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString()));
env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString()));
TraceHelper.TraceEvent(
TraceEventType.Information,
"[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.",
callId,
BrokerSettingsConstants.Secure,
startInfo.Secure,
BrokerSettingsConstants.TransportScheme,
startInfo.TransportScheme);
env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName()));
env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE"));
//Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR
env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir));
return(env);
}
var environment = ConstructEnvironmentVariable();
ResourceFile GetResourceFileReference(string containerName, string blobPrefix)
{
var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read);
ResourceFile rf;
if (string.IsNullOrEmpty(blobPrefix))
{
rf = ResourceFile.FromStorageContainerUrl(sasToken);
}
else
{
rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix);
}
return(rf);
}
async Task <string> CreateJobAsync()
{
//TODO: need a function to test if all parameters are legal.
if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0)
{
throw new ArgumentException("Maxunit value is invalid.");
}
string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId());
Debug.Assert(batchClient != null, nameof(batchClient) + " != null");
var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation()
{
PoolId = AzureBatchConfiguration.BatchPoolName
});
job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine);
job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
job.JobPreparationTask.ResourceFiles = new List <ResourceFile>()
{
GetResourceFileReference(ServiceRegistrationContainer, null),
GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder),
GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower())
};
job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine);
job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// List<ResourceFile> resourceFiles = new List<ResourceFile>();
// resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
// resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null));
// // job.JobManagerTask = new JobManagerTask("Broker",
// // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}");
// job.JobManagerTask = new JobManagerTask("List",
// $@"cmd /c dir & set");
// job.JobManagerTask.ResourceFiles = resourceFiles;
// job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
// Set Meta Data
if (job.Metadata == null)
{
job.Metadata = new List <MetadataItem>();
}
Dictionary <string, string> jobMetadata = new Dictionary <string, string>()
{
{ BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() },
{ BrokerSettingsConstants.Secure, startInfo.Secure.ToString() },
{ BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() },
{ BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() },
};
if (startInfo.ServiceVersion != null)
{
jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString());
}
if (startInfo.MaxUnits != null)
{
jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString());
}
Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>()
{
{ BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout },
{ BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout },
{ BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold },
{ BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold },
{ BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout },
{ BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize },
{ BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout },
{ BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink }
};
job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value)))
.Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList();
job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service";
await job.CommitAsync();
return(job.Id);
}
var jobId = await CreateJobAsync();
string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId);
if (!sessionId.Equals("-1"))
{
sessionAllocateInfo.Id = sessionId;
}
else
{
TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId);
}
Task AddTasksAsync()
{
int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count;
var comparer = new EnvironmentSettingComparer();
CloudTask CreateTask(string taskId)
{
CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone");
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
CloudTask CreateBrokerTask(bool direct)
{
List <ResourceFile> resourceFiles = new List <ResourceFile>();
resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder));
string cmd;
if (direct)
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
else
{
cmd =
$@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}";
}
CloudTask cloudTask = new CloudTask("Broker", cmd);
cloudTask.ResourceFiles = resourceFiles;
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool));
cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList();
return(cloudTask);
}
//TODO: task id type should be changed from int to string
var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray();
if (!brokerPerfMode)
{
tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray();
}
else
{
tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray();
}
return(batchClient.JobOperations.AddTaskAsync(jobId, tasks));
}
await AddTasksAsync();
async Task WaitBatchBrokerLauncher()
{
var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker");
TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor();
await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout);
await brokerTask.RefreshAsync();
var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress;
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) };
}
if (brokerPerfMode)
{
//If broker node and session launcher node is not the same node, this line should be modified.
sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) };
}
else
{
await WaitBatchBrokerLauncher();
}
return(sessionAllocateInfo);
}
}
catch (Exception ex)
{
TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}");
throw;
}
}
BrokerInitializationResult IBrokerLauncher.CreateDurable(SessionStartInfoContract info, string sessionId)
{
throw new NotSupportedException();
}
