/// <summary>
/// Initializes a new instance of the BrokerLauncherClient class.
/// </summary>
/// <param name="uri">The broker launcher EPR</param>
public BrokerLauncherClient(Uri uri, string certThrumbprint)
: base(GetBinding(uri), GetEndpoint(uri, certThrumbprint))
{
string thumbpint = TelepathyContext.Get().GetSSLThumbprint().GetAwaiter().GetResult();
this.ClientCredentials.UseInternalAuthentication(thumbpint);
if (!SoaHelper.IsOnAzure())
{
this.ClientCredentials.Windows.AllowedImpersonationLevel = TokenImpersonationLevel.Impersonation;
}
}
/// <summary>
/// This method closes the session async with the given ID
/// </summary>
/// <param name="headNode">Headnode name</param>
/// <param name="sessionId">The ID of the session to be closed</param>
/// <param name="binding">indicting the binding</param>
/// <param name="isAadUser">If the session is belong to a AAD user</param>
/// <param name="token">The cancellation token.</param>
public static async Task CloseSessionAsync(string headNode, string sessionId, Binding binding, bool isAadUser, CancellationToken token)
{
Utility.ThrowIfEmpty(headNode, "headNode");
SessionLauncherClient client = null;
BrokerLauncherClient broker = null;
string headNodeMachine = await TelepathyContext.GetOrAdd(headNode).ResolveSessionLauncherNodeAsync().ConfigureAwait(false);
try
{
client = new SessionLauncherClient(headNodeMachine, binding, isAadUser);
client.InnerChannel.OperationTimeout = GetTimeout(DateTime.MaxValue);
// TODO: need to change the endpoint prefix for https
SessionInfo info = null;
if (binding is NetTcpBinding)
{
info = Utility.BuildSessionInfoFromDataContract(await client.GetInfoAsync(SessionLauncherClient.EndpointPrefix, sessionId).ConfigureAwait(false));
}
#if !net40
else if (binding is BasicHttpBinding || binding is NetHttpBinding || binding is NetHttpsBinding)
{
info = Utility.BuildSessionInfoFromDataContract(await client.GetInfoAsync(SessionLauncherClient.HttpsEndpointPrefix, sessionId).ConfigureAwait(false));
}
#endif
broker = new BrokerLauncherClient(info, binding, new Uri(info.BrokerLauncherEpr));
broker.InnerChannel.OperationTimeout = GetTimeout(DateTime.MaxValue);
broker.Close(sessionId);
}
catch (FaultException <SessionFault> e)
{
throw Utility.TranslateFaultException(e);
}
finally
{
if (client != null)
{
Utility.SafeCloseCommunicateObject(client);
}
if (broker != null)
{
Utility.SafeCloseCommunicateObject(broker);
}
}
}
/// <summary>
/// Set service credential to use internal authentication
/// </summary>
/// <param name="serviceCredentials"></param>
/// <param name="winService"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <ServiceCredentials> UseInternalAuthenticationAsync(
this ServiceCredentials serviceCredentials,
bool winService = false,
CancellationToken cancellationToken = default(CancellationToken))
{
IRegistry registry = winService ? Registry : TelepathyContext.GetOrAdd().Registry;
serviceCredentials.ClientCertificate.Authentication.CertificateValidationMode = X509CertificateValidationMode.PeerOrChainTrust;
serviceCredentials.ClientCertificate.Authentication.RevocationMode = X509RevocationMode.NoCheck;
serviceCredentials.ServiceCertificate.SetCertificate(
StoreLocation.LocalMachine,
StoreName.My,
X509FindType.FindByThumbprint,
await registry.GetSSLThumbprint(cancellationToken).ConfigureAwait(false));
return(serviceCredentials);
}
/// <summary>
/// Set service host to use specified <see cref="ServiceAuthorizationManager"/>
/// </summary>
/// <param name="serviceHost"></param>
/// <param name="authorizationManager"></param>
/// <param name="winService"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <ServiceHost> UseServiceAuthorizationManagerAsync(
this ServiceHost serviceHost,
ServiceAuthorizationManager authorizationManager,
bool winService = false,
CancellationToken cancellationToken = default(CancellationToken))
{
IRegistry registry = winService ? Registry : TelepathyContext.GetOrAdd().Registry;
serviceHost.Credentials.ServiceCertificate.SetCertificate(
StoreLocation.LocalMachine,
StoreName.My,
X509FindType.FindByThumbprint,
await registry.GetSSLThumbprint(cancellationToken).ConfigureAwait(false));
serviceHost.Authorization.ServiceAuthorizationManager = authorizationManager;
return(serviceHost);
}
/// <summary>
/// Start scheduler delegation service
/// </summary>
private void StartSchedulerDelegationService()
{
string schedulerDelegationAddress = SoaHelper.GetSchedulerDelegationAddress("localhost");
this.delegationHost = new ServiceHost(this.schedulerDelegation, new Uri(schedulerDelegationAddress));
BindingHelper.ApplyDefaultThrottlingBehavior(this.delegationHost);
#if HPCPACK
if (this.schedulerDelegation is IHpcSchedulerAdapterInternal)
{
this.delegationHost.AddServiceEndpoint(typeof(IHpcSchedulerAdapterInternal), BindingHelper.HardCodedInternalSchedulerDelegationBinding, "Internal");
this.delegationHost.AddServiceEndpoint(typeof(IHpcSchedulerAdapter), BindingHelper.HardCodedInternalSchedulerDelegationBinding, string.Empty);
this.delegationHost.Credentials.ClientCertificate.Authentication.CertificateValidationMode = X509CertificateValidationMode.PeerOrChainTrust;
this.delegationHost.Credentials.ClientCertificate.Authentication.RevocationMode = X509RevocationMode.NoCheck;
this.delegationHost.Credentials.ServiceCertificate.SetCertificate(
StoreLocation.LocalMachine,
StoreName.My,
X509FindType.FindByThumbprint,
TelepathyContext.Get().GetSSLThumbprint().GetAwaiter().GetResult());
}
else
#endif
{
// Use insecure binding until unified authentication logic is implemented
this.delegationHost.AddServiceEndpoint(typeof(ISchedulerAdapter), BindingHelper.HardCodedUnSecureNetTcpBinding, string.Empty);
// if (SessionLauncherRuntimeConfiguration.OpenAzureStorageListener)
// {
// this.delegationHost.AddServiceEndpoint(
// typeof(ISchedulerAdapter),
// new TableTransportBinding() { ConnectionString = SessionLauncherRuntimeConfiguration.SessionLauncherStorageConnectionString, TargetPartitionKey = "all" },
// TelepathyConstants.SessionSchedulerDelegationAzureTableBindingAddress);
// }
}
this.delegationHost.Faulted += SchedulerDelegationHostFaultHandler;
this.delegationHost.Open();
TraceHelper.TraceEvent(TraceEventType.Information, "Open scheduler delegation service at {0}", schedulerDelegationAddress);
}
/// <summary>
/// Initializes a new instance of the BrokerNodesManager class with the specified head node.
/// </summary>
/// <param name="headNode">the head node name.</param>
public BrokerNodesManager()
{
this.schedulerField = CommonSchedulerHelper.GetScheduler(TelepathyContext.Get().CancellationToken).GetAwaiter().GetResult();
// Initialize performance counters
this.totalFailoverBrokerNodeCount = SessionPerformanceCounterHelper.GetPerfCounter(SessionPerformanceCounterKey.TotalFailoverBrokerNodeCount);
this.totalFailoverClusterCount = SessionPerformanceCounterHelper.GetPerfCounter(SessionPerformanceCounterKey.TotalFailoverClusterCount);
this.activeBrokerNodeCount = SessionPerformanceCounterHelper.GetPerfCounter(SessionPerformanceCounterKey.ActiveBrokerNodeCount);
this.activeBrokerResourceGroupCount = SessionPerformanceCounterHelper.GetPerfCounter(SessionPerformanceCounterKey.ActiveBrokerResourceGroupCount);
if (SoaHelper.IsOnAzure())
{
this.brokerNodeTimer = new Timer(this.UpdateAvailableAzureBroker, null, 0, CheckNodesPerClusterInterval);
}
else
{
// poll the broker nodes immediately once connect to the scheduler.
this.BrokerNodesTimerCallback(null);
// Start the timer that will poll broker nodes
// TODO: Consider a timer per cluster
this.brokerNodeTimer = new Timer(this.BrokerNodesTimerCallback, null, CheckNodesPerClusterInterval, CheckNodesPerClusterInterval);
}
}
private static void Main(string[] args)
{
var log = new LoggerConfiguration().ReadFrom.AppSettings().Enrich.WithMachineName().CreateLogger();
Log.Logger = log;
if (!ParseAndSetBrokerLauncherSettings(args, BrokerLauncherSettings.Default))
{
// parsing failed
return;
}
if (ConfigureLogging)
{
Trace.TraceInformation("Log configuration for Broker Launcher has done successfully.");
Log.CloseAndFlush();
return;
}
// clusterconnectionstring could be a machine name (for single headnode) or a connection string
ITelepathyContext context;
string clusterConnectionString = SoaHelper.GetSchedulerName();
context = TelepathyContext.GetOrAdd(clusterConnectionString);
Trace.TraceInformation("Get diag trace enabled internal.");
SoaDiagTraceHelper.IsDiagTraceEnabledInternal = (sessionId) =>
{
try
{
using (ISchedulerHelper helper = SchedulerHelperFactory.GetSchedulerHelper(context))
{
return(helper.IsDiagTraceEnabled(sessionId).GetAwaiter().GetResult());
}
}
catch (Exception e)
{
Trace.TraceError("[SoaDiagTraceHelper] Failed to get IsDiagTraceEnabled property: {0}", e);
return(false);
}
};
TraceHelper.IsDiagTraceEnabled = SoaDiagTraceHelper.IsDiagTraceEnabled;
LauncherHostService host = null;
BrokerManagement brokerManagement = null;
// richci : Run as a console application if user wants to debug (-D) or run in MSCS (-FAILOVER)
if (BrokerLauncherSettings.Default.AsConsole)
{
try
{
host = new LauncherHostService(true, context);
// This instance of HpcBroker is running as a failover generic application or in debug
// mode so startup the brokerManagement WCF service to accept management commands
brokerManagement = new BrokerManagement(host.BrokerLauncher);
brokerManagement.Open();
Console.WriteLine("Press any key to exit...");
Thread.Sleep(-1);
}
finally
{
if (host != null)
{
try
{
host.Stop();
}
catch (Exception e)
{
Trace.TraceError("Exception stopping HpcBroker service - " + e);
}
}
if (brokerManagement != null)
{
try
{
brokerManagement.Close();
}
catch (Exception e)
{
Trace.TraceError("Exception closing broker managment WCF service - " + e);
}
}
}
}
else
{
ServiceBase[] servicesToRun;
servicesToRun = new ServiceBase[] { new LauncherHostService(context) };
ServiceBase.Run(servicesToRun);
}
Log.CloseAndFlush();
}
/// <summary>
/// Run the broker
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="brokerInfo">indicate the broker start info</param>
/// <returns>initialization result</returns>
public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId);
try
{
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology);
// Step 1: Initialize configuration and shared data
ServiceConfiguration serviceConfig;
BrokerConfigurations brokerConfig;
BindingsSection bindings;
SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed.
ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings);
this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded.");
Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}");
// Step 2: Initialize broker queue
ClientInfo[] clientInfo;
this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded.");
// Step 3: Initialize observer
this.observer = new BrokerObserver(this.sharedData, clientInfo);
this.sharedData.Observer = this.observer;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded.");
// Step 4: Initialize state manager
this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0);
this.stateManager.UnloadBroker += this.UnloadBroker;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded.");
// Step 5: Initialize service job monitor
var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode);
if (SoaCommonConfig.WithoutSessionLayer)
{
this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
else
{
this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded.");
// Step 6: Initalize broker authorization
this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded.");
// Step 7: Initialize dispatcher manager
DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded.");
// Step 8: Start service job monitor
this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded.");
// Step 9: Initailize client manager
this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded.");
// if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist
string[] requestQueueUris = { };
string requestBlobUri = string.Empty;
string controllerRequestQueueUri = string.Empty;
string controllerResponseQueueUri = string.Empty;
if (startInfo.UseAzureStorage)
{
int clusterHash = 0;
if (!string.IsNullOrEmpty(brokerInfo.ClusterId))
{
string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant();
clusterHash = clusterIdString.GetHashCode();
}
else if (!string.IsNullOrEmpty(brokerInfo.ClusterName))
{
string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant();
clusterHash = clusterNameString.GetHashCode();
}
else
{
throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined.");
}
if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString))
{
this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString);
requestQueueUris = this.azureQueueProxy.RequestQueueUris;
requestBlobUri = this.azureQueueProxy.RequestBlobUri;
var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId);
var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId);
controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
requestQName,
CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult();
controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
responseQName,
CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult();
if (this.SessionId == SessionStartInfo.StandaloneSessionId)
{
CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName });
}
}
else
{
BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set.");
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable);
}
}
// Step 10: Initialize frontend
this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy);
////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize;
////this.readerQuotas = this.frontendResult.ReaderQuotas;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded.");
// Step 11: Start frontend, Initialization finished after this step
this.OpenFrontend();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded.");
// Step 12: Build initialization result and retrun to client
BrokerInitializationResult result = BrokerEntry.BuildInitializationResult(
this.frontendResult,
dispatcherManager,
this.sharedData.Config.LoadBalancing.ServiceOperationTimeout,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount,
requestQueueUris,
requestBlobUri,
controllerRequestQueueUri,
controllerResponseQueueUri,
startInfo.UseAzureStorage);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded.");
BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded.");
return(result);
}
catch (Exception ex)
{
BrokerTracing.TraceError(ex.ToString());
throw;
}
finally
{
if (this.sharedData != null)
{
this.sharedData.InitializationFinished();
}
}
}
private static void Main(string[] args)
{
var log = new LoggerConfiguration().ReadFrom.AppSettings().Enrich.WithMachineName().CreateLogger();
Log.Logger = log;
try
{
if (!ParseAndSetGlobalConfiguration(args))
{
// Parsing error
return;
}
}
catch (Exception e)
{
Trace.TraceError("Excepetion parsing and setting configuration - " + e);
Log.CloseAndFlush();
return;
}
if (SessionLauncherRuntimeConfiguration.ConfigureLogging)
{
Trace.TraceInformation("Log configuration for Session Launcher has done successfully.");
Log.CloseAndFlush();
return;
}
TelepathyContext.GetOrAdd();
LauncherHostService host = null;
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(CurrentDomain_UnhandledException);
if (SessionLauncherRuntimeConfiguration.AsConsole)
{
try
{
host = new LauncherHostService();
host.OpenService().Wait();
Console.WriteLine("Press any key to exit...");
Thread.Sleep(-1);
}
finally
{
if (host != null)
{
try
{
host.StopService();
}
catch (Exception e)
{
Trace.TraceError("Exception stopping service - " + e);
}
}
}
}
else
{
ServiceBase[] servicesToRun;
servicesToRun = new ServiceBase[] { new LauncherHostService() };
ServiceBase.Run(servicesToRun);
}
Log.CloseAndFlush();
}
/// <summary>
/// Start session launcher service
/// </summary>
private void StartSessionLauncherService()
{
try
{
string sessionLauncherAddress = SoaHelper.GetSessionLauncherAddress("localhost");
this.launcherHost = new ServiceHost(this.sessionLauncher, new Uri(sessionLauncherAddress));
BindingHelper.ApplyDefaultThrottlingBehavior(this.launcherHost);
#if AZURE_STORAGE_BINDING
if (SessionLauncherRuntimeConfiguration.OpenAzureStorageListener)
{
this.launcherHost.AddServiceEndpoint(
typeof(ISessionLauncher),
new TableTransportBinding()
{
ConnectionString = SessionLauncherRuntimeConfiguration.SessionLauncherStorageConnectionString, TargetPartitionKey = "all"
},
TelepathyConstants.SessionLauncherAzureTableBindingAddress);
TraceHelper.TraceEvent(TraceEventType.Information, "Add session launcher service endpoint {0}", TelepathyConstants.SessionLauncherAzureTableBindingAddress);
}
#endif
if (SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.HpcPack)
{
this.launcherHost.AddServiceEndpoint(typeof(ISessionLauncher), BindingHelper.HardCodedSessionLauncherNetTcpBinding, string.Empty);
this.launcherHost.AddServiceEndpoint(typeof(ISessionLauncher), BindingHelper.HardCodedNoAuthSessionLauncherNetTcpBinding, "AAD");
this.launcherHost.AddServiceEndpoint(typeof(ISessionLauncher), BindingHelper.HardCodedInternalSessionLauncherNetTcpBinding, "Internal");
TraceHelper.TraceEvent(TraceEventType.Information, "Open session launcher find cert {0}", TelepathyContext.Get().GetSSLThumbprint().GetAwaiter().GetResult());
this.launcherHost.Credentials.UseInternalAuthenticationAsync().GetAwaiter().GetResult();
TraceHelper.TraceEvent(TraceEventType.Information, "Add session launcher service endpoint {0}", sessionLauncherAddress);
}
else
{
this.launcherHost.AddServiceEndpoint(typeof(ISessionLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, string.Empty);
this.launcherHost.AddServiceEndpoint(typeof(ISessionLauncher), BindingHelper.HardCodedUnSecureNetTcpBinding, "Internal");
TraceHelper.TraceEvent(TraceEventType.Information, "Add session launcher service endpoint {0}", sessionLauncherAddress);
}
this.launcherHost.Faulted += this.SessionLauncherHostFaultHandler;
ServiceAuthorizationBehavior myServiceBehavior =
this.launcherHost.Description.Behaviors.Find <ServiceAuthorizationBehavior>();
myServiceBehavior.PrincipalPermissionMode = PrincipalPermissionMode.None;
this.launcherHost.Open();
TraceHelper.TraceEvent(TraceEventType.Information, "Open session launcher service");
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
throw;
}
}
