/// <summary>
/// Poll the scheduler for the newly published graceful preemption info.
/// </summary>
/// <param name="schedulerAdapter">the scheduler adapter</param>
public async Task <bool> RefreshGracefulPreemptionInfo(ISchedulerAdapter schedulerAdapter)
{
BrokerTracing.TraceVerbose(
"[GracefulPreemptionHandler].RefreshPreemptionRequest: query scheduler adapter");
bool result;
List <string> taskIds;
List <string> runningTaskIds;
BalanceInfo balanceInfo;
(result, balanceInfo, taskIds, runningTaskIds) = await schedulerAdapter.GetGracefulPreemptionInfoAsync(this.sessionId);
if (result)
{
BrokerTracing.TraceVerbose(
"[GracefulPreemptionHandler].RefreshPreemptionRequest: get new request taskIds count = {0}, running taskIds count = {1}, maxAllowedCoreCount = {2}, fastBalance = {3}",
taskIds.Count,
runningTaskIds.Count,
balanceInfo.AllowedCoreCount,
balanceInfo.UseFastBalance);
BrokerTracing.TraceVerbose(
"[GracefulPreemptionHandler].RefreshPreemptionRequest: replace old request taskIds count = {0}, maxAllowedCoreCount = {1}, fastBalance = {2}",
this.exitingTaskIds == null ? 0 : this.exitingTaskIds.Count,
this.BalanceInfo.AllowedCoreCount,
this.BalanceInfo.UseFastBalance);
foreach (var request in balanceInfo.BalanceRequests)
{
if (request.AllowedCoreCount <= NoPreemptionCores)
{
request.AllowedCoreCount = int.MaxValue;
}
}
this.BalanceInfo = balanceInfo;
this.exitingTaskIds = new HashSet <string>(taskIds);
this.runningTaskIds = new HashSet <string>(runningTaskIds);
return(true);
}
else
{
BrokerTracing.TraceVerbose(
"[GracefulPreemptionHandler].RefreshPreemptionRequest: Refresh failed.");
return(false);
}
}
/// <summary>
/// Update the broker info
/// </summary>
/// <param name="info">broker info</param>
private async Task UpdateBrokerInfo(BrokerStartInfo info)
{
int retry = 3;
Dictionary <string, object> properties = new Dictionary <string, object>();
properties.Add(BrokerSettingsConstants.BrokerNode, Constant.InprocessBrokerNode);
properties.Add(BrokerSettingsConstants.Suspended, info.Durable);
properties.Add(BrokerSettingsConstants.Durable, info.Durable);
properties.Add(BrokerSettingsConstants.PersistVersion, info.PersistVersion);
//SchedulerAdapterInternalClient client = new SchedulerAdapterInternalClient(await this.startInfo.ResolveHeadnodeMachineAsync().ConfigureAwait(false));
ISchedulerAdapter client = SessionServiceContainer.SchedulerAdapterInstance;
try
{
while (retry > 0)
{
try
{
if (await client.UpdateBrokerInfoAsync(info.SessionId, properties).ConfigureAwait(false))
{
return;
}
}
catch (Exception ex)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[BrokerLauncher.SchedulerHelper] UpdateBrokerInfo failed: Exception = {0}", ex);
}
retry--;
}
}
finally
{
var communicateObj = client as ICommunicationObject;
if (communicateObj != null)
{
Utility.SafeCloseCommunicateObject(communicateObj);
}
}
throw new InvalidOperationException("Can not update the properties in the scheduler database for EPRs");
}
/// <summary>
/// Create client
/// </summary>
private async Task CreateClient()
{
BrokerTracing.TraceVerbose(
"[SchedulerAdapterClientFactory] Creating client with StartInfo.IpAddress: {0}, StartInfo.EprList: {1}, WithoutSessionLayer: {2}",
this.sharedData.StartInfo.IpAddress != null,
this.sharedData.StartInfo.EprList != null,
SoaCommonConfig.WithoutSessionLayer);
if (SoaCommonConfig.WithoutSessionLayer)
{
this.schedulerAdapterClient = new DummySchedulerAdapterClient(this.sharedData.StartInfo.IpAddress, this.dispatcherManager);
}
else
{
// TODO: implementing new authentication logic between brokerworker and sessionlauncher
string headnodeMachine = await this.context.ResolveSessionLauncherNodeAsync();
// certThrumbprint = await this.context.GetSSLThumbprint();
// if (this.monitor.TransportScheme == TransportScheme.AzureStorage)
// {
// this.schedulerAdapterClient = new SchedulerAdapterClient(
// new TableTransportBinding() { ConnectionString = this.monitor.SharedData.BrokerInfo.AzureStorageConnectionString, TargetPartitionKey = Guid.NewGuid().ToString() },
// new EndpointAddress(new Uri(TelepathyConstants.SessionSchedulerDelegationAzureTableBindingAddress)),
// this.sharedData.StartInfo.IpAddress,
// this.dispatcherManager);
// }
// else
// {
// this.schedulerAdapterClient = new HpcSchedulerAdapterClient(headnodeMachine, certThrumbprint, new System.ServiceModel.InstanceContext(this.monitor));
this.schedulerAdapterClient = new SchedulerAdapterClient(
BindingHelper.HardCodedUnSecureNetTcpBinding,
new EndpointAddress(new Uri(SoaHelper.GetSchedulerDelegationAddress(headnodeMachine))),
this.sharedData.StartInfo.IpAddress,
this.dispatcherManager,
new System.ServiceModel.InstanceContext(this.monitor));
// }
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="connectionPool">Connection pool</param>
/// <param name="adapter">Scheduler adapter</param>
public RexSchedulerWrapper(IConnectionPool connectionPool, ISchedulerAdapter adapter)
{
_log = LogManager.GetLogger(typeof(RexSchedulerWrapper));
_connectionPool = connectionPool;
_adapter = adapter;
}
/// <summary>
/// Initializes a new instance of the LauncherHostService class
/// This will be called when run in debug mode
/// </summary>
/// <param name="launcherChoose">launcher choose</param>
//internal LauncherHostService(bool console)
//{
// InitializeComponent();
// this.OpenService();
//}
public void StopService(bool isNtService = false)
{
try
{
// DataService instance is created by session launcher. To make sure data service exit gracefully, close DataService before closing session launcher.
if (this.dataServiceHost != null)
{
this.dataServiceHost.Faulted -= DataServiceHostFaultHandler;
this.dataServiceHost.Close();
this.dataServiceHost = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "SOA data service endpoint closed");
}
if (this.launcherHost != null)
{
this.launcherHost.Close();
}
this.launcherHost = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "Session launcher endpoint closed");
this.delegationHost?.Close();
this.delegationHost = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "Scheduler delegation service closed");
#if HPCPACK
// session launcher host has been closed, remember to close the data service instance
if (this.dataService != null)
{
this.dataService.Close();
this.dataService = null;
TraceHelper.TraceEvent(TraceEventType.Verbose, "Data service instance closed");
}
#endif
if (!isNtService)
{
// only need to get cleaned in SF serivce
if (this.schedulerDelegation is IDisposable disposable)
{
disposable.Dispose();
TraceHelper.TraceEvent(TraceEventType.Verbose, "Scheduler delegation closed");
}
this.schedulerDelegation = null;
if (this.sessionLauncher != null)
{
this.sessionLauncher.Close();
TraceHelper.TraceEvent(TraceEventType.Verbose, "Session launcher closed");
}
this.sessionLauncher = null;
TraceHelper.IsDiagTraceEnabled = null;
SoaDiagTraceHelper.IsDiagTraceEnabledInternal = null;
if (this.brokerNodesManager is IDisposable d)
{
d.Dispose();
}
this.brokerNodesManager = null;
}
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Error, "Failed to close the service host - {0}", e);
}
}
/// <summary>
/// Open the session launcher service
/// </summary>
public async Task OpenService()
{
try
{
// If running on Azure, monitor HAController service and terminate this service
// if HAController dies. Note that SCM service dependency monitoring does not provide
// this
if (SoaHelper.IsOnAzure())
{
ServiceControllerHelpers.MonitorHAControllerStopAsync(HpcServiceNames.HpcSession);
TraceHelper.TraceEvent(TraceEventType.Information, "Azure HAController service monitoring enabled");
}
if (SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.HpcPack)
{
#if HPCPACK
this.brokerNodesManager = new BrokerNodesManager();
this.sessionLauncher = SessionLauncherFactory.CreateHpcPackSessionLauncher(SoaHelper.GetSchedulerName(), false, this.brokerNodesManager);
this.schedulerDelegation = new HpcSchedulerDelegation(this.sessionLauncher, this.brokerNodesManager);
#endif
}
else if (SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.AzureBatch)
{
var instance = SessionLauncherFactory.CreateAzureBatchSessionLauncher();
this.sessionLauncher = instance;
this.schedulerDelegation = new AzureBatchSchedulerDelegation(instance);
}
else if (SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.Local)
{
var instance = SessionLauncherFactory.CreateLocalSessionLauncher();
this.sessionLauncher = instance;
this.schedulerDelegation = new LocalSchedulerDelegation(instance);
}
TraceHelper.IsDiagTraceEnabled = _ => true;
#if HPCPACK
// Bug 18448: Need to enable traces only for those who have enabled trace
if (this.schedulerDelegation is IHpcSchedulerAdapterInternal hpcAdapterInternal)
{
SoaDiagTraceHelper.IsDiagTraceEnabledInternal = hpcAdapterInternal.IsDiagTraceEnabled;
TraceHelper.IsDiagTraceEnabled = SoaDiagTraceHelper.IsDiagTraceEnabled;
}
#endif
// start session launcher service
this.StartSessionLauncherService();
if (SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.HpcPack ||
SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.Local ||
SessionLauncherRuntimeConfiguration.SchedulerType == SchedulerType.AzureBatch)
{
// start scheduler delegation service
this.StartSchedulerDelegationService();
}
#if HPCPACK
// start data service
if (!SoaHelper.IsOnAzure() && this.sessionLauncher is HpcPackSessionLauncher hpcSessionLauncher)
{
this.dataService = hpcSessionLauncher.GetDataService();
this.StartDataWcfService();
this.StartDataRestService(this.dataService);
}
#endif
}
catch (Exception e)
{
TraceHelper.TraceEvent(TraceEventType.Critical, "Failed to open the service host - {0}", e);
throw;
}
await Task.CompletedTask;
}
