/// <summary>
/// Prepares broker launcher client for heart beat with timeout specific
/// </summary>
/// <param name="brokerLauncherRef">indicating the reference to broker launcher</param>
/// <param name="timeout">indicating the timeout</param>
private void PrepareBrokerLauncherClient(ref IBrokerLauncher brokerLauncherRef, int timeoutMilliseconds)
{
if (brokerLauncherRef == null)
{
lock (this.lockCreateChannel)
{
if (brokerLauncherRef == null)
{
brokerLauncherRef = new BrokerLauncherClient(this.uri, this.info, this.binding);
}
}
}
if (brokerLauncherRef is ClientBase <IBrokerLauncher> )
{
ClientBase <IBrokerLauncher> broker = (ClientBase <IBrokerLauncher>)brokerLauncherRef;
if (timeoutMilliseconds == Timeout.Infinite)
{
broker.InnerChannel.OperationTimeout = TimeSpan.MaxValue;
}
else
{
broker.InnerChannel.OperationTimeout = SessionBase.GetTimeout(DateTime.Now.AddMilliseconds(timeoutMilliseconds));
}
}
}
/// <summary>
/// Initializes a new instance of the BrokerLauncherClientFactory class
/// </summary>
/// <param name="info">indicating the session info</param>
/// <param name="binding">indicating the binding</param>
public BrokerLauncherClientFactory(SessionInfoBase info, Binding binding)
{
this.info = info;
this.binding = binding;
Debug.Assert(info is SessionInfo);
#if DEBUG
this.allowsHeartbeat = true;
#endif
SessionInfo sessionInfo = (SessionInfo)info;
if (info.UseInprocessBroker)
{
this.brokerLauncher = sessionInfo.InprocessBrokerAdapter;
this.brokerLauncherForHeartbeat = sessionInfo.InprocessBrokerAdapter;
}
else if (sessionInfo.BrokerLauncherEpr == SessionInternalConstants.BrokerConnectionStringToken)
{
Trace.TraceInformation($"[{nameof(BrokerLauncherClientFactory)}] will not connect to frontend as EPR is {nameof(SessionInternalConstants.BrokerConnectionStringToken)}.");
}
else
{
this.uri = new Uri(sessionInfo.BrokerLauncherEpr);
this.heartbeatInterval = sessionInfo.ClientBrokerHeartbeatInterval;
}
// else
// {
// // info is WebSessionInfo
// WebSessionInfo webSessionInfo = (WebSessionInfo)info;
// this.brokerLauncher = new WebBrokerLauncherClient(webSessionInfo.HeadNode, webSessionInfo.Credential);
// #if DEBUG
// this.allowsHeartbeat = false;
// #endif
// }
}
internal BrokerLauncherCloudQueueWatcher(IBrokerLauncher instance, IQueueListener <CloudQueueCmdDto> listener, IQueueWriter <CloudQueueResponseDto> writer)
{
this.instance = instance;
this.QueueListener = listener;
this.QueueWriter = writer;
this.QueueListener.MessageReceivedCallback = this.InvokeInstanceMethodFromCmdObj;
this.RegisterCmdDelegates();
}
/// <summary>
/// Close the broker controller client so it can be recreated for next ping
/// </summary>
public void CloseBrokerLauncherClientForHeartbeat()
{
lock (this.lockCreateChannel)
{
if (this.brokerLauncherForHeartbeat != null && this.brokerLauncherForHeartbeat is ICommunicationObject)
{
Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncherForHeartbeat);
this.brokerLauncherForHeartbeat = null;
}
}
}
internal BrokerLauncherCloudQueueWatcher(IBrokerLauncher instance, string connectionString)
{
this.instance = instance;
CloudQueueSerializer serializer = new CloudQueueSerializer(CloudQueueCmdTypeBinder.BrokerLauncherBinder);
this.QueueListener = new CloudQueueListener <CloudQueueCmdDto>(connectionString, CloudQueueConstants.BrokerLauncherRequestQueueName, serializer, this.InvokeInstanceMethodFromCmdObj);
this.QueueWriter = new CloudQueueWriter <CloudQueueResponseDto>(connectionString, CloudQueueConstants.BrokerLauncherResponseQueueName, serializer);
this.QueueListener.StartListen();
this.RegisterCmdDelegates();
Trace.TraceInformation("BrokerLauncherCloudQueueWatcher started.");
}
/// <summary>
/// Dispose the BrokerLauncherClientFactory instance
/// </summary>
/// <param name="disposing">indicating whether it is disposing</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (this.brokerLauncher != null && this.brokerLauncher is ICommunicationObject)
{
Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncher);
this.brokerLauncher = null;
}
if (this.brokerLauncherForHeartbeat != null && this.brokerLauncherForHeartbeat is ICommunicationObject)
{
Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncherForHeartbeat);
this.brokerLauncherForHeartbeat = null;
}
}
base.Dispose(disposing);
}
/// <summary>
/// Dispose the service job if autoclose is true
/// </summary>
/// <param name="disposing">if we called from destructor</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
// Nothing to do here
}
// Auto close the jobs if the autoCloseJob is set to true
if (this.autoCloseJob)
{
try
{
IBrokerLauncher broker = this.BrokerLauncherClientFactory.GetBrokerLauncherClient(-1);
broker.Close(this.Id);
}
catch (Exception e)
{
// Swallow the exception
SessionBase.TraceSource.TraceInformation(e.ToString());
}
}
base.Dispose(disposing);
}
/// <summary>
/// Create broker
/// </summary>
/// <param name="startInfo">indicating the session start information</param>
/// <param name="sessionId">indicating the session id</param>
/// <param name="targetTimeout">indicating the target timeout</param>
/// <param name="eprs">indicating the broker epr list</param>
/// <param name="epr">output selected epr</param>
/// <param name="binding">indicting the binding</param>
/// <returns>returns the session information</returns>
public async Task <SessionBase> CreateBroker(SessionStartInfo startInfo, string sessionId, DateTime targetTimeout, string[] eprs, Binding binding)
{
Exception innerException = null;
IEnumerable <string> endpoints = eprs;
if (startInfo.UseAzureQueue.GetValueOrDefault() && !endpoints.Contains(SessionInternalConstants.BrokerConnectionStringToken))
{
endpoints = endpoints.Concat(new[] { SessionInternalConstants.BrokerConnectionStringToken });
}
foreach (string epr in endpoints)
{
TimeSpan timeout = SessionBase.GetTimeout(targetTimeout);
IBrokerLauncher brokerLauncher = null;
try
{
SessionBase.TraceSource.TraceInformation("[Session:{0}] Try to create broker... BrokerLauncherEpr = {1}", sessionId, epr);
void RenewBrokerLauncherClient()
{
if (epr == SessionInternalConstants.BrokerConnectionStringToken)
{
brokerLauncher = new BrokerLauncherCloudQueueClient(startInfo.BrokerLauncherStorageConnectionString);
}
else
{
var client = new BrokerLauncherClient(new Uri(epr), startInfo, binding);
client.InnerChannel.OperationTimeout = timeout;
brokerLauncher = client;
}
}
RenewBrokerLauncherClient();
BrokerInitializationResult result = null;
int retry = 20;
while (retry > 0)
{
try
{
if (this.durable)
{
result = brokerLauncher.CreateDurable(startInfo.Data, sessionId);
}
else
{
result = brokerLauncher.Create(startInfo.Data, sessionId);
}
break;
}
catch (Exception ex)
{
if (retry <= 0)
{
throw;
}
retry--;
Debug.WriteLine($"Waiting for Broker Launcher running. Detail: {ex.Message}");
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
RenewBrokerLauncherClient();
}
}
Debug.Assert(result != null);
SessionBase.TraceSource.TraceInformation("[Session:{0}] Succesfully created broker.", sessionId);
SessionInfo info = SessionBase.BuildSessionInfo(result, this.durable, sessionId, epr, startInfo.Data.ServiceVersion, startInfo);
if (this.durable)
{
#if net40
return(new DurableSession(info, startInfo.Headnode, binding));
#else
return(new DurableSession(info, startInfo.Headnode, binding));
#endif
}
else
{
var session = new V3Session(info, startInfo.Headnode, startInfo.ShareSession, binding);
if (startInfo.UseAzureStorage)
{
session.BrokerLauncherClient = brokerLauncher;
}
return(session);
}
}
catch (FaultException <SessionFault> e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Fault exception occured while creating broker: {1}. FaultCode = {2}", sessionId, e, e.Detail.Code);
switch (e.Detail.Code)
{
// Continue if current broker node is being taken offline
case SOAFaultCode.Broker_BrokerIsOffline:
continue;
}
throw Utility.TranslateFaultException(e);
}
catch (TimeoutException te)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] TimeoutException occured while creating broker: {1}", sessionId, te);
// don't continue when we timeout
throw new TimeoutException(string.Format(SR.ConectBrokerLauncherTimeout, epr, Constant.DefaultCreateSessionTimeout), te);
}
catch (CommunicationException ex)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Failed to create broker: {1}", sessionId, ex);
innerException = ex;
SessionBase.TraceSource.TraceInformation(ex.ToString());
continue;
}
finally
{
var client = brokerLauncher as BrokerLauncherClient;
if (client != null)
{
Utility.SafeCloseCommunicateObject(client);
}
}
}
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] Failed to create broker after trying all available eprs.", sessionId);
throw new SessionException(SR.NoBrokerNodeFound, innerException);
}
/// <summary>
/// Timer callback for heartbeat
/// </summary>
/// <param name="state">unused state param</param>
private void RunHeartbeat(object state)
{
// If the heartbeat interval needs to be ignored, do so now and return
if (this.ignoreNextHeartbeatInterval)
{
this.ignoreNextHeartbeatInterval = false;
return;
}
// If the timer is closed, return
if (this.heartbeatTimer == null)
{
return;
}
bool isBrokerLoaded = false;
bool pingSucceeded = false;
try
{
// Connect to the broker launcher
IBrokerLauncher launcher = this.factory.GetBrokerLauncherClientForHeartbeat();
if (this.supportNewPing)
{
try
{
string result = launcher.PingBroker2(this.sessionId);
#if API
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BrokerHeartbeatHelper] Ping succeeded. Result = {0}, ExpectedResult = {1}", result, this.expectPingResult);
#endif
pingSucceeded = true;
if (this.expectPingResult == null)
{
this.expectPingResult = result;
}
if (result == Constant.PingBroker2Result_BrokerNotExist ||
result != this.expectPingResult)
{
isBrokerLoaded = false;
}
else
{
isBrokerLoaded = true;
}
}
catch (ActionNotSupportedException)
{
if (this.expectPingResult == null)
{
this.supportNewPing = false;
}
else
{
throw;
}
}
}
if (!this.supportNewPing)
{
// Ping it
isBrokerLoaded = launcher.PingBroker(this.sessionId);
pingSucceeded = true;
}
}
catch (Exception e)
{
#if API
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[BrokerHeartbeatHelper] Exception occured when pinging broker: {0}", e);
#elif WebAPI
Microsoft.Hpc.RuntimeTrace.TraceHelper.TraceInfo(this.sessionId, "[BrokerHeartbeatHelper] Exception occured when pinging broker: {0}", e);
#endif
// If the max missed heartbeats have been hit
if (++this.missedHeartbeats == this.clientBrokerHeartbeatRetryCount)
{
// Signal the broker is down
this.SendBrokerDownSignal(true);
// Shutdown the heartbeat
this.Stop();
}
else
{
// Recreate the broker controller client so it can be used for next ping
this.factory.CloseBrokerLauncherClientForHeartbeat();
}
}
// If ping call succeeded
if (pingSucceeded)
{
// If broker isnt loaded
if (!isBrokerLoaded)
{
// Signal the broker is down
this.SendBrokerDownSignal(false);
// Shutdown the heartbeat
this.Stop();
}
else
{
// Suceeded so reset missed heartbeat count
this.missedHeartbeats = 0;
}
}
}
