/// <summary> /// Prepares broker launcher client for heart beat with timeout specific /// </summary> /// <param name="brokerLauncherRef">indicating the reference to broker launcher</param> /// <param name="timeout">indicating the timeout</param> private void PrepareBrokerLauncherClient(ref IBrokerLauncher brokerLauncherRef, int timeoutMilliseconds) { if (brokerLauncherRef == null) { lock (this.lockCreateChannel) { if (brokerLauncherRef == null) { brokerLauncherRef = new BrokerLauncherClient(this.uri, this.info, this.binding); } } } if (brokerLauncherRef is ClientBase <IBrokerLauncher> ) { ClientBase <IBrokerLauncher> broker = (ClientBase <IBrokerLauncher>)brokerLauncherRef; if (timeoutMilliseconds == Timeout.Infinite) { broker.InnerChannel.OperationTimeout = TimeSpan.MaxValue; } else { broker.InnerChannel.OperationTimeout = SessionBase.GetTimeout(DateTime.Now.AddMilliseconds(timeoutMilliseconds)); } } }
/// <summary> /// Initializes a new instance of the BrokerLauncherClientFactory class /// </summary> /// <param name="info">indicating the session info</param> /// <param name="binding">indicating the binding</param> public BrokerLauncherClientFactory(SessionInfoBase info, Binding binding) { this.info = info; this.binding = binding; Debug.Assert(info is SessionInfo); #if DEBUG this.allowsHeartbeat = true; #endif SessionInfo sessionInfo = (SessionInfo)info; if (info.UseInprocessBroker) { this.brokerLauncher = sessionInfo.InprocessBrokerAdapter; this.brokerLauncherForHeartbeat = sessionInfo.InprocessBrokerAdapter; } else if (sessionInfo.BrokerLauncherEpr == SessionInternalConstants.BrokerConnectionStringToken) { Trace.TraceInformation($"[{nameof(BrokerLauncherClientFactory)}] will not connect to frontend as EPR is {nameof(SessionInternalConstants.BrokerConnectionStringToken)}."); } else { this.uri = new Uri(sessionInfo.BrokerLauncherEpr); this.heartbeatInterval = sessionInfo.ClientBrokerHeartbeatInterval; } // else // { // // info is WebSessionInfo // WebSessionInfo webSessionInfo = (WebSessionInfo)info; // this.brokerLauncher = new WebBrokerLauncherClient(webSessionInfo.HeadNode, webSessionInfo.Credential); // #if DEBUG // this.allowsHeartbeat = false; // #endif // } }
internal BrokerLauncherCloudQueueWatcher(IBrokerLauncher instance, IQueueListener <CloudQueueCmdDto> listener, IQueueWriter <CloudQueueResponseDto> writer) { this.instance = instance; this.QueueListener = listener; this.QueueWriter = writer; this.QueueListener.MessageReceivedCallback = this.InvokeInstanceMethodFromCmdObj; this.RegisterCmdDelegates(); }
/// <summary> /// Close the broker controller client so it can be recreated for next ping /// </summary> public void CloseBrokerLauncherClientForHeartbeat() { lock (this.lockCreateChannel) { if (this.brokerLauncherForHeartbeat != null && this.brokerLauncherForHeartbeat is ICommunicationObject) { Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncherForHeartbeat); this.brokerLauncherForHeartbeat = null; } } }
internal BrokerLauncherCloudQueueWatcher(IBrokerLauncher instance, string connectionString) { this.instance = instance; CloudQueueSerializer serializer = new CloudQueueSerializer(CloudQueueCmdTypeBinder.BrokerLauncherBinder); this.QueueListener = new CloudQueueListener <CloudQueueCmdDto>(connectionString, CloudQueueConstants.BrokerLauncherRequestQueueName, serializer, this.InvokeInstanceMethodFromCmdObj); this.QueueWriter = new CloudQueueWriter <CloudQueueResponseDto>(connectionString, CloudQueueConstants.BrokerLauncherResponseQueueName, serializer); this.QueueListener.StartListen(); this.RegisterCmdDelegates(); Trace.TraceInformation("BrokerLauncherCloudQueueWatcher started."); }
/// <summary> /// Dispose the BrokerLauncherClientFactory instance /// </summary> /// <param name="disposing">indicating whether it is disposing</param> protected override void Dispose(bool disposing) { if (disposing) { if (this.brokerLauncher != null && this.brokerLauncher is ICommunicationObject) { Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncher); this.brokerLauncher = null; } if (this.brokerLauncherForHeartbeat != null && this.brokerLauncherForHeartbeat is ICommunicationObject) { Utility.SafeCloseCommunicateObject((ICommunicationObject)this.brokerLauncherForHeartbeat); this.brokerLauncherForHeartbeat = null; } } base.Dispose(disposing); }
/// <summary> /// Dispose the service job if autoclose is true /// </summary> /// <param name="disposing">if we called from destructor</param> protected override void Dispose(bool disposing) { if (disposing) { // Nothing to do here } // Auto close the jobs if the autoCloseJob is set to true if (this.autoCloseJob) { try { IBrokerLauncher broker = this.BrokerLauncherClientFactory.GetBrokerLauncherClient(-1); broker.Close(this.Id); } catch (Exception e) { // Swallow the exception SessionBase.TraceSource.TraceInformation(e.ToString()); } } base.Dispose(disposing); }
/// <summary> /// Create broker /// </summary> /// <param name="startInfo">indicating the session start information</param> /// <param name="sessionId">indicating the session id</param> /// <param name="targetTimeout">indicating the target timeout</param> /// <param name="eprs">indicating the broker epr list</param> /// <param name="epr">output selected epr</param> /// <param name="binding">indicting the binding</param> /// <returns>returns the session information</returns> public async Task <SessionBase> CreateBroker(SessionStartInfo startInfo, string sessionId, DateTime targetTimeout, string[] eprs, Binding binding) { Exception innerException = null; IEnumerable <string> endpoints = eprs; if (startInfo.UseAzureQueue.GetValueOrDefault() && !endpoints.Contains(SessionInternalConstants.BrokerConnectionStringToken)) { endpoints = endpoints.Concat(new[] { SessionInternalConstants.BrokerConnectionStringToken }); } foreach (string epr in endpoints) { TimeSpan timeout = SessionBase.GetTimeout(targetTimeout); IBrokerLauncher brokerLauncher = null; try { SessionBase.TraceSource.TraceInformation("[Session:{0}] Try to create broker... BrokerLauncherEpr = {1}", sessionId, epr); void RenewBrokerLauncherClient() { if (epr == SessionInternalConstants.BrokerConnectionStringToken) { brokerLauncher = new BrokerLauncherCloudQueueClient(startInfo.BrokerLauncherStorageConnectionString); } else { var client = new BrokerLauncherClient(new Uri(epr), startInfo, binding); client.InnerChannel.OperationTimeout = timeout; brokerLauncher = client; } } RenewBrokerLauncherClient(); BrokerInitializationResult result = null; int retry = 20; while (retry > 0) { try { if (this.durable) { result = brokerLauncher.CreateDurable(startInfo.Data, sessionId); } else { result = brokerLauncher.Create(startInfo.Data, sessionId); } break; } catch (Exception ex) { if (retry <= 0) { throw; } retry--; Debug.WriteLine($"Waiting for Broker Launcher running. Detail: {ex.Message}"); await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); RenewBrokerLauncherClient(); } } Debug.Assert(result != null); SessionBase.TraceSource.TraceInformation("[Session:{0}] Succesfully created broker.", sessionId); SessionInfo info = SessionBase.BuildSessionInfo(result, this.durable, sessionId, epr, startInfo.Data.ServiceVersion, startInfo); if (this.durable) { #if net40 return(new DurableSession(info, startInfo.Headnode, binding)); #else return(new DurableSession(info, startInfo.Headnode, binding)); #endif } else { var session = new V3Session(info, startInfo.Headnode, startInfo.ShareSession, binding); if (startInfo.UseAzureStorage) { session.BrokerLauncherClient = brokerLauncher; } return(session); } } catch (FaultException <SessionFault> e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Fault exception occured while creating broker: {1}. FaultCode = {2}", sessionId, e, e.Detail.Code); switch (e.Detail.Code) { // Continue if current broker node is being taken offline case SOAFaultCode.Broker_BrokerIsOffline: continue; } throw Utility.TranslateFaultException(e); } catch (TimeoutException te) { SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] TimeoutException occured while creating broker: {1}", sessionId, te); // don't continue when we timeout throw new TimeoutException(string.Format(SR.ConectBrokerLauncherTimeout, epr, Constant.DefaultCreateSessionTimeout), te); } catch (CommunicationException ex) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Failed to create broker: {1}", sessionId, ex); innerException = ex; SessionBase.TraceSource.TraceInformation(ex.ToString()); continue; } finally { var client = brokerLauncher as BrokerLauncherClient; if (client != null) { Utility.SafeCloseCommunicateObject(client); } } } SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] Failed to create broker after trying all available eprs.", sessionId); throw new SessionException(SR.NoBrokerNodeFound, innerException); }
/// <summary> /// Timer callback for heartbeat /// </summary> /// <param name="state">unused state param</param> private void RunHeartbeat(object state) { // If the heartbeat interval needs to be ignored, do so now and return if (this.ignoreNextHeartbeatInterval) { this.ignoreNextHeartbeatInterval = false; return; } // If the timer is closed, return if (this.heartbeatTimer == null) { return; } bool isBrokerLoaded = false; bool pingSucceeded = false; try { // Connect to the broker launcher IBrokerLauncher launcher = this.factory.GetBrokerLauncherClientForHeartbeat(); if (this.supportNewPing) { try { string result = launcher.PingBroker2(this.sessionId); #if API SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[BrokerHeartbeatHelper] Ping succeeded. Result = {0}, ExpectedResult = {1}", result, this.expectPingResult); #endif pingSucceeded = true; if (this.expectPingResult == null) { this.expectPingResult = result; } if (result == Constant.PingBroker2Result_BrokerNotExist || result != this.expectPingResult) { isBrokerLoaded = false; } else { isBrokerLoaded = true; } } catch (ActionNotSupportedException) { if (this.expectPingResult == null) { this.supportNewPing = false; } else { throw; } } } if (!this.supportNewPing) { // Ping it isBrokerLoaded = launcher.PingBroker(this.sessionId); pingSucceeded = true; } } catch (Exception e) { #if API SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[BrokerHeartbeatHelper] Exception occured when pinging broker: {0}", e); #elif WebAPI Microsoft.Hpc.RuntimeTrace.TraceHelper.TraceInfo(this.sessionId, "[BrokerHeartbeatHelper] Exception occured when pinging broker: {0}", e); #endif // If the max missed heartbeats have been hit if (++this.missedHeartbeats == this.clientBrokerHeartbeatRetryCount) { // Signal the broker is down this.SendBrokerDownSignal(true); // Shutdown the heartbeat this.Stop(); } else { // Recreate the broker controller client so it can be used for next ping this.factory.CloseBrokerLauncherClientForHeartbeat(); } } // If ping call succeeded if (pingSucceeded) { // If broker isnt loaded if (!isBrokerLoaded) { // Signal the broker is down this.SendBrokerDownSignal(false); // Shutdown the heartbeat this.Stop(); } else { // Suceeded so reset missed heartbeat count this.missedHeartbeats = 0; } } }