/// <summary>
/// Initializes a new instance of the PullResponsesHandler class
/// </summary>
/// <param name="queue">indicating the broker queue</param>
/// <param name="action">indicating the action</param>
/// <param name="timeoutManager">indicating the timeout manager</param>
/// <param name="observer">indicating the observer</param>
/// <param name="sharedData">indicating the shared data</param>
/// <param name="version">indicating the message version</param>
public PullResponsesHandler(BrokerQueue queue, string action, TimeoutManager timeoutManager, BrokerObserver observer, SharedData sharedData, MessageVersion version)
: base(queue, action, timeoutManager, observer, sharedData, version)
{
}
/// <summary>
/// Initializes a new instance of the GetResponsesHandler class
/// </summary>
/// <param name="queue">indicating the broker queue</param>
/// <param name="action">indicating the action</param>
/// <param name="clientData">indicating the client data</param>
/// <param name="clientId">indicating the client</param>
/// <param name="timeoutManager">indicating the timeout manager</param>
/// <param name="observer">indicating the observer</param>
/// <param name="sharedData">indicating the shared data</param>
/// <param name="version">indicating the message version</param>
public GetResponsesHandler(BrokerQueue queue, string action, string clientData, string clientId, TimeoutManager timeoutManager, BrokerObserver observer, SharedData sharedData, MessageVersion version)
: base(queue, action, timeoutManager, observer, sharedData, version)
{
this.clientId = clientId;
this.clientData = clientData;
// For now, as REST service is the only frontend on Azure. Broker can know if it is
// REST service connecting by checking if broker is running on Azure.
// Need to fix this logic if we reopen net.tcp frontend on Azure.
if (SoaHelper.IsOnAzure())
{
this.cacheBrokerQueueItem = true;
}
#if DEBUG
// For debug purpose, if the incoming message header contains a header which
// indicates it is REST service calling, also set the flag to true
// This is to enable on-premise test.
if (OperationContext.Current != null && OperationContext.Current.IncomingMessageHeaders.FindHeader("HpcSOAWebSvc", Constant.HpcHeaderNS) >= 0)
{
this.cacheBrokerQueueItem = true;
}
#endif
}
/// <summary>
/// Initializes a new instance of the PullResponseMessage class
/// </summary>
/// <param name="count">indicating the count</param>
/// <param name="queue">indicating the queue</param>
/// <param name="convertMessage">indicating the convert message delegate</param>
/// <param name="resetTimeout">indicating the reset timeout delegate</param>
/// <param name="sharedData">indicating the shared data</param>
public PullResponseMessage(int count, BrokerQueue queue, ConvertMessageDelegate convertMessage, ResetTimeoutDelegate resetTimeout, SharedData sharedData)
{
this.count = count;
this.queue = queue;
this.convertMessage = convertMessage;
this.resetTimeout = resetTimeout;
this.sharedData = sharedData;
}
/// <summary>
/// Initializes a new instance of the BrokerClient class
/// </summary>
/// <param name="clientId">indicating the client Id</param>
/// <param name="userName">indicating the user name</param>
/// <param name="queueFactory">indicating the queue factory</param>
/// <param name="observer">indicating the observer</param>
/// <param name="stateManager">indicating the state manager</param>
/// <param name="monitor">indicating the monitor</param>
/// <param name="sharedData">indicating the shared data</param>
public BrokerClient(string clientId, string userName, BrokerQueueFactory queueFactory, BrokerObserver observer, BrokerStateManager stateManager, ServiceJobMonitorBase monitor, SharedData sharedData)
{
bool isNewCreated;
// Set the "signletonInstanceConnected" property if
// SessionStartInfo.AutoDisposeBrokerClient is set. This property is only possibly to
// be set to true after if HTTP transport scheme is specified. And it is by design so.
if (sharedData.StartInfo.AutoDisposeBrokerClient.HasValue)
{
this.singletonInstanceConnected = !sharedData.StartInfo.AutoDisposeBrokerClient.Value;
}
this.clientId = clientId;
this.v2ProxyClient = clientId.StartsWith(FrontEndBase.DefaultClientPrefix);
this.sharedData = sharedData;
this.observer = observer;
this.monitor = monitor;
this.stateManager = stateManager;
this.stateManager.OnFailed += new BrokerStateManager.SessionFailedEventHandler(this.StateManager_OnFailed);
try
{
this.queue = queueFactory.GetPersistQueueByClient(clientId, userName, out isNewCreated);
}
catch (BrokerQueueException e)
{
// Catch the exception about the username not match and translte it to session fault
if (e.ErrorCode == (int)BrokerQueueErrorCode.E_BQ_USER_NOT_MATCH)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.AccessDenied_BrokerQueue, SR.AccessDenied_BrokerQueue, clientId, userName);
}
else
{
throw;
}
}
this.queue.OnEvent += new BrokerQueueEventDelegate(this.Queue_OnEvent);
this.queue.OnPutResponsesSuccessEvent += new EventHandler <PutResponsesSuccessEventArgs>(this.Queue_OnPutResponsesSuccessEvent);
this.queue.OnFatalExceptionEvent += new EventHandler <ExceptionEventArgs>(this.Queue_OnFatalExceptionEvent);
this.timeoutManager = new TimeoutManager("BrokerClient " + clientId);
BrokerTracing.EtwTrace.LogBrokerClientCreated(this.sharedData.BrokerInfo.SessionId, clientId);
if (this.queue.IsAllRequestsProcessed || monitor.ServiceJobState == ServiceJobState.Finished)
{
// If the queue has processed all the request or the service job is finished, the broker client can only get responses
this.state = BrokerClientState.GetResponse;
this.endOfMessageCalled = true;
BrokerTracing.EtwTrace.LogBrokerClientStateTransition(this.sharedData.BrokerInfo.SessionId, this.clientId, "GetResponse");
this.timeoutManager.RegisterTimeout(this.sharedData.Config.Monitor.ClientIdleTimeout, this.TimeoutToDisconnected, this.state);
}
else
{
if (!this.queue.EOMReceived)
{
// If EndOfMessage is not received, the client is in the ClientConnected state and is ready to accept messages
this.state = BrokerClientState.ClientConnected;
BrokerTracing.EtwTrace.LogBrokerClientStateTransition(this.sharedData.BrokerInfo.SessionId, this.clientId, "ClientConnected");
this.timeoutManager.RegisterTimeout(this.sharedData.Config.Monitor.ClientIdleTimeout, this.TimeoutToDisconnected, this.state);
}
else
{
// If EndOfMessage has been received, the client is in the EndOfMessage state and does not accept any more requests.
this.state = BrokerClientState.EndRequests;
this.endOfMessageCalled = true;
BrokerTracing.EtwTrace.LogBrokerClientStateTransition(this.sharedData.BrokerInfo.SessionId, this.clientId, "EndOfMessage");
}
}
}
/// <summary>
/// Close the broker
/// </summary>
/// <param name="cleanData">indicate whether the broker should clean up the data</param>
public async Task Close(bool cleanData)
{
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Start closing: cleanData = {0}", cleanData);
this.sharedData.WaitForInitializationComplete();
if (Interlocked.Increment(ref this.closeFlag) != 1)
{
BrokerTracing.TraceInfo("[BrokerEntry] Close race condition detected, quit.");
return;
}
this.cleanData = cleanData;
int step = 0;
// Step 1: Close Frontend
if (this.frontendResult != null)
{
for (int i = 0; i < this.frontendResult.ServiceHostList.Length; i++)
{
try
{
if (this.frontendResult.ServiceHostList[i] != null)
{
this.frontendResult.ServiceHostList[i].Close(CloseTimeout);
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close {1} controller frontend succeeded.", ++step, FrontendResult.GetTransportSchemeNameByIndex(i));
}
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close {2} controller frontend failed: {0}", e, ++step, FrontendResult.GetTransportSchemeNameByIndex(i));
}
}
for (int i = 0; i < this.frontendResult.FrontendList.Length; i++)
{
try
{
if (this.frontendResult.FrontendList[i] != null)
{
this.frontendResult.FrontendList[i].Close();
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close {1} frontend succeeded.", ++step, FrontendResult.GetTransportSchemeNameByIndex(i));
}
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close {2} frontend failed: {0}", e, ++step, FrontendResult.GetTransportSchemeNameByIndex(i));
}
}
}
// Step 2: Close client manager
List <string> activeClientIdList;
if (this.clientManager != null)
{
activeClientIdList = this.clientManager.GetAllActiveClientIds();
try
{
if (cleanData)
{
this.clientManager.DeleteAllQueues();
}
this.clientManager.Dispose();
this.clientManager = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close client manager succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close client manager failed: {1}", ++step, e);
}
}
else
{
activeClientIdList = new List <string>();
}
//Check the StrategyConfig.WithoutSessionLayer for the close progress.
//Step 3: Finish the service job if it is needed.
// We only finish the service job if clean data is required, in other cases, the service job monitor will finish the service job according to the service job life cycle before we enter this stage
if (this.monitor != null && !SoaCommonConfig.WithoutSessionLayer)
{
try
{
if (cleanData)
{
await this.monitor.FinishServiceJob("Close Session");
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Finish service job succeeded.", ++step);
}
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Finish service job failed: {1}", ++step, e);
}
}
// Step 4: Close monitor
if (this.monitor != null)
{
try
{
// Update suspended state
if (!SoaCommonConfig.WithoutSessionLayer)
{
await this.monitor.UpdateSuspended(!cleanData);
}
this.monitor.Close();
this.monitor = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close monitor succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close monitor failed: {0}", e, ++step);
}
}
// Step 5: Close state manager
if (this.stateManager != null)
{
try
{
this.stateManager.Close();
this.stateManager = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close state manager succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close state manager failed: {0}", e, ++step);
}
}
// Step 7: Close broker queue
if (this.brokerQueueFactory != null)
{
foreach (ClientInfo clientInfo in this.brokerQueueFactory.AllClientInfos)
{
if (activeClientIdList.Contains(clientInfo.ClientId))
{
continue;
}
try
{
bool isNewCreated;
BrokerQueue queue = this.brokerQueueFactory.GetPersistQueueByClient(clientInfo.ClientId, clientInfo.UserName, out isNewCreated);
Debug.Assert(!isNewCreated, "[BrokerEntry] Close: Should only get exsiting persist queue");
if (cleanData)
{
queue.Close();
}
else
{
queue.Dispose();
}
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue {1} succeeded.", ++step, clientInfo.ClientId);
}
catch (Exception e)
{
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue {1} failed: {2}", ++step, clientInfo.ClientId, e);
}
}
try
{
this.brokerQueueFactory.Dispose();
this.brokerQueueFactory = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue factory succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close broker queue factory failed: {0}", e, ++step);
}
}
// Step 8: Clean up shared data
if (this.sharedData != null)
{
try
{
this.sharedData.Dispose();
this.sharedData = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close shared data succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Close shared data failed: {1}", ++step, e);
}
}
// Step 9: Dispose node mapping
if (this.nodeMappingData != null)
{
try
{
this.nodeMappingData.Dispose();
this.nodeMappingData = null;
BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Disposing node mapping succeeded.", ++step);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Disposing node mapping failed: {1}", ++step, e);
}
}
#if DEBUG
if (!ReferenceObject.CheckDisposed())
{
BrokerTracing.TraceEvent(TraceEventType.Warning, 0, "[BrokerEntry] Reference object not disposed after closing proceduer");
}
#endif
BrokerTracing.TraceVerbose("[BrokerEntry] Close finished.");
if (this.BrokerFinished != null)
{
this.BrokerFinished(this, EventArgs.Empty);
}
}
/// <summary>
/// Run the broker
/// </summary>
/// <param name="startInfo">session start info</param>
/// <param name="brokerInfo">indicate the broker start info</param>
/// <returns>initialization result</returns>
public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo)
{
BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId);
try
{
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology);
// Step 1: Initialize configuration and shared data
ServiceConfiguration serviceConfig;
BrokerConfigurations brokerConfig;
BindingsSection bindings;
SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed.
ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings);
this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded.");
Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}");
// Step 2: Initialize broker queue
ClientInfo[] clientInfo;
this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded.");
// Step 3: Initialize observer
this.observer = new BrokerObserver(this.sharedData, clientInfo);
this.sharedData.Observer = this.observer;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded.");
// Step 4: Initialize state manager
this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0);
this.stateManager.UnloadBroker += this.UnloadBroker;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded.");
// Step 5: Initialize service job monitor
var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode);
if (SoaCommonConfig.WithoutSessionLayer)
{
this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
else
{
this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context);
}
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded.");
// Step 6: Initalize broker authorization
this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded.");
// Step 7: Initialize dispatcher manager
DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded.");
// Step 8: Start service job monitor
this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded.");
// Step 9: Initailize client manager
this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded.");
// if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist
string[] requestQueueUris = { };
string requestBlobUri = string.Empty;
string controllerRequestQueueUri = string.Empty;
string controllerResponseQueueUri = string.Empty;
if (startInfo.UseAzureStorage)
{
int clusterHash = 0;
if (!string.IsNullOrEmpty(brokerInfo.ClusterId))
{
string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant();
clusterHash = clusterIdString.GetHashCode();
}
else if (!string.IsNullOrEmpty(brokerInfo.ClusterName))
{
string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant();
clusterHash = clusterNameString.GetHashCode();
}
else
{
throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined.");
}
if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString))
{
this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString);
requestQueueUris = this.azureQueueProxy.RequestQueueUris;
requestBlobUri = this.azureQueueProxy.RequestBlobUri;
var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId);
var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId);
controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
requestQName,
CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult();
controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas(
brokerInfo.AzureStorageConnectionString,
responseQName,
CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult();
if (this.SessionId == SessionStartInfo.StandaloneSessionId)
{
CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName });
}
}
else
{
BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set.");
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable);
}
}
// Step 10: Initialize frontend
this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy);
////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize;
////this.readerQuotas = this.frontendResult.ReaderQuotas;
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded.");
// Step 11: Start frontend, Initialization finished after this step
this.OpenFrontend();
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded.");
// Step 12: Build initialization result and retrun to client
BrokerInitializationResult result = BrokerEntry.BuildInitializationResult(
this.frontendResult,
dispatcherManager,
this.sharedData.Config.LoadBalancing.ServiceOperationTimeout,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval,
this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount,
requestQueueUris,
requestBlobUri,
controllerRequestQueueUri,
controllerResponseQueueUri,
startInfo.UseAzureStorage);
BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded.");
BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded.");
return(result);
}
catch (Exception ex)
{
BrokerTracing.TraceError(ex.ToString());
throw;
}
finally
{
if (this.sharedData != null)
{
this.sharedData.InitializationFinished();
}
}
}
/// <summary>
/// Initializes a new instance of the BrokerClientManager class
/// </summary>
/// <param name="clientList">indicating the client info list</param>
/// <param name="queueFactory">indicating the queue factory</param>
/// <param name="observer">indicating the observer</param>
/// <param name="stateManager">indicating the state manager</param>
/// <param name="monitor">indicating the monitor</param>
/// <param name="sharedData">indicating the shared data</param>
public BrokerClientManager(ClientInfo[] clientList, BrokerQueueFactory queueFactory, BrokerObserver observer, BrokerStateManager stateManager, ServiceJobMonitorBase monitor, SharedData sharedData)
{
this.clientDic = new Dictionary <string, BrokerClient>(StringComparer.OrdinalIgnoreCase);
this.queueFactory = queueFactory;
this.observer = observer;
this.stateManager = stateManager;
this.monitor = monitor;
this.sharedData = sharedData;
foreach (ClientInfo client in clientList)
{
try
{
// Bug 5193: Only raise client that has requests to process.
if (client.TotalRequestsCount != client.ProcessedRequestsCount)
{
this.AddNewClient(client.ClientId, client.UserName);
}
}
catch (Exception e)
{
// Create client may fail because of broker queue failure, ignore the client in this situation and trys other client instead.
BrokerTracing.TraceError("[BrokerClientManager] Failed to create client {0}, Exception = {1}", client.ClientId, e);
}
}
this.CheckIfAllRequestDone();
this.CheckIfEOMCalled();
}
/// <summary>
/// Initializes a new instance of the BaseResponsesHandler class
/// </summary>
/// <param name="queue">indicating the broker queue</param>
/// <param name="action">indicating the action</param>
/// <param name="timeoutManager">indicating the timeout manager</param>
/// <param name="observer">indicating the observer</param>
/// <param name="sharedData">indicating the shared data</param>
/// <param name="version">indicating the message version</param>
protected BaseResponsesHandler(BrokerQueue queue, string action, TimeoutManager timeoutManager, BrokerObserver observer, SharedData sharedData, MessageVersion version)
{
this.queue = queue;
this.action = action;
this.timeoutManager = timeoutManager;
this.observer = observer;
this.sharedData = sharedData;
this.version = version;
}