/// <summary> /// Close the broker /// </summary> /// <param name="cleanData">indicate whether the broker should clean up the data</param> public async Task Close(bool cleanData) { BrokerTracing.TraceVerbose("[BrokerEntry] Close: Start closing: cleanData = {0}", cleanData); this.sharedData.WaitForInitializationComplete(); if (Interlocked.Increment(ref this.closeFlag) != 1) { BrokerTracing.TraceInfo("[BrokerEntry] Close race condition detected, quit."); return; } this.cleanData = cleanData; int step = 0; // Step 1: Close Frontend if (this.frontendResult != null) { for (int i = 0; i < this.frontendResult.ServiceHostList.Length; i++) { try { if (this.frontendResult.ServiceHostList[i] != null) { this.frontendResult.ServiceHostList[i].Close(CloseTimeout); BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close {1} controller frontend succeeded.", ++step, FrontendResult.GetTransportSchemeNameByIndex(i)); } } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close {2} controller frontend failed: {0}", e, ++step, FrontendResult.GetTransportSchemeNameByIndex(i)); } } for (int i = 0; i < this.frontendResult.FrontendList.Length; i++) { try { if (this.frontendResult.FrontendList[i] != null) { this.frontendResult.FrontendList[i].Close(); BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close {1} frontend succeeded.", ++step, FrontendResult.GetTransportSchemeNameByIndex(i)); } } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close {2} frontend failed: {0}", e, ++step, FrontendResult.GetTransportSchemeNameByIndex(i)); } } } // Step 2: Close client manager List <string> activeClientIdList; if (this.clientManager != null) { activeClientIdList = this.clientManager.GetAllActiveClientIds(); try { if (cleanData) { this.clientManager.DeleteAllQueues(); } this.clientManager.Dispose(); this.clientManager = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close client manager succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close client manager failed: {1}", ++step, e); } } else { activeClientIdList = new List <string>(); } //Check the StrategyConfig.WithoutSessionLayer for the close progress. //Step 3: Finish the service job if it is needed. // We only finish the service job if clean data is required, in other cases, the service job monitor will finish the service job according to the service job life cycle before we enter this stage if (this.monitor != null && !SoaCommonConfig.WithoutSessionLayer) { try { if (cleanData) { await this.monitor.FinishServiceJob("Close Session"); BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Finish service job succeeded.", ++step); } } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Finish service job failed: {1}", ++step, e); } } // Step 4: Close monitor if (this.monitor != null) { try { // Update suspended state if (!SoaCommonConfig.WithoutSessionLayer) { await this.monitor.UpdateSuspended(!cleanData); } this.monitor.Close(); this.monitor = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close monitor succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close monitor failed: {0}", e, ++step); } } // Step 5: Close state manager if (this.stateManager != null) { try { this.stateManager.Close(); this.stateManager = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close state manager succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close state manager failed: {0}", e, ++step); } } // Step 7: Close broker queue if (this.brokerQueueFactory != null) { foreach (ClientInfo clientInfo in this.brokerQueueFactory.AllClientInfos) { if (activeClientIdList.Contains(clientInfo.ClientId)) { continue; } try { bool isNewCreated; BrokerQueue queue = this.brokerQueueFactory.GetPersistQueueByClient(clientInfo.ClientId, clientInfo.UserName, out isNewCreated); Debug.Assert(!isNewCreated, "[BrokerEntry] Close: Should only get exsiting persist queue"); if (cleanData) { queue.Close(); } else { queue.Dispose(); } BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue {1} succeeded.", ++step, clientInfo.ClientId); } catch (Exception e) { BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue {1} failed: {2}", ++step, clientInfo.ClientId, e); } } try { this.brokerQueueFactory.Dispose(); this.brokerQueueFactory = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close broker queue factory succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {1}: Close broker queue factory failed: {0}", e, ++step); } } // Step 8: Clean up shared data if (this.sharedData != null) { try { this.sharedData.Dispose(); this.sharedData = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Close shared data succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Close shared data failed: {1}", ++step, e); } } // Step 9: Dispose node mapping if (this.nodeMappingData != null) { try { this.nodeMappingData.Dispose(); this.nodeMappingData = null; BrokerTracing.TraceVerbose("[BrokerEntry] Close: Step {0}: Disposing node mapping succeeded.", ++step); } catch (Exception e) { BrokerTracing.TraceWarning("[BrokerEntry] Close: Step {0}: Disposing node mapping failed: {1}", ++step, e); } } #if DEBUG if (!ReferenceObject.CheckDisposed()) { BrokerTracing.TraceEvent(TraceEventType.Warning, 0, "[BrokerEntry] Reference object not disposed after closing proceduer"); } #endif BrokerTracing.TraceVerbose("[BrokerEntry] Close finished."); if (this.BrokerFinished != null) { this.BrokerFinished(this, EventArgs.Empty); } }
/// <summary> /// Run the broker /// </summary> /// <param name="startInfo">session start info</param> /// <param name="brokerInfo">indicate the broker start info</param> /// <returns>initialization result</returns> public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo) { BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId); try { BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology); // Step 1: Initialize configuration and shared data ServiceConfiguration serviceConfig; BrokerConfigurations brokerConfig; BindingsSection bindings; SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed. ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings); this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded."); Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}"); // Step 2: Initialize broker queue ClientInfo[] clientInfo; this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded."); // Step 3: Initialize observer this.observer = new BrokerObserver(this.sharedData, clientInfo); this.sharedData.Observer = this.observer; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded."); // Step 4: Initialize state manager this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0); this.stateManager.UnloadBroker += this.UnloadBroker; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded."); // Step 5: Initialize service job monitor var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode); if (SoaCommonConfig.WithoutSessionLayer) { this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context); } else { this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context); } BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded."); // Step 6: Initalize broker authorization this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded."); // Step 7: Initialize dispatcher manager DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded."); // Step 8: Start service job monitor this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult(); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded."); // Step 9: Initailize client manager this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded."); // if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist string[] requestQueueUris = { }; string requestBlobUri = string.Empty; string controllerRequestQueueUri = string.Empty; string controllerResponseQueueUri = string.Empty; if (startInfo.UseAzureStorage) { int clusterHash = 0; if (!string.IsNullOrEmpty(brokerInfo.ClusterId)) { string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant(); clusterHash = clusterIdString.GetHashCode(); } else if (!string.IsNullOrEmpty(brokerInfo.ClusterName)) { string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant(); clusterHash = clusterNameString.GetHashCode(); } else { throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined."); } if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString)) { this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString); requestQueueUris = this.azureQueueProxy.RequestQueueUris; requestBlobUri = this.azureQueueProxy.RequestBlobUri; var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId); var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId); controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas( brokerInfo.AzureStorageConnectionString, requestQName, CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult(); controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas( brokerInfo.AzureStorageConnectionString, responseQName, CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult(); if (this.SessionId == SessionStartInfo.StandaloneSessionId) { CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName }); } } else { BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set."); ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable); } } // Step 10: Initialize frontend this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy); ////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize; ////this.readerQuotas = this.frontendResult.ReaderQuotas; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded."); // Step 11: Start frontend, Initialization finished after this step this.OpenFrontend(); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded."); // Step 12: Build initialization result and retrun to client BrokerInitializationResult result = BrokerEntry.BuildInitializationResult( this.frontendResult, dispatcherManager, this.sharedData.Config.LoadBalancing.ServiceOperationTimeout, this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval, this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount, requestQueueUris, requestBlobUri, controllerRequestQueueUri, controllerResponseQueueUri, startInfo.UseAzureStorage); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded."); BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded."); return(result); } catch (Exception ex) { BrokerTracing.TraceError(ex.ToString()); throw; } finally { if (this.sharedData != null) { this.sharedData.InitializationFinished(); } } }