/// <summary> /// Initializes a new instance of the SharedData class /// </summary> /// <param name="brokerInfo">indicating the broker info</param> /// <param name="startInfo">indicating the start info</param> /// <param name="config">indicating the broker configuration</param> /// <param name="serviceCnfig">indicating the service configuration</param> public SharedData(BrokerStartInfo brokerInfo, SessionStartInfoContract startInfo, BrokerConfigurations config, ServiceConfiguration serviceConfig) { this.brokerInfo = brokerInfo; this.startInfo = startInfo; this.config = config; this.serviceConfig = serviceConfig; }
/// <summary> /// Ask broker to initialize /// </summary> /// <param name="startInfo">indicating the start info</param> /// <param name="brokerInfo">indicating the broker info</param> /// <returns>returns broker initialization result</returns> public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo) { ParamCheckUtility.ThrowIfNull(startInfo, "startInfo"); ParamCheckUtility.ThrowIfNull(brokerInfo, "brokerInfo"); // Concurrent mode is set to single so no need to lock here if (this.entry != null) { ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AlreadyInitialized, SR.AlreadyInitialized); } // Notice: We get the session Id here and expect to change the // cosmos log file name to include session id. But cosmos log has a // limitation that it is a static class without close method, and // it can't be initialized twice. HpcTraceListener is just a wrapper // for that static class. So creating a new HpcTraceListener can not // workaround this issue. TraceHelper.TraceInfo( brokerInfo.SessionId, "[BrokerManagementService].Initialize: Broker worker initializes for session {0}", brokerInfo.SessionId); // Set the configuration indicating enable/disable diag trace SoaDiagTraceHelper.SetDiagTraceEnabledFlag(brokerInfo.SessionId, brokerInfo.EnableDiagTrace); TraceHelper.IsDiagTraceEnabled = SoaDiagTraceHelper.IsDiagTraceEnabled; Environment.SetEnvironmentVariable(SessionIdEnv, brokerInfo.SessionId.ToString(), EnvironmentVariableTarget.Process); #if HPCPACK // create the session id mapping file foreach (TraceListener listener in TraceHelper.RuntimeTrace.CosmosTrace.Listeners) { if (listener is HpcTraceListener) { HpcTraceListener hpcListener = listener as HpcTraceListener; if (hpcListener.IsPerSessionLogging) { string logIdFileName = string.Format("{0}_{1}", HpcTraceListener.LogFileBaseName, brokerInfo.SessionId); string logIdFilePath = Path.Combine(HpcTraceListener.LogDir, logIdFileName); try { using (FileStream file = File.Open(logIdFilePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.None)) { } } catch (Exception e) { TraceHelper.TraceError(brokerInfo.SessionId, "[BrokerManagementService].Initialize: Create log session Id match file failed with Exception {0}", e.ToString()); } } } } #endif this.entry = new BrokerEntry(brokerInfo.SessionId); this.entry.BrokerFinished += new EventHandler(this.Entry_BrokerFinished); return(this.entry.Run(startInfo, brokerInfo)); }
/// <summary> /// Ask broker to initialize /// </summary> /// <param name="startInfo">indicating the start info</param> /// <param name="brokerInfo">indicating the broker info</param> public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo) { // Call async version and block on completion in order to workaround System.Net.Socket bug #750028 IAsyncResult result = this.Channel.BeginInitialize(startInfo, brokerInfo, null, null); return(this.Channel.EndInitialize(result)); }
public void BuildFrontEndTest_NetTcp() { SessionStartInfoContract info = new SessionStartInfoContract(); info.ServiceName = "CcpEchoSvc"; BrokerStartInfo startInfo = new BrokerStartInfo(); string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile; ExeConfigurationFileMap map = new ExeConfigurationFileMap(); map.ExeConfigFilename = filename; Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None); BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config); startInfo.Durable = true; startInfo.SessionId = "100"; startInfo.ConfigurationFile = "CcpEchoSvc.config"; ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings); var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig); var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null); Assert.IsTrue(Regex.IsMatch(result.ControllerUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/Controller")); Assert.IsTrue(Regex.IsMatch(result.GetResponseUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp/GetResponse")); Assert.IsTrue(Regex.IsMatch(result.FrontendUriList.FirstOrDefault(), @"net\.tcp://.+:9091/100/NetTcp")); }
/// <summary> /// Create a session with the specific sessionid /// </summary> /// <param name="info">Session Start Info</param> /// <param name="sessionid">the session id</param> /// <returns>the session info</returns> public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionid) { // Call async version and block on completion in order to workaround System.Net.Socket bug #750028 IAsyncResult result = this.Channel.BeginCreate(info, sessionid, null, null); return(this.Channel.EndCreate(result)); }
public void BuildFrontEndTest_Http() { SessionStartInfoContract info = new SessionStartInfoContract(); info.ServiceName = "CcpEchoSvc"; info.TransportScheme = TransportScheme.Http; info.Secure = true; BrokerStartInfo startInfo = new BrokerStartInfo(); MockBrokerAuthorization auth = new MockBrokerAuthorization(); string filename = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile; ExeConfigurationFileMap map = new ExeConfigurationFileMap(); map.ExeConfigFilename = filename; Configuration config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None); BrokerConfigurations brokerConfig = BrokerConfigurations.GetSectionGroup(config); startInfo.Durable = false; startInfo.SessionId = "100"; startInfo.ConfigurationFile = "CcpEchoSvc.config"; ConfigurationHelper.LoadConfiguration(info, startInfo, out brokerConfig, out var serviceConfig, out var bindings); var sharedData = new SharedData(startInfo, info, brokerConfig, serviceConfig); var result = FrontEndBuilder.BuildFrontEnd(sharedData, new BrokerObserver(sharedData, new ClientInfo[0]), null, null, bindings, null); Assert.IsTrue(Regex.IsMatch(result.ControllerUriList[1], @"https://.+/100/Http/Controller")); Assert.AreEqual(null, result.GetResponseUriList.FirstOrDefault()); Assert.IsTrue(Regex.IsMatch(result.FrontendUriList[1], @"https://.+/100/Http")); }
/// <summary> /// Ask broker to initialize /// </summary> /// <param name="startInfo">indicating the start info</param> /// <param name="brokerInfo">indicating the broker info</param> /// <returns>returns broker initialization result</returns> public BrokerInitializationResult Initialize(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo) { ThrowIfNull(startInfo, "startInfo"); ThrowIfNull(brokerInfo, "brokerInfo"); this.entry = new BrokerEntry(startInfo, brokerInfo, this); return(this.entry.Result); }
/// <summary> /// Create a new broker which working under Durable manner. /// </summary> /// <param name="info">session start info</param> /// <param name="sessionid">the session id which is also service job id</param> /// <returns>The created session info</returns> public BrokerInitializationResult CreateDurable(SessionStartInfoContract info, string sessionId) { this.ThrowIfInitialized(); this.startInfoContract = info; this.BuildBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, true, this.attached).GetAwaiter().GetResult(); this.CreateInternal(sessionId).GetAwaiter().GetResult(); return(this.result); }
/// <summary> /// Allocate a session and get a list of brokerlauncher EPR /// </summary> /// <param name="info">Session start info</param> /// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param> /// <param name="sessionid">the sessionid returns</param> /// <param name="serviceVersion">the service version</param> /// <param name="sessionInfo">the session info</param> /// <returns>The EPRs of the broker launchers</returns> public async Task <SessionAllocateInfoContract> AllocateAsync(SessionStartInfoContract info, string endpointPrefix) { return(await this.Channel.AllocateAsync(info, endpointPrefix).ConfigureAwait(false)); // Call async version and block on completion in order to workaround System.Net.Socket bug #750028 //IAsyncResult result = this.Channel.BeginAllocate(info, endpointPrefix, null, null); //return this.Channel.EndAllocate(result); }
/// <summary> /// Create a session with the specific sessionid /// </summary> /// <param name="info">Session Start Info</param> /// <param name="sessionid">the session id</param> /// <param name="callback">The async callback</param> /// <param name="state">async state object</param> /// <returns>the async result</returns> public IAsyncResult BeginCreate( SessionStartInfoContract info, string sessionid, AsyncCallback callback, object state) { return(this.Channel.BeginCreate(info, sessionid, callback, state)); }
public static async Task OpenSvcHostsAsync(string sessionId, SessionStartInfoContract startInfo, Func <List <TaskInfo>, Task> taskStateChangedCallBack) { for (int i = 0; i < startInfo.IpAddress.Length; i++) { OpenSvcHostWithRetryAsync(sessionId, i, startInfo.IpAddress[i], startInfo.RegPath, startInfo.ServiceName, startInfo.ServiceVersion, startInfo.Environments, startInfo.DependFilesStorageInfo) .ContinueWith(t => taskStateChangedCallBack(new List <TaskInfo> { t.Result })); } }
protected abstract Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath);
/// <summary> /// Initializes a new instance of the BrokerInfo class from broker recover info /// </summary> /// <param name="recoverInfo">indicating the broker recover info</param> /// <param name="brokerInfo">indicating the broker start info</param> /// <param name="auth">indicating the broker auth</param> /// <param name="customBroker">indicating the custom broker configuration</param> /// <param name="pool">indicating the broker process pool</param> public BrokerInfo(BrokerRecoverInfo recoverInfo, BrokerStartInfo brokerInfo, BrokerAuthorization auth, CustomBrokerRegistration customBroker, BrokerProcessPool pool) { this.callbackToCloseBroker = new ThreadHelper <IAsyncResult>(new AsyncCallback(this.OnCloseBroker)).CallbackRoot; this.durable = recoverInfo.Durable; this.brokerInfo = brokerInfo; this.sessionId = recoverInfo.SessionId; this.sessionStartInfo = recoverInfo.StartInfo; this.auth = auth; this.customBroker = customBroker; this.pool = pool; this.sessionStartInfo.IpAddress = BrokerLauncherSettings.Default.SvcHostList.Cast <string>().ToArray(); this.sessionStartInfo.RegPath = BrokerLauncherSettings.Default.CCP_SERVICEREGISTRATION_PATH; }
/// <summary> /// Create a new broker /// </summary> /// <param name="info">session start info</param> /// <param name="sessionid">the session id which is also service job id</param> /// <returns>The session Info</returns> public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionId) { // TODO: SF: change to async this.ThrowIfInitialized(); this.startInfoContract = info; if (info.IsNoSession) { this.BuildStandaloneBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, false, this.attached); } else { this.BuildBrokerStartInfo(info.ServiceName, info.ServiceVersion, sessionId, false, this.attached).GetAwaiter().GetResult(); } this.CreateInternal(sessionId).GetAwaiter().GetResult(); return(this.result); }
/// <summary> /// Initializes a new instance of the BrokerEntry class /// </summary> /// <param name="startInfo">indicating the start info</param> /// <param name="brokerInfo">indicating the broker info</param> public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance) { if (startInfo.TransportScheme != TransportScheme.NetTcp) { throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp."); } if (brokerInfo.Durable) { throw new NotSupportedException("Sample broker does not support durable session."); } this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance); Frontend frontend = new Frontend(this.clientManager); WebHttpBinding binding = new WebHttpBinding(); binding.MaxBufferPoolSize = 5000000; binding.MaxBufferSize = 5000000; binding.MaxReceivedMessageSize = 5000000; binding.ReaderQuotas.MaxArrayLength = 5000000; binding.ReaderQuotas.MaxBytesPerRead = 5000000; binding.ReaderQuotas.MaxDepth = 5000000; binding.ReaderQuotas.MaxNameTableCharCount = 5000000; binding.ReaderQuotas.MaxStringContentLength = 5000000; this.frontendServiceHost = new WebServiceHost(frontend, new Uri(String.Format("http://{0}:8081/", Environment.MachineName))); ServiceEndpoint endpoint = this.frontendServiceHost.AddServiceEndpoint(typeof(IWebHttpFrontendService), binding, String.Empty); endpoint.Behaviors.Add(new WebHttpBehavior()); string listenUri = endpoint.ListenUri.AbsoluteUri; this.frontendServiceHost.Open(); this.result = new BrokerInitializationResult(); this.result.BrokerEpr = new string[3] { listenUri, null, null }; this.result.ControllerEpr = new string[3]; this.result.ResponseEpr = new string[3]; }
/// <summary> /// Create a session /// </summary> /// <param name="info">session start info</param> /// <param name="sessionId">session id</param> /// <returns>returns broker initialization result</returns> public BrokerInitializationResult Create(SessionStartInfoContract info, string sessionId) { if ((!this.AllowNewSession) || (!this.IsOnline && String.IsNullOrEmpty(info.DiagnosticBrokerNode))) { ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerIsOffline, SR.BrokerIsOffline); } // Handle invalid input parameters try { ParamCheckUtility.ThrowIfNull(info, "info"); if (!BrokerLauncherEnvironment.Standalone) { this.CheckAccess(sessionId); } TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerLauncher] Create: SessionId = {0}", sessionId); //TODO: make it async BrokerInitializationResult returnValue = this.brokerManager.CreateNewBrokerDomain(info, sessionId, false).GetAwaiter().GetResult(); #region Debug Failure Test SimulateFailure.FailOperation(1); #endregion TraceHelper.RuntimeTrace.LogSessionCreated(sessionId); TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Information, "[BrokerLauncher] Create Broker {0} Succeeded.", sessionId); return(returnValue); } catch (Exception e) { TraceHelper.RuntimeTrace.LogFailedToCreateSession(sessionId); // Bug 10614: Throw a proper exception when the broker node is being taken offline if ((!this.AllowNewSession) || (!this.IsOnline && String.IsNullOrEmpty(info.DiagnosticBrokerNode))) { ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerIsOffline, SR.BrokerIsOffline); } TraceHelper.TraceEvent(sessionId, System.Diagnostics.TraceEventType.Error, "[BrokerLauncher] Create Broker {0} failed: {1}", sessionId, e.ToString()); throw ExceptionHelper.ConvertExceptionToFaultException(e); } }
/// <summary> /// Initializes a new instance of the BrokerEntry class /// </summary> /// <param name="startInfo">indicating the start info</param> /// <param name="brokerInfo">indicating the broker info</param> public BrokerEntry(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo, BrokerManagementService serviceInstance) { if (startInfo.TransportScheme != TransportScheme.NetTcp) { throw new NotSupportedException("Sample broker does not support transport scheme other than NetTcp."); } if (brokerInfo.Durable) { throw new NotSupportedException("Sample broker does not support durable session."); } this.clientManager = new ServiceClientManager(brokerInfo.SessionId, brokerInfo.Headnode, serviceInstance); Frontend frontend = new Frontend(this.clientManager); // Choose different binding configuration by start info NetTcpBinding frontendBinding; if (startInfo.Secure) { frontendBinding = new NetTcpBinding(); } else { frontendBinding = new NetTcpBinding(SecurityMode.None); } frontendBinding.PortSharingEnabled = true; this.frontendServiceHost = new ServiceHost(frontend, new Uri(String.Format("net.tcp://{0}:9091/SampleBroker", Environment.MachineName))); string listenUri = this.frontendServiceHost.AddServiceEndpoint(typeof(IDuplexService), frontendBinding, String.Empty).ListenUri.AbsoluteUri; this.frontendServiceHost.Open(); this.result = new BrokerInitializationResult(); this.result.BrokerEpr = new string[3] { listenUri, null, null }; this.result.ControllerEpr = new string[3]; this.result.ResponseEpr = new string[3]; }
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath) { // string sessionId = LocalSessionConfiguration.GetNextSessionId(); string cmd; if (true) { cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath}"; } else { cmd = $"-d --ServiceRegistrationPath {LocalSessionConfiguration.ServiceRegistrationPath} --AzureStorageConnectionString {LocalSessionConfiguration.BrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True"; } string sessionId = SessionStartInfo.StandaloneSessionId; this.brokerLauncherProcess = Process.Start( LocalSessionConfiguration.BrokerLauncherExePath, cmd); this.svcHostProcess = Process.Start(LocalSessionConfiguration.ServiceHostExePath, "-standalone"); sessionAllocateInfo.Id = sessionId; // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken }; sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress("localhost") }; return(sessionAllocateInfo); }
/// <summary> /// Override Start method and start dummy service job monitor /// </summary> /// <param name="startInfo"></param> /// <param name="dispatcherManager"></param> /// <param name="observer"></param> /// <returns></returns> public override async Task Start(SessionStartInfoContract startInfo, DispatcherManager dispatcherManager, BrokerObserver observer) { BrokerTracing.TraceVerbose("[ServiceJobMonitor].Start: Enter"); this.dispatcherManager = dispatcherManager; this.observer = observer; this.gracefulPreemptionHandler = new GracefulPreemptionHandler(this.dispatcherManager, this.sharedData.BrokerInfo.SessionId, taskId => this.FinishTask(taskId, isRunAwayTask: true)); this.schedulerAdapterClientFactory = new SchedulerAdapterClientFactory(this.sharedData, this, dispatcherManager, this.context); if (this.sharedData.Config.Monitor.AllocationAdjustInterval != Timeout.Infinite && this.sharedData.StartInfo.EprList == null) { BrokerTracing.TraceVerbose("[ServiceJobMonitor].Start: start the allocation adjust thread, interval={0}", this.sharedData.Config.Monitor.AllocationAdjustInterval); this.allocationAdjustThread.Start(); } // await SvcHostRestModule.OpenSvcHostsAsync(this.sharedData.BrokerInfo.SessionId, startInfo, ((ISchedulerNotify)this).TaskStateChanged); await((ISchedulerNotify)this).TaskStateChanged(SvcHostRestModule.CreateDummyTaskInfos(startInfo.IpAddress)); }
/// <summary> /// Create a new broker application domain /// </summary> /// <param name="info">session start info</param> /// <param name="sessionid">session id</param> /// <param name="durable">indicate if the session is durable</param> /// <returns>returns broker initialization result</returns> public async Task <BrokerInitializationResult> CreateNewBrokerDomain(SessionStartInfoContract info, string sessionid, bool durable) { string userName = (OperationContext.Current != null && OperationContext.Current.ServiceSecurityContext != null && OperationContext.Current.ServiceSecurityContext.WindowsIdentity != null) ? OperationContext.Current.ServiceSecurityContext.WindowsIdentity.Name : String.Empty; TraceHelper.RuntimeTrace.LogSessionCreating(sessionid, userName); TraceHelper.TraceEvent(sessionid, System.Diagnostics.TraceEventType.Information, "[BrokerManager] Create new broker domain: {0}", sessionid); BrokerRecoverInfo recoverInfo = new BrokerRecoverInfo(); recoverInfo.StartInfo = info; recoverInfo.SessionId = sessionid; recoverInfo.Durable = durable; if (this.schedulerHelper == null) { this.schedulerHelper = SchedulerHelperFactory.GetSchedulerHelper(this.context); } ClusterInfoContract clusterInfo = await this.schedulerHelper.GetClusterInfoAsync(); return(await this.CreateBrokerAndRun(recoverInfo, false, clusterInfo)); }
IAsyncResult IBrokerLauncher.BeginCreateDurable(SessionStartInfoContract info, string sessionId, AsyncCallback callback, object state) { throw new NotSupportedException(); }
public virtual async Task <SessionAllocateInfoContract> AllocateAsync(SessionStartInfoContract info, string endpointPrefix) { return(await this.AllocateInternalAsync(info, endpointPrefix, false)); }
/// <summary> /// Allocate a new durable or non-durable session /// </summary> /// <param name="startInfo">session start info</param> /// <param name="durable">whether session should be durable</param> /// <param name="endpointPrefix">the endpoint prefix, net.tcp:// or https:// </param> /// <returns>the Broker Launcher EPRs, sorted by the preference.</returns> protected virtual async Task <SessionAllocateInfoContract> AllocateInternalAsync(SessionStartInfoContract startInfo, string endpointPrefix, bool durable) { TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] Begin: AllocateInternalAsync"); SessionAllocateInfoContract sessionAllocateInfo = new SessionAllocateInfoContract(); ParamCheckUtility.ThrowIfNull(startInfo, "startInfo"); ParamCheckUtility.ThrowIfNullOrEmpty(startInfo.ServiceName, "startInfo.ServiceName"); ParamCheckUtility.ThrowIfNullOrEmpty(endpointPrefix, "endpointPrefix"); #if HPCPACK // check client api version, 4.3 or older client is not supported by 4.4 server for the broken changes if (startInfo.ClientVersion == null || startInfo.ClientVersion < new Version(4, 4)) { TraceHelper.TraceEvent(TraceEventType.Error, "[SessionLauncher] .AllocateInternalAsync: ClientVersion {0} does not match ServerVersion {1}.", startInfo.ClientVersion, ServerVersion); ThrowHelper.ThrowSessionFault(SOAFaultCode.ClientServerVersionMismatch, SR.SessionLauncher_ClientServerVersionMismatch, startInfo.ClientVersion == null ? "NULL" : startInfo.ClientVersion.ToString(), ServerVersion.ToString()); } #endif // Init service version to the service version passed in if (startInfo.ServiceVersion != null) { sessionAllocateInfo.ServiceVersion = startInfo.ServiceVersion; TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is {0}", sessionAllocateInfo.ServiceVersion); } else { sessionAllocateInfo.ServiceVersion = null; TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Original service version is null."); } string callId = Guid.NewGuid().ToString(); this.CheckAccess(); SecureString securePassword = CreateSecureString(startInfo.Password); startInfo.Password = null; // BUG 4522 : Use CCP_SCHEDULER when referencing service registration file share so HA HN virtual name is used when needed //var reliableRegistry = new ReliableRegistry(this.fabricClient.PropertyManager); //string defaultServiceRegistrationServerName = await reliableRegistry.GetValueAsync<string>(HpcConstants.HpcFullKeyName, HpcConstants.FileShareServerRegVal, this.token); //if (String.IsNullOrEmpty(defaultServiceRegistrationServerName)) //{ // defaultServiceRegistrationServerName = "localhost"; //} // the reg repo path is from scheduler environments, defaultServiceRegistrationServerName is actually not used string serviceConfigFile; ServiceRegistrationRepo serviceRegistration = this.GetRegistrationRepo(callId); serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, startInfo.ServiceVersion); // If the serviceConfigFile wasnt found and serviceversion isnt specified, try getitng the service config based on the service's latest version if (string.IsNullOrEmpty(serviceConfigFile) && (startInfo.ServiceVersion == null)) { TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Try to find out versioned service."); // Get service version in ServiceRegistrationRepo Version dynamicServiceVersion = serviceRegistration.GetServiceVersionInternal(startInfo.ServiceName, false); if (dynamicServiceVersion != null) { TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Selected dynamicServiceVersion is {0}.", dynamicServiceVersion.ToString()); } serviceConfigFile = serviceRegistration.GetServiceRegistrationPath(startInfo.ServiceName, dynamicServiceVersion); // If a config file is found, update the serviceVersion that is returned to client and stored in recovery info if (!string.IsNullOrEmpty(serviceConfigFile)) { TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: serviceConfigFile is {0}.", serviceConfigFile); startInfo.ServiceVersion = dynamicServiceVersion; if (dynamicServiceVersion != null) { sessionAllocateInfo.ServiceVersion = dynamicServiceVersion; } } } string serviceName = ServiceRegistrationRepo.GetServiceRegistrationFileName(startInfo.ServiceName, startInfo.ServiceVersion); TraceHelper.TraceEvent(TraceEventType.Verbose, "[SessionLauncher] .AllocateInternalAsync: Service name = {0}, Configuration file = {1}", serviceName, serviceConfigFile); // If the service is not found and user code doesn't specify // version, we will use the latest version. if (string.IsNullOrEmpty(serviceConfigFile)) { if (startInfo.ServiceVersion != null) { ThrowHelper.ThrowSessionFault(SOAFaultCode.ServiceVersion_NotFound, SR.SessionLauncher_ServiceVersionNotFound, startInfo.ServiceName, startInfo.ServiceVersion.ToString()); } else { ThrowHelper.ThrowSessionFault(SOAFaultCode.Service_NotFound, SR.SessionLauncher_ServiceNotFound, startInfo.ServiceName); } } ExeConfigurationFileMap map = new ExeConfigurationFileMap(); map.ExeConfigFilename = serviceConfigFile; ServiceRegistration registration = null; BrokerConfigurations brokerConfigurations = null; string hostpath = null; string traceSwitchValue = null; try { Configuration config = null; RetryManager.RetryOnceAsync( () => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None), TimeSpan.FromSeconds(1), ex => ex is ConfigurationErrorsException).GetAwaiter().GetResult(); Debug.Assert(config != null, "Configuration is not opened properly."); registration = ServiceRegistration.GetSectionGroup(config); brokerConfigurations = BrokerConfigurations.GetSectionGroup(config); if (registration != null && registration.Host != null && registration.Host.Path != null) { hostpath = registration.Host.Path; } else { // x86 or x64 hostpath = registration.Service.Architecture == ServiceArch.X86 ? TaskCommandLine32 : TaskCommandLine64; } traceSwitchValue = registration.Service.SoaDiagTraceLevel; // TODO: should deprecate the previous settings if (string.IsNullOrEmpty(traceSwitchValue)) { traceSwitchValue = ConfigurationHelper.GetTraceSwitchValue(config); } } catch (ConfigurationErrorsException e) { ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, SR.SessionLauncher_ConfigFileInvalid, e.ToString()); } catch (Exception ex) { TraceHelper.TraceEvent(TraceEventType.Error, ex.ToString()); throw; } // after figuring out the service and version, and the session pool size, we check if the service pool already has the instance. sessionAllocateInfo.Id = "0"; sessionAllocateInfo.SessionInfo = null; if (startInfo.UseSessionPool) { if (this.TryGetSessionAllocateInfoFromPooled(endpointPrefix, durable, sessionAllocateInfo, serviceConfigFile, registration, out var allocateInternal)) { return(allocateInternal); } } // for sessions to add in session pool try { var sessionAllocateInfoContract = await this.CreateAndSubmitSessionJob( startInfo, endpointPrefix, durable, callId, securePassword, registration, sessionAllocateInfo, traceSwitchValue, serviceName, brokerConfigurations, hostpath); if (sessionAllocateInfoContract != null) { return(sessionAllocateInfoContract); } } finally { // Add the submitted job to the session pool. if (startInfo.UseSessionPool) { this.AddSessionToPool(Path.GetFileNameWithoutExtension(serviceConfigFile), durable, sessionAllocateInfo.Id, registration.Service.MaxSessionPoolSize); } } return(null); }
/// <summary> /// Run the broker /// </summary> /// <param name="startInfo">session start info</param> /// <param name="brokerInfo">indicate the broker start info</param> /// <returns>initialization result</returns> public BrokerInitializationResult Run(SessionStartInfoContract startInfo, BrokerStartInfo brokerInfo) { BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Information, 0, "[BrokerEntry] Broker is starting initialization, ID = {0}", brokerInfo.SessionId); try { BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: ClusterTopology is {0}", brokerInfo.NetworkTopology); // Step 1: Initialize configuration and shared data ServiceConfiguration serviceConfig; BrokerConfigurations brokerConfig; BindingsSection bindings; SoaCommonConfig.WithoutSessionLayer = startInfo.IsNoSession; // TODO: this is a hack. Working mode should be decided by something like a *SchedulerType* filed. ConfigurationHelper.LoadConfiguration(startInfo, brokerInfo, out brokerConfig, out serviceConfig, out bindings); this.sharedData = new SharedData(brokerInfo, startInfo, brokerConfig, serviceConfig); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 1: Loading configuration and shared data succeeded."); Debug.WriteLine($"[BrokerEntry](Debug) UseAad:{startInfo.UseAad}"); // Step 2: Initialize broker queue ClientInfo[] clientInfo; this.brokerQueueFactory = BrokerEntry.InitBrokerQueue(this.sharedData, out clientInfo); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 2: Initialize broker queue succeeded."); // Step 3: Initialize observer this.observer = new BrokerObserver(this.sharedData, clientInfo); this.sharedData.Observer = this.observer; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 3: Initialize broker observer succeeded."); // Step 4: Initialize state manager this.stateManager = new BrokerStateManager(this.sharedData, clientInfo.Length != 0); this.stateManager.UnloadBroker += this.UnloadBroker; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 4: Initialize broker state manager succeeded."); // Step 5: Initialize service job monitor var context = TelepathyContext.GetOrAdd(this.sharedData.BrokerInfo.Headnode); if (SoaCommonConfig.WithoutSessionLayer) { this.monitor = new DummyServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context); } else { this.monitor = new ServiceJobMonitor.ServiceJobMonitor(this.sharedData, this.stateManager, this.nodeMappingData, context); } BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 5: Initialize service job monitor succeeded."); // Step 6: Initalize broker authorization this.brokerAuth = BrokerEntry.BuildBrokerAuthorization(this.sharedData); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 6: Initialize broker authorization succeeded."); // Step 7: Initialize dispatcher manager DispatcherManager dispatcherManager = new DispatcherManager(bindings, this.sharedData, this.observer, this.monitor, this.brokerQueueFactory, context); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 7: Initialize dispatcher manager succeeded."); // Step 8: Start service job monitor this.monitor.Start(startInfo, dispatcherManager, this.observer).GetAwaiter().GetResult(); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 8: Start service job monitor succeeded."); // Step 9: Initailize client manager this.clientManager = new BrokerClientManager(clientInfo, this.brokerQueueFactory, this.observer, this.stateManager, this.monitor, this.sharedData); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 9: Initialize client manager succeeded."); // if using AzureQueue, retrieve the connection string and build the request and response message queues if not exist string[] requestQueueUris = { }; string requestBlobUri = string.Empty; string controllerRequestQueueUri = string.Empty; string controllerResponseQueueUri = string.Empty; if (startInfo.UseAzureStorage) { int clusterHash = 0; if (!string.IsNullOrEmpty(brokerInfo.ClusterId)) { string clusterIdString = brokerInfo.ClusterId.ToLowerInvariant(); clusterHash = clusterIdString.GetHashCode(); } else if (!string.IsNullOrEmpty(brokerInfo.ClusterName)) { string clusterNameString = brokerInfo.ClusterName.ToLowerInvariant(); clusterHash = clusterNameString.GetHashCode(); } else { throw new InvalidOperationException($"Both {nameof(brokerInfo.ClusterId)} and {nameof(brokerInfo.ClusterName)} are null or empty. No {nameof(clusterHash)} can be determined."); } if (!string.IsNullOrEmpty(brokerInfo.AzureStorageConnectionString)) { this.azureQueueProxy = new AzureQueueProxy(brokerInfo.ClusterName, clusterHash, this.SessionId, brokerInfo.AzureStorageConnectionString); requestQueueUris = this.azureQueueProxy.RequestQueueUris; requestBlobUri = this.azureQueueProxy.RequestBlobUri; var requestQName = CloudQueueConstants.GetBrokerWorkerControllerRequestQueueName(this.SessionId); var responseQName = CloudQueueConstants.GetBrokerWorkerControllerResponseQueueName(this.SessionId); controllerRequestQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas( brokerInfo.AzureStorageConnectionString, requestQName, CloudQueueCreationModule.AddMessageSasPolicy).GetAwaiter().GetResult(); controllerResponseQueueUri = CloudQueueCreationModule.CreateCloudQueueAndGetSas( brokerInfo.AzureStorageConnectionString, responseQName, CloudQueueCreationModule.ProcessMessageSasPolicy).GetAwaiter().GetResult(); if (this.SessionId == SessionStartInfo.StandaloneSessionId) { CloudQueueCreationModule.ClearCloudQueuesAsync(brokerInfo.AzureStorageConnectionString, new[] { requestQName, responseQName }); } } else { BrokerTracing.TraceError("[BrokerEntry] Initialization: Use Azure Queue is specified, however the Azure connection string is not set."); ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_AzureConnectionStringNotAvailable, SR.Broker_AzureConnectionStringNotAvailable); } } // Step 10: Initialize frontend this.frontendResult = FrontEndBuilder.BuildFrontEnd(this.sharedData, this.observer, this.clientManager, this.brokerAuth, bindings, this.azureQueueProxy); ////this.maxMessageSize = (int)this.frontendResult.MaxMessageSize; ////this.readerQuotas = this.frontendResult.ReaderQuotas; BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 10: Initialize frontend succeeded."); // Step 11: Start frontend, Initialization finished after this step this.OpenFrontend(); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 11: Open frontend succeeded."); // Step 12: Build initialization result and retrun to client BrokerInitializationResult result = BrokerEntry.BuildInitializationResult( this.frontendResult, dispatcherManager, this.sharedData.Config.LoadBalancing.ServiceOperationTimeout, this.sharedData.Config.Monitor.ClientBrokerHeartbeatInterval, this.sharedData.Config.Monitor.ClientBrokerHeartbeatRetryCount, requestQueueUris, requestBlobUri, controllerRequestQueueUri, controllerResponseQueueUri, startInfo.UseAzureStorage); BrokerTracing.TraceVerbose("[BrokerEntry] Initialization: Step 12: Build initialization result suceeded."); BrokerTracing.TraceInfo("[BrokerEntry] Initialization succeeded."); return(result); } catch (Exception ex) { BrokerTracing.TraceError(ex.ToString()); throw; } finally { if (this.sharedData != null) { this.sharedData.InitializationFinished(); } } }
/// <summary> /// override the Start Method in internal service job monitor /// </summary> /// <param name="startInfo"></param> /// <param name="dispatcherManager"></param> /// <param name="observer"></param> /// <returns></returns> public override async Task Start(SessionStartInfoContract startInfo, DispatcherManager dispatcherManager, BrokerObserver observer) { await this.Start(dispatcherManager, observer); }
internal static async Task <ServiceTaskDispatcherInfo> OpenSvcHostsAsync(string sessionId, SessionStartInfoContract startInfo, ServiceTaskDispatcherInfo taskDispatcherInfo) { return(await OpenSvcHostWithRetryAsync(sessionId, taskDispatcherInfo, startInfo.RegPath, startInfo.ServiceName, startInfo.ServiceVersion, startInfo.Environments, startInfo.DependFilesStorageInfo)); }
/// <summary> /// This method will only be internally called by session launcher, /// before the job is submitted. It combines the user's job properties and the /// items set in the service start info. The job properties will override the start info items. /// </summary> /// <param name="startInfo">the session start info</param> /// <param name="schedulerJob">service job</param> /// <param name="traceLevel">diag trace level</param> internal static void MakeJobProperties(SessionStartInfoContract startInfo, ISchedulerJob schedulerJob, string traceLevel) { Debug.Assert(startInfo != null, "The session startInfo cannot be null."); Debug.Assert(!string.IsNullOrEmpty(startInfo.ServiceName), "The service name in the session start info cannnot be null or empty."); schedulerJob.ServiceName = startInfo.ServiceName; schedulerJob.SetEnvironmentVariable("CCP_SERVICENAME", startInfo.ServiceName); if (startInfo.CanPreempt.HasValue) { schedulerJob.CanPreempt = startInfo.CanPreempt.Value; } if (!string.IsNullOrEmpty(startInfo.JobTemplate)) { schedulerJob.SetJobTemplate(startInfo.JobTemplate); } if (startInfo.Priority.HasValue) { schedulerJob.Priority = (JobPriority)startInfo.Priority.Value; } if (startInfo.ExtendedPriority.HasValue) { schedulerJob.ExpandedPriority = startInfo.ExtendedPriority.Value; } schedulerJob.Progress = 0; // For max units if (startInfo.MaxUnits != null) { schedulerJob.MaximumNumberOfCores = schedulerJob.MaximumNumberOfSockets = schedulerJob.MaximumNumberOfNodes = startInfo.MaxUnits.Value; schedulerJob.AutoCalculateMax = false; } // For min units if (startInfo.MinUnits != null) { schedulerJob.MinimumNumberOfCores = schedulerJob.MinimumNumberOfSockets = schedulerJob.MinimumNumberOfNodes = startInfo.MinUnits.Value; schedulerJob.AutoCalculateMin = false; } // Should set UnitType after above resource count update if (startInfo.ResourceUnitType != null) { schedulerJob.UnitType = (JobUnitType)startInfo.ResourceUnitType.Value; } schedulerJob.Name = string.IsNullOrEmpty(startInfo.ServiceJobName) ? string.Format("{0} - WCF service", startInfo.ServiceName) : startInfo.ServiceJobName; if (!string.IsNullOrEmpty(startInfo.ServiceJobProject)) { schedulerJob.Project = startInfo.ServiceJobProject; } if (!string.IsNullOrEmpty(startInfo.NodeGroupsStr)) { string[] nodes = startInfo.NodeGroupsStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); foreach (string node in nodes) { schedulerJob.NodeGroups.Add(node); } } if (!string.IsNullOrEmpty(startInfo.RequestedNodesStr)) { schedulerJob.RequestedNodes = new StringCollection(startInfo.RequestedNodesStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)); } if (startInfo.Runtime >= 0) { schedulerJob.Runtime = startInfo.Runtime; } // start adding the broker settings. schedulerJob.SetCustomProperty(BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString()); schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAad, startInfo.UseAad.ToString()); schedulerJob.SetCustomProperty(BrokerSettingsConstants.Secure, startInfo.Secure.ToString()); schedulerJob.SetCustomProperty(BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString()); schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString()); schedulerJob.SetCustomProperty(BrokerSettingsConstants.LocalUser, (startInfo.LocalUser == true).ToString()); var context = HpcContext.Get(); var principal = Thread.CurrentPrincipal; if (principal.IsHpcAadPrincipal(context)) { string identity = principal.GenerateSecurityIdentifierFromAadPrincipal(context).Value + ";" + principal.Identity.Name; schedulerJob.SetCustomProperty(BrokerSettingsConstants.AadUserIdentity, identity); } // Save ServiceVersion if set if (startInfo.ServiceVersion != null) { schedulerJob.SetCustomProperty(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion.ToString()); } string[] customPropNames = new string[] { BrokerSettingsConstants.AllocationGrowLoadRatioThreshold, BrokerSettingsConstants.AllocationShrinkLoadRatioThreshold, BrokerSettingsConstants.ClientIdleTimeout, BrokerSettingsConstants.SessionIdleTimeout, BrokerSettingsConstants.MessagesThrottleStartThreshold, BrokerSettingsConstants.MessagesThrottleStopThreshold, BrokerSettingsConstants.ClientConnectionTimeout, BrokerSettingsConstants.ServiceConfigMaxMessageSize, BrokerSettingsConstants.ServiceConfigOperationTimeout, BrokerSettingsConstants.DispatcherCapacityInGrowShrink }; int?[] intNullableValues = new int?[] { startInfo.AllocationGrowLoadRatioThreshold, startInfo.AllocationShrinkLoadRatioThreshold, startInfo.ClientIdleTimeout, startInfo.SessionIdleTimeout, startInfo.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStopThreshold, startInfo.ClientConnectionTimeout, startInfo.MaxMessageSize, startInfo.ServiceOperationTimeout, startInfo.DispatcherCapacityInGrowShrink }; Debug.Assert(intNullableValues.Length == customPropNames.Length); for (int i = 0; i < customPropNames.Length; i++) { if (intNullableValues[i].HasValue) { schedulerJob.SetCustomProperty(customPropNames[i], intNullableValues[i].Value.ToString()); } } // add soa diag settings schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceLevel, traceLevel); schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceCleanup, Boolean.FalseString); }
/// <summary> /// Load the configuration from the configuration file /// If no configuration is found or some part is missing, we will fill the blank with default value /// </summary> /// <param name="brokerSettings">indicate the broker settings from the session start info, this settings will override the settings load from the configuration file</param> /// <param name="brokerInfo">indicating the broker info</param> /// <param name="brokerConfig">out the broker configurations</param> /// <param name="serviceConfig">out the service configurations</param> /// <param name="bindings">output the bindings</param> public static void LoadConfiguration(SessionStartInfoContract brokerSettings, BrokerStartInfo brokerInfo, out BrokerConfigurations brokerConfig, out ServiceConfiguration serviceConfig, out BindingsSection bindings) { // Init config file string filename = brokerInfo.ConfigurationFile; BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 1: Load configuration file name: {0}", filename); brokerConfig = null; serviceConfig = null; bindings = null; try { ExeConfigurationFileMap map = new ExeConfigurationFileMap(); map.ExeConfigFilename = filename; Configuration config = null; RetryManager.RetryOnceAsync( () => config = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None), TimeSpan.FromSeconds(1), ex => ex is ConfigurationErrorsException) .GetAwaiter() .GetResult(); brokerConfig = BrokerConfigurations.GetSectionGroup(config); serviceConfig = ServiceRegistration.GetSectionGroup(config).Service; bindings = (BindingsSection)config.GetSection(BindingsSectionName); } catch (ConfigurationErrorsException e) { ThrowHelper.ThrowSessionFault(SOAFaultCode.ConfigFile_Invalid, "{0}", e.ToString()); } if (brokerConfig == null) { // Set the default value brokerConfig = defaultBrokerConfiguration; BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the broker config from the configuration file, use the default configuration"); } else { // Set the default configuration if the very section is not found if (brokerConfig.Monitor == null) { brokerConfig.Monitor = defaultBrokerConfiguration.Monitor; BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the monitor config from the configuration file, use the default configuration"); } if (brokerConfig.Services == null) { brokerConfig.Services = defaultBrokerConfiguration.Services; BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the services config from the configuration file, use the default configuration"); } if (brokerConfig.LoadBalancing == null) { brokerConfig.LoadBalancing = defaultBrokerConfiguration.LoadBalancing; BrokerTracing.TraceEvent(TraceEventType.Information, 0, "[ConfigurationHelper] Didn't find the load balancing config from the configuration file, use the default configuration"); } } BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 2: Load broker config and service config succeeded."); if (brokerConfig.Monitor.AllocationAdjustInterval < MinAllocationAdjustTime && brokerConfig.Monitor.AllocationAdjustInterval != System.Threading.Timeout.Infinite) { brokerConfig.Monitor.AllocationAdjustInterval = MinAllocationAdjustTime; } // Update the broker config using the session start info if (brokerSettings.ClientIdleTimeout.HasValue) { brokerConfig.Monitor.ClientIdleTimeout = brokerSettings.ClientIdleTimeout.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientIdleTimeout to {0}", brokerConfig.Monitor.ClientIdleTimeout); } if (brokerSettings.ClientConnectionTimeout.HasValue) { brokerConfig.Monitor.ClientConnectionTimeout = brokerSettings.ClientConnectionTimeout.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientConnectionTimeout to {0}", brokerConfig.Monitor.ClientConnectionTimeout); } if (brokerSettings.SessionIdleTimeout.HasValue) { brokerConfig.Monitor.SessionIdleTimeout = brokerSettings.SessionIdleTimeout.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default SessionIdleTimeout to {0}", brokerConfig.Monitor.SessionIdleTimeout); } if (brokerSettings.MessagesThrottleStartThreshold.HasValue) { brokerConfig.Monitor.MessageThrottleStartThreshold = brokerSettings.MessagesThrottleStartThreshold.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStartThreshold to {0}", brokerConfig.Monitor.MessageThrottleStartThreshold); } if (brokerSettings.MessagesThrottleStopThreshold.HasValue) { brokerConfig.Monitor.MessageThrottleStopThreshold = brokerSettings.MessagesThrottleStopThreshold.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MessageThrottleStopThreshold to {0}", brokerConfig.Monitor.MessageThrottleStopThreshold); } if (brokerSettings.ClientBrokerHeartbeatRetryCount.HasValue) { brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount = brokerSettings.ClientBrokerHeartbeatRetryCount.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatRetryCount to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatRetryCount); } if (brokerSettings.ClientBrokerHeartbeatInterval.HasValue) { brokerConfig.Monitor.ClientBrokerHeartbeatInterval = brokerSettings.ClientBrokerHeartbeatInterval.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ClientBrokerHeartbeatInterval to {0}", brokerConfig.Monitor.ClientBrokerHeartbeatInterval); } if (brokerSettings.ServiceOperationTimeout.HasValue) { brokerConfig.LoadBalancing.ServiceOperationTimeout = brokerSettings.ServiceOperationTimeout.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default ServiceOperationTimeout to {0}", brokerConfig.LoadBalancing.ServiceOperationTimeout); } if (brokerSettings.DispatcherCapacityInGrowShrink.HasValue) { brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink = brokerSettings.DispatcherCapacityInGrowShrink.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default DispatcherCapacityInGrowShrink to {0}", brokerConfig.LoadBalancing.DispatcherCapacityInGrowShrink); } if (brokerSettings.MaxMessageSize.HasValue) { serviceConfig.MaxMessageSize = brokerSettings.MaxMessageSize.Value; BrokerTracing.TraceEvent(TraceEventType.Verbose, 0, "[ConfigurationHelper] Modified default MaxMessageSize to {0}", serviceConfig.MaxMessageSize); } BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 3: Override broker settings using session start info succeeded."); // Validate the config section string configError; bool validateSucceeded; try { validateSucceeded = brokerConfig.Validate(out configError); } catch (ConfigurationErrorsException e) { validateSucceeded = false; configError = e.Message; } if (!validateSucceeded) { BrokerTracing.TraceEvent(TraceEventType.Error, 0, "[ConfigurationHelper] Invalid broker configuration section. Error {0}", configError); ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_InvalidConfiguration, configError); } BrokerTracing.TraceVerbose("[ConfigurationHelper] LoadConfiguration. Step 4: Validate broker configuration succeeded."); StringBuilder sb = new StringBuilder(); sb.AppendLine("[Monitor]"); BrokerTracing.WriteProperties(sb, brokerConfig.Monitor, 3, typeof(int), typeof(string)); sb.AppendLine("[BaseAddress]"); sb.AppendFormat(" Http = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("http")); sb.AppendFormat(" Https = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("https")); sb.AppendFormat(" NetTcp = {0}\n", brokerConfig.Services.GetBrokerBaseAddress("net.tcp")); sb.AppendLine("[LoadBalancing]"); BrokerTracing.WriteProperties(sb, brokerConfig.LoadBalancing, 3, typeof(int), typeof(string)); BrokerTracing.TraceVerbose("[ConfigurationHelper] BrokerConfiguration: \n{0}", sb.ToString()); sb = new StringBuilder(); sb.AppendLine("[Service]"); BrokerTracing.WriteProperties(sb, serviceConfig, 3, typeof(int), typeof(string)); BrokerTracing.TraceVerbose("[ConfigurationHelper] ServiceConfiguration: \n{0}", sb.ToString()); }
protected override async Task <SessionAllocateInfoContract> CreateAndSubmitSessionJob( SessionStartInfoContract startInfo, string endpointPrefix, bool durable, string callId, SecureString securePassword, ServiceRegistration registration, SessionAllocateInfoContract sessionAllocateInfo, string traceSwitchValue, string serviceName, BrokerConfigurations brokerConfigurations, string hostpath) { try { bool brokerPerfMode = true; // TODO: implement separated broker mode if (brokerPerfMode) { TraceHelper.TraceEvent(TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: broker perf mode"); } TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, endpointPrefix={1}, durable={2}.", callId, endpointPrefix, durable); using (var batchClient = AzureBatchConfiguration.GetBatchClient()) { var pool = await batchClient.PoolOperations.GetPoolAsync(AzureBatchConfiguration.BatchPoolName); ODATADetailLevel detailLevel = new ODATADetailLevel(); detailLevel.SelectClause = "affinityId, ipAddress"; //detailLevel.FilterClause = @"state eq 'idle'"; var nodes = await pool.ListComputeNodes(detailLevel).ToListAsync(); if (nodes.Count < 1) { throw new InvalidOperationException("Compute node count in selected pool is less then 1."); } sessionAllocateInfo.Id = string.Empty; // sessionAllocateInfo.BrokerLauncherEpr = new[] { SessionInternalConstants.BrokerConnectionStringToken }; IList <EnvironmentSetting> ConstructEnvironmentVariable() { List <EnvironmentSetting> env = new List <EnvironmentSetting>(); // Can change to set to ensure no unintended overwrite foreach (NameValueConfigurationElement entry in registration.Service.EnvironmentVariables) { env.Add(new EnvironmentSetting(entry.Name, entry.Value)); } // pass service serviceInitializationTimeout as job environment variables env.Add(new EnvironmentSetting(Constant.ServiceInitializationTimeoutEnvVar, registration.Service.ServiceInitializationTimeout.ToString())); if (startInfo.ServiceHostIdleTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, registration.Service.ServiceHostIdleTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHostIdleTimeoutEnvVar, startInfo.ServiceHostIdleTimeout.ToString())); } if (startInfo.ServiceHangTimeout == null) { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, registration.Service.ServiceHangTimeout.ToString())); } else { env.Add(new EnvironmentSetting(Constant.ServiceHangTimeoutEnvVar, startInfo.ServiceHangTimeout.ToString())); } // pass MessageLevelPreemption switcher as job environment variables env.Add(new EnvironmentSetting(Constant.EnableMessageLevelPreemptionEnvVar, registration.Service.EnableMessageLevelPreemption.ToString())); // pass trace switcher to svchost if (!string.IsNullOrEmpty(traceSwitchValue)) { env.Add(new EnvironmentSetting(Constant.TraceSwitchValue, traceSwitchValue)); } // pass taskcancelgraceperiod as environment variable to svchosts env.Add(new EnvironmentSetting(Constant.CancelTaskGracePeriodEnvVar, Constant.DefaultCancelTaskGracePeriod.ToString())); // pass service config file name to services env.Add(new EnvironmentSetting(Constant.ServiceConfigFileNameEnvVar, serviceName)); // pass maxMessageSize to service hosts int maxMessageSize = startInfo.MaxMessageSize.HasValue ? startInfo.MaxMessageSize.Value : registration.Service.MaxMessageSize; env.Add(new EnvironmentSetting(Constant.ServiceConfigMaxMessageEnvVar, maxMessageSize.ToString())); // pass service operation timeout to service hosts int?serviceOperationTimeout = null; if (startInfo.ServiceOperationTimeout.HasValue) { serviceOperationTimeout = startInfo.ServiceOperationTimeout; } else if (brokerConfigurations != null && brokerConfigurations.LoadBalancing != null) { serviceOperationTimeout = brokerConfigurations.LoadBalancing.ServiceOperationTimeout; } if (serviceOperationTimeout.HasValue) { env.Add(new EnvironmentSetting(Constant.ServiceConfigServiceOperatonTimeoutEnvVar, serviceOperationTimeout.Value.ToString())); } if (startInfo.Environments != null) { foreach (KeyValuePair <string, string> entry in startInfo.Environments) { env.Add(new EnvironmentSetting(entry.Key, entry.Value)); } } // Each SOA job is assigned a GUID "secret", which is used // to identify soa job owner. When a job running in Azure // tries to access common data, it sends this "secret" together // with a data request to data service. Data service trusts // the data request only if the job id and job "secret" // match. env.Add(new EnvironmentSetting(Constant.JobSecretEnvVar, Guid.NewGuid().ToString())); // Set CCP_SERVICE_SESSIONPOOL env var of the job if (startInfo.UseSessionPool) { env.Add(new EnvironmentSetting(Constant.ServiceUseSessionPoolEnvVar, bool.TrueString)); } void SetBrokerNodeAuthenticationInfo() { // TODO: set the information needed by compute node to authenticate broker node return; } SetBrokerNodeAuthenticationInfo(); env.Add(new EnvironmentSetting(BrokerSettingsConstants.Secure, startInfo.Secure.ToString())); env.Add(new EnvironmentSetting(BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme.ToString())); TraceHelper.TraceEvent( TraceEventType.Information, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: callId={0}, set job environment: {1}={2}, {3}={4}.", callId, BrokerSettingsConstants.Secure, startInfo.Secure, BrokerSettingsConstants.TransportScheme, startInfo.TransportScheme); env.Add(new EnvironmentSetting(TelepathyConstants.SchedulerEnvironmentVariableName, Dns.GetHostName())); env.Add(new EnvironmentSetting(Constant.OverrideProcNumEnvVar, "TRUE")); //Establish a link via ev between TELEPATHY_SERVICE_WORKING_DIR and AZ_BATCH_JOB_PREP_WORKING_DIR env.Add(new EnvironmentSetting(TelepathyConstants.ServiceWorkingDirEnvVar, AzureBatchPrepJobWorkingDir)); return(env); } var environment = ConstructEnvironmentVariable(); ResourceFile GetResourceFileReference(string containerName, string blobPrefix) { var sasToken = AzureStorageUtil.ConstructContainerSas(this.cloudStorageAccount, containerName, SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read); ResourceFile rf; if (string.IsNullOrEmpty(blobPrefix)) { rf = ResourceFile.FromStorageContainerUrl(sasToken); } else { rf = ResourceFile.FromStorageContainerUrl(sasToken, blobPrefix: blobPrefix); } return(rf); } async Task <string> CreateJobAsync() { //TODO: need a function to test if all parameters are legal. if (startInfo.MaxUnits != null && startInfo.MaxUnits <= 0) { throw new ArgumentException("Maxunit value is invalid."); } string newJobId = AzureBatchSessionJobIdConverter.ConvertToAzureBatchJobId(AzureBatchSessionIdGenerator.GenerateSessionId()); Debug.Assert(batchClient != null, nameof(batchClient) + " != null"); var job = batchClient.JobOperations.CreateJob(newJobId, new PoolInformation() { PoolId = AzureBatchConfiguration.BatchPoolName }); job.JobPreparationTask = new JobPreparationTask(JobPrepCmdLine); job.JobPreparationTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); job.JobPreparationTask.ResourceFiles = new List <ResourceFile>() { GetResourceFileReference(ServiceRegistrationContainer, null), GetResourceFileReference(RuntimeContainer, CcpServiceHostFolder), GetResourceFileReference(ServiceAssemblyContainer, startInfo.ServiceName.ToLower()) }; job.JobReleaseTask = new JobReleaseTask(JobReleaseCmdLine); job.JobReleaseTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // List<ResourceFile> resourceFiles = new List<ResourceFile>(); // resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); // resourceFiles.Add(GetResourceFileReference(ServiceRegistrationContainer, null)); // // job.JobManagerTask = new JobManagerTask("Broker", // // $@"cmd /c {AzureBatchTaskWorkingDirEnvVar}\broker\HpcBroker.exe -d --ServiceRegistrationPath {AzureBatchTaskWorkingDirEnvVar} --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"); // job.JobManagerTask = new JobManagerTask("List", // $@"cmd /c dir & set"); // job.JobManagerTask.ResourceFiles = resourceFiles; // job.JobManagerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); // Set Meta Data if (job.Metadata == null) { job.Metadata = new List <MetadataItem>(); } Dictionary <string, string> jobMetadata = new Dictionary <string, string>() { { BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString() }, { BrokerSettingsConstants.Secure, startInfo.Secure.ToString() }, { BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString() }, { BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString() }, }; if (startInfo.ServiceVersion != null) { jobMetadata.Add(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion?.ToString()); } if (startInfo.MaxUnits != null) { jobMetadata.Add("MaxUnits", startInfo.MaxUnits.ToString()); } Dictionary <string, int?> jobOptionalMetadata = new Dictionary <string, int?>() { { BrokerSettingsConstants.ClientIdleTimeout, startInfo.ClientIdleTimeout }, { BrokerSettingsConstants.SessionIdleTimeout, startInfo.SessionIdleTimeout }, { BrokerSettingsConstants.MessagesThrottleStartThreshold, startInfo.MessagesThrottleStartThreshold }, { BrokerSettingsConstants.MessagesThrottleStopThreshold, startInfo.MessagesThrottleStopThreshold }, { BrokerSettingsConstants.ClientConnectionTimeout, startInfo.ClientConnectionTimeout }, { BrokerSettingsConstants.ServiceConfigMaxMessageSize, startInfo.MaxMessageSize }, { BrokerSettingsConstants.ServiceConfigOperationTimeout, startInfo.ServiceOperationTimeout }, { BrokerSettingsConstants.DispatcherCapacityInGrowShrink, startInfo.DispatcherCapacityInGrowShrink } }; job.Metadata = job.Metadata.Concat(jobMetadata.Select(p => new MetadataItem(p.Key, p.Value))) .Concat(jobOptionalMetadata.Where(p => p.Value.HasValue).Select(p => new MetadataItem(p.Key, p.Value.ToString()))).ToList(); job.DisplayName = $"{job.Id} - {startInfo.ServiceName} - WCF Service"; await job.CommitAsync(); return(job.Id); } var jobId = await CreateJobAsync(); string sessionId = AzureBatchSessionJobIdConverter.ConvertToSessionId(jobId); if (!sessionId.Equals("-1")) { sessionAllocateInfo.Id = sessionId; } else { TraceHelper.TraceEvent(TraceEventType.Error, "[AzureBatchSessionLauncher] .CreateAndSubmitSessionJob: JobId was failed to parse. callId={0}, jobId={1}.", callId, jobId); } Task AddTasksAsync() { int numTasks = startInfo.MaxUnits != null ? (int)startInfo.MaxUnits : nodes.Count; var comparer = new EnvironmentSettingComparer(); CloudTask CreateTask(string taskId) { CloudTask cloudTask = new CloudTask(taskId, $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\ccpservicehost\CcpServiceHost.exe -standalone"); cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } CloudTask CreateBrokerTask(bool direct) { List <ResourceFile> resourceFiles = new List <ResourceFile>(); resourceFiles.Add(GetResourceFileReference(RuntimeContainer, BrokerFolder)); string cmd; if (direct) { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } else { cmd = $@"cmd /c %{TelepathyConstants.ServiceWorkingDirEnvVar}%\broker\HpcBroker.exe -d --AzureStorageConnectionString {AzureBatchConfiguration.SoaBrokerStorageConnectionString} --EnableAzureStorageQueueEndpoint True --SvcHostList {string.Join(",", nodes.Select(n => n.IPAddress))}"; } CloudTask cloudTask = new CloudTask("Broker", cmd); cloudTask.ResourceFiles = resourceFiles; cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)); cloudTask.EnvironmentSettings = cloudTask.EnvironmentSettings == null ? environment : environment.Union(cloudTask.EnvironmentSettings, comparer).ToList(); return(cloudTask); } //TODO: task id type should be changed from int to string var tasks = Enumerable.Range(0, numTasks - 1).Select(_ => CreateTask(Guid.NewGuid().ToString())).ToArray(); if (!brokerPerfMode) { tasks = tasks.Union(new[] { CreateBrokerTask(true) }).ToArray(); } else { tasks = tasks.Union(new[] { CreateTask(Guid.NewGuid().ToString()) }).ToArray(); } return(batchClient.JobOperations.AddTaskAsync(jobId, tasks)); } await AddTasksAsync(); async Task WaitBatchBrokerLauncher() { var brokerTask = await batchClient.JobOperations.GetTaskAsync(jobId, "Broker"); TaskStateMonitor monitor = batchClient.Utilities.CreateTaskStateMonitor(); await monitor.WhenAll(new[] { brokerTask }, TaskState.Running, SchedulingTimeout); await brokerTask.RefreshAsync(); var brokerNodeIp = nodes.First(n => n.AffinityId == brokerTask.ComputeNodeInformation.AffinityId).IPAddress; sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(brokerNodeIp) }; } if (brokerPerfMode) { //If broker node and session launcher node is not the same node, this line should be modified. sessionAllocateInfo.BrokerLauncherEpr = new[] { SoaHelper.GetBrokerLauncherAddress(Environment.MachineName) }; } else { await WaitBatchBrokerLauncher(); } return(sessionAllocateInfo); } } catch (Exception ex) { TraceHelper.TraceEvent(TraceEventType.Error, $"[{nameof(AzureBatchSessionLauncher)}] .{nameof(this.CreateAndSubmitSessionJob)}: Exception happens: {ex.ToString()}"); throw; } }
BrokerInitializationResult IBrokerLauncher.CreateDurable(SessionStartInfoContract info, string sessionId) { throw new NotSupportedException(); }