private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null) { // Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations. var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService)) { try { await _hostStartSemaphore.WaitAsync(); // Now that we're inside the semaphore, set this task as completed. This prevents // restarts from being invoked (via the PlaceholderSpecializationMiddleware) before // the IHostedService has ever started. _hostStartedSource.TrySetResult(true); await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode); } finally { activeOperation.Dispose(); _hostStartSemaphore.Release(); } } }
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { cancellationToken.ThrowIfCancellationRequested(); try { // if we were in an error state retain that, // otherwise move to default if (State != ScriptHostState.Error) { State = ScriptHostState.Default; } bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath); State = isOffline ? ScriptHostState.Offline : State; bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError); // If we're in a non-transient error state or offline, skip host initialization bool skipJobHostStartup = isOffline || hasNonTransientErrors; _host = BuildHost(skipJobHostStartup, skipHostJsonConfiguration: startupMode == JobHostStartupMode.HandlingConfigurationParsingError); var scriptHost = (ScriptHost)_host.Services.GetService <ScriptHost>(); if (scriptHost != null) { scriptHost.HostInitializing += OnHostInitializing; } LogInitialization(isOffline, attemptCount, ++_hostStartCount); await _host.StartAsync(cancellationToken); if (!startupMode.HasFlag(JobHostStartupMode.HandlingError)) { LastError = null; if (!isOffline) { State = ScriptHostState.Running; } } } catch (OperationCanceledException) { throw; } catch (Exception exc) { LastError = exc; State = ScriptHostState.Error; attemptCount++; ILogger logger = GetHostLogger(); logger.LogError(exc, "A host error has occurred"); if (ShutdownHostIfUnhealthy()) { return; } var orphanTask = Orphan(_host, logger) .ContinueWith(t => { if (t.IsFaulted) { t.Exception.Handle(e => true); } }, TaskContinuationOptions.ExecuteSynchronously); cancellationToken.ThrowIfCancellationRequested(); var nextStartupAttemptMode = JobHostStartupMode.Normal; if (exc is HostConfigurationException) { // Try starting the host without parsing host.json. This will start up a // minimal host and allow the portal to see the error. Any modification will restart again. nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError; } else if (exc is HostInitializationException) { nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError; } if (nextStartupAttemptMode != JobHostStartupMode.Normal) { Task ignore = StartHostAsync(cancellationToken, attemptCount, nextStartupAttemptMode); } else { await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2)) .ContinueWith(t => { cancellationToken.ThrowIfCancellationRequested(); return(StartHostAsync(cancellationToken, attemptCount)); }); } } }
/// <summary> /// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned /// hosts or an incorrect ActiveHost. /// </summary> private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { IHost localHost = null; var currentCancellationToken = activeOperation.CancellationTokenSource.Token; _logger.StartupOperationStarting(activeOperation.Id); try { currentCancellationToken.ThrowIfCancellationRequested(); // if we were in an error state retain that, // otherwise move to default if (State != ScriptHostState.Error) { State = ScriptHostState.Default; } bool isOffline = Utility.CheckAppOffline(_environment, _applicationHostOptions.CurrentValue.ScriptPath); State = isOffline ? ScriptHostState.Offline : State; bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError); bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError); // If we're in a non-transient error state or offline, skip host initialization bool skipJobHostStartup = isOffline || hasNonTransientErrors; bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError; _logger.Building(skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost)) { localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration); } ActiveHost = localHost; var scriptHost = (ScriptHost)ActiveHost.Services.GetService <ScriptHost>(); if (scriptHost != null) { scriptHost.HostInitializing += OnHostInitializing; if (!handlingError) { // Services may be initialized, but we don't want set the state to Initialized as we're // handling an error and want to retain the Error state. scriptHost.HostInitialized += OnHostInitialized; } } LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id); if (!_scriptWebHostEnvironment.InStandbyMode) { // At this point we know that App Insights is initialized (if being used), so we // can dispose this early request tracking module, which forces our new one to take over. DisposeRequestTrackingModule(); } currentCancellationToken.ThrowIfCancellationRequested(); var hostInstanceId = GetHostInstanceId(localHost); _logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost)) { await localHost.StartAsync(currentCancellationToken); } if (!handlingError) { LastError = null; if (!isOffline) { State = ScriptHostState.Running; } } } catch (OperationCanceledException) { GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id); throw; } catch (Exception exc) { bool isActiveHost = ReferenceEquals(localHost, ActiveHost); ILogger logger = GetHostLogger(localHost); if (isActiveHost) { LastError = exc; State = ScriptHostState.Error; logger.ErrorOccuredDuringStartupOperation(activeOperation.Id, exc); } else { // Another host has been created before this host // threw its startup exception. We want to make sure it // doesn't control the state of the service. logger.ErrorOccuredInactive(activeOperation.Id, exc); } attemptCount++; if (ShutdownHostIfUnhealthy()) { return; } if (isActiveHost) { // We don't want to return disposed services via the Services property, so // set this to null before calling Orphan(). ActiveHost = null; } var orphanTask = Orphan(localHost) .ContinueWith(t => { if (t.IsFaulted) { t.Exception.Handle(e => true); } }, TaskContinuationOptions.ExecuteSynchronously); // Use the fallback logger now, as we cannot trust when the host // logger will be disposed. logger = _logger; if (currentCancellationToken.IsCancellationRequested) { logger.CancellationRequested(activeOperation.Id); currentCancellationToken.ThrowIfCancellationRequested(); } var nextStartupAttemptMode = JobHostStartupMode.Normal; if (exc is HostConfigurationException) { // Try starting the host without parsing host.json. This will start up a // minimal host and allow the portal to see the error. Any modification will restart again. nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError; } else if (exc is HostInitializationException) { nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError; } if (nextStartupAttemptMode != JobHostStartupMode.Normal) { logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'."); Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id); } else { logger.LogDebug($"Will start a new host after delay."); await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2), logger : logger); if (currentCancellationToken.IsCancellationRequested) { logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started."); currentCancellationToken.ThrowIfCancellationRequested(); } logger.LogDebug("Starting new host after delay."); Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id); } } }
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { cancellationToken.ThrowIfCancellationRequested(); IHost localHost = null; try { // if we were in an error state retain that, // otherwise move to default if (State != ScriptHostState.Error) { State = ScriptHostState.Default; } bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath); State = isOffline ? ScriptHostState.Offline : State; bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError); // If we're in a non-transient error state or offline, skip host initialization bool skipJobHostStartup = isOffline || hasNonTransientErrors; localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration: startupMode == JobHostStartupMode.HandlingConfigurationParsingError); _host = localHost; var scriptHost = (ScriptHost)_host.Services.GetService <ScriptHost>(); if (scriptHost != null) { scriptHost.HostInitializing += OnHostInitializing; } LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount); if (!_scriptWebHostEnvironment.InStandbyMode) { // At this point we know that App Insights is initialized (if being used), so we // can dispose this early request tracking module, which forces our new one to take over. DisposeRequestTrackingModule(); } await _host.StartAsync(cancellationToken); if (!startupMode.HasFlag(JobHostStartupMode.HandlingError)) { LastError = null; if (!isOffline) { State = ScriptHostState.Running; } } } catch (OperationCanceledException) { GetHostLogger(localHost).LogDebug("Host startup was canceled."); throw; } catch (Exception exc) { bool isActiveHost = ReferenceEquals(localHost, _host); ILogger logger = GetHostLogger(localHost); if (isActiveHost) { LastError = exc; State = ScriptHostState.Error; logger.LogError(exc, "A host error has occurred"); } else { // Another host has been created before this host // threw its startup exception. We want to make sure it // doesn't control the state of the service. logger.LogWarning(exc, "A host error has occurred on an inactive host"); } attemptCount++; if (ShutdownHostIfUnhealthy()) { return; } var orphanTask = Orphan(localHost, logger) .ContinueWith(t => { if (t.IsFaulted) { t.Exception.Handle(e => true); } }, TaskContinuationOptions.ExecuteSynchronously); if (cancellationToken.IsCancellationRequested) { logger.LogDebug($"Cancellation requested. A new host will not be started."); cancellationToken.ThrowIfCancellationRequested(); } var nextStartupAttemptMode = JobHostStartupMode.Normal; if (exc is HostConfigurationException) { // Try starting the host without parsing host.json. This will start up a // minimal host and allow the portal to see the error. Any modification will restart again. nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError; } else if (exc is HostInitializationException) { nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError; } if (nextStartupAttemptMode != JobHostStartupMode.Normal) { Task ignore = StartHostAsync(cancellationToken, attemptCount, nextStartupAttemptMode); } else { await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2)) .ContinueWith(t => { cancellationToken.ThrowIfCancellationRequested(); return(StartHostAsync(cancellationToken, attemptCount)); }); } } }
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { try { await _hostStartSemaphore.WaitAsync(); // Now that we're inside the semaphore, set this task as completed. This prevents // restarts from being invoked (via the PlaceholderSpecializationMiddleware) before // the IHostedService has ever started. _hostStartedSource.TrySetResult(true); await UnsynchronizedStartHostAsync(cancellationToken, attemptCount, startupMode); } finally { _hostStartSemaphore.Release(); } }
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { try { await _hostStartSemaphore.WaitAsync(); await UnsynchronizedStartHostAsync(cancellationToken, attemptCount, startupMode); } finally { _hostStartSemaphore.Release(); } }