private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null) { // Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations. var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService)) { try { await _hostStartSemaphore.WaitAsync(); // Now that we're inside the semaphore, set this task as completed. This prevents // restarts from being invoked (via the PlaceholderSpecializationMiddleware) before // the IHostedService has ever started. _hostStartedSource.TrySetResult(true); await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode); } finally { activeOperation.Dispose(); _hostStartSemaphore.Release(); } } }
/// <summary> /// Creates a new <see cref="ScriptHostStartupOperation"/> and adds it to the <see cref="ActiveOperations"/> collection. /// </summary> /// <param name="cancellationToken">A CancellationToken to be linked to <see cref="CancellationTokenSource"/>.</param> /// <param name="logger">A logger.</param> /// <param name="parentId">Ther parent operation id, if needed.</param> /// <returns>A new instance.</returns> public static ScriptHostStartupOperation Create(CancellationToken cancellationToken, ILogger logger, Guid?parentId = null) { var operation = new ScriptHostStartupOperation(cancellationToken, logger, parentId); _startupOperations.AddOrUpdate(operation, addValue: null, (k, v) => null); Log.StartupOperationCreated(logger, operation.Id, operation.ParentId); return(operation); }
public async Task RestartHostAsync(CancellationToken cancellationToken) { if (ShutdownRequested) { return; } using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerRestartService)) { // Do not invoke a restart if the host has not yet been started. This can lead // to invalid state. if (!_hostStarted.IsCompleted) { _logger.RestartBeforeStart(); await _hostStarted; } _logger.EnteringRestart(); // If anything is mid-startup, cancel it. _startupLoopTokenSource?.Cancel(); foreach (var startupOperation in ScriptHostStartupOperation.ActiveOperations) { _logger.CancelingStartupOperationForRestart(startupOperation.Id); try { startupOperation.CancellationTokenSource.Cancel(); } catch (ObjectDisposedException) { // This can be disposed at any time. } } try { await _hostStartSemaphore.WaitAsync(); if (State == ScriptHostState.Stopping || State == ScriptHostState.Stopped) { _logger.SkipRestart(State.ToString()); return; } State = ScriptHostState.Default; _logger.Restarting(); var previousHost = ActiveHost; ActiveHost = null; using (var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger)) { Task startTask, stopTask; // If we are running in development mode with core tools, do not overlap the restarts. // Overlapping restarts are problematic when language worker processes are listening // to the same debug port if (ShouldEnforceSequentialRestart()) { stopTask = Orphan(previousHost, cancellationToken); await stopTask; startTask = UnsynchronizedStartHostAsync(activeOperation); } else { startTask = UnsynchronizedStartHostAsync(activeOperation); stopTask = Orphan(previousHost, cancellationToken); } await startTask; } _logger.Restarted(); } catch (OperationCanceledException) { if (cancellationToken.IsCancellationRequested) { _logger.ScriptHostServiceRestartCanceledByRuntime(); throw; } // If the exception was triggered by our startup operation cancellation token, just ignore as // it doesn't indicate an issue. } finally { _hostStartSemaphore.Release(); } } }
/// <summary> /// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned /// hosts or an incorrect ActiveHost. /// </summary> private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { IHost localHost = null; var currentCancellationToken = activeOperation.CancellationTokenSource.Token; _logger.StartupOperationStarting(activeOperation.Id); try { currentCancellationToken.ThrowIfCancellationRequested(); // if we were in an error state retain that, // otherwise move to default if (State != ScriptHostState.Error) { State = ScriptHostState.Default; } bool isOffline = Utility.CheckAppOffline(_environment, _applicationHostOptions.CurrentValue.ScriptPath); State = isOffline ? ScriptHostState.Offline : State; bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError); bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError); // If we're in a non-transient error state or offline, skip host initialization bool skipJobHostStartup = isOffline || hasNonTransientErrors; bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError; _logger.Building(skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost)) { localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration); } ActiveHost = localHost; var scriptHost = (ScriptHost)ActiveHost.Services.GetService <ScriptHost>(); if (scriptHost != null) { scriptHost.HostInitializing += OnHostInitializing; if (!handlingError) { // Services may be initialized, but we don't want set the state to Initialized as we're // handling an error and want to retain the Error state. scriptHost.HostInitialized += OnHostInitialized; } } LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id); if (!_scriptWebHostEnvironment.InStandbyMode) { // At this point we know that App Insights is initialized (if being used), so we // can dispose this early request tracking module, which forces our new one to take over. DisposeRequestTrackingModule(); } currentCancellationToken.ThrowIfCancellationRequested(); var hostInstanceId = GetHostInstanceId(localHost); _logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost)) { await localHost.StartAsync(currentCancellationToken); } if (!handlingError) { LastError = null; if (!isOffline) { State = ScriptHostState.Running; } } } catch (OperationCanceledException) { GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id); throw; } catch (Exception exc) { bool isActiveHost = ReferenceEquals(localHost, ActiveHost); ILogger logger = GetHostLogger(localHost); if (isActiveHost) { LastError = exc; State = ScriptHostState.Error; logger.ErrorOccuredDuringStartupOperation(activeOperation.Id, exc); } else { // Another host has been created before this host // threw its startup exception. We want to make sure it // doesn't control the state of the service. logger.ErrorOccuredInactive(activeOperation.Id, exc); } attemptCount++; if (ShutdownHostIfUnhealthy()) { return; } if (isActiveHost) { // We don't want to return disposed services via the Services property, so // set this to null before calling Orphan(). ActiveHost = null; } var orphanTask = Orphan(localHost) .ContinueWith(t => { if (t.IsFaulted) { t.Exception.Handle(e => true); } }, TaskContinuationOptions.ExecuteSynchronously); // Use the fallback logger now, as we cannot trust when the host // logger will be disposed. logger = _logger; if (currentCancellationToken.IsCancellationRequested) { logger.CancellationRequested(activeOperation.Id); currentCancellationToken.ThrowIfCancellationRequested(); } var nextStartupAttemptMode = JobHostStartupMode.Normal; if (exc is HostConfigurationException) { // Try starting the host without parsing host.json. This will start up a // minimal host and allow the portal to see the error. Any modification will restart again. nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError; } else if (exc is HostInitializationException) { nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError; } if (nextStartupAttemptMode != JobHostStartupMode.Normal) { logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'."); Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id); } else { logger.LogDebug($"Will start a new host after delay."); await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2), logger : logger); if (currentCancellationToken.IsCancellationRequested) { logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started."); currentCancellationToken.ThrowIfCancellationRequested(); } logger.LogDebug("Starting new host after delay."); Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id); } } }