private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0,
JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null)
{
// Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations.
var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService))
{
try
{
await _hostStartSemaphore.WaitAsync();
// Now that we're inside the semaphore, set this task as completed. This prevents
// restarts from being invoked (via the PlaceholderSpecializationMiddleware) before
// the IHostedService has ever started.
_hostStartedSource.TrySetResult(true);
await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode);
}
finally
{
activeOperation.Dispose();
_hostStartSemaphore.Release();
}
}
}
/// <summary>
/// Creates a new <see cref="ScriptHostStartupOperation"/> and adds it to the <see cref="ActiveOperations"/> collection.
/// </summary>
/// <param name="cancellationToken">A CancellationToken to be linked to <see cref="CancellationTokenSource"/>.</param>
/// <param name="logger">A logger.</param>
/// <param name="parentId">Ther parent operation id, if needed.</param>
/// <returns>A new instance.</returns>
public static ScriptHostStartupOperation Create(CancellationToken cancellationToken, ILogger logger, Guid?parentId = null)
{
var operation = new ScriptHostStartupOperation(cancellationToken, logger, parentId);
_startupOperations.AddOrUpdate(operation, addValue: null, (k, v) => null);
Log.StartupOperationCreated(logger, operation.Id, operation.ParentId);
return(operation);
}
public async Task RestartHostAsync(CancellationToken cancellationToken)
{
if (ShutdownRequested)
{
return;
}
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerRestartService))
{
// Do not invoke a restart if the host has not yet been started. This can lead
// to invalid state.
if (!_hostStarted.IsCompleted)
{
_logger.RestartBeforeStart();
await _hostStarted;
}
_logger.EnteringRestart();
// If anything is mid-startup, cancel it.
_startupLoopTokenSource?.Cancel();
foreach (var startupOperation in ScriptHostStartupOperation.ActiveOperations)
{
_logger.CancelingStartupOperationForRestart(startupOperation.Id);
try
{
startupOperation.CancellationTokenSource.Cancel();
}
catch (ObjectDisposedException)
{
// This can be disposed at any time.
}
}
try
{
await _hostStartSemaphore.WaitAsync();
if (State == ScriptHostState.Stopping || State == ScriptHostState.Stopped)
{
_logger.SkipRestart(State.ToString());
return;
}
State = ScriptHostState.Default;
_logger.Restarting();
var previousHost = ActiveHost;
ActiveHost = null;
using (var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger))
{
Task startTask, stopTask;
// If we are running in development mode with core tools, do not overlap the restarts.
// Overlapping restarts are problematic when language worker processes are listening
// to the same debug port
if (ShouldEnforceSequentialRestart())
{
stopTask = Orphan(previousHost, cancellationToken);
await stopTask;
startTask = UnsynchronizedStartHostAsync(activeOperation);
}
else
{
startTask = UnsynchronizedStartHostAsync(activeOperation);
stopTask = Orphan(previousHost, cancellationToken);
}
await startTask;
}
_logger.Restarted();
}
catch (OperationCanceledException)
{
if (cancellationToken.IsCancellationRequested)
{
_logger.ScriptHostServiceRestartCanceledByRuntime();
throw;
}
// If the exception was triggered by our startup operation cancellation token, just ignore as
// it doesn't indicate an issue.
}
finally
{
_hostStartSemaphore.Release();
}
}
}
/// <summary>
/// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore
/// before calling this method. Host starts and restarts must be synchronous to prevent orphaned
/// hosts or an incorrect ActiveHost.
/// </summary>
private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
IHost localHost = null;
var currentCancellationToken = activeOperation.CancellationTokenSource.Token;
_logger.StartupOperationStarting(activeOperation.Id);
try
{
currentCancellationToken.ThrowIfCancellationRequested();
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
bool isOffline = Utility.CheckAppOffline(_environment, _applicationHostOptions.CurrentValue.ScriptPath);
State = isOffline ? ScriptHostState.Offline : State;
bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError);
// If we're in a non-transient error state or offline, skip host initialization
bool skipJobHostStartup = isOffline || hasNonTransientErrors;
bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError;
_logger.Building(skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost))
{
localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration);
}
ActiveHost = localHost;
var scriptHost = (ScriptHost)ActiveHost.Services.GetService <ScriptHost>();
if (scriptHost != null)
{
scriptHost.HostInitializing += OnHostInitializing;
if (!handlingError)
{
// Services may be initialized, but we don't want set the state to Initialized as we're
// handling an error and want to retain the Error state.
scriptHost.HostInitialized += OnHostInitialized;
}
}
LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id);
if (!_scriptWebHostEnvironment.InStandbyMode)
{
// At this point we know that App Insights is initialized (if being used), so we
// can dispose this early request tracking module, which forces our new one to take over.
DisposeRequestTrackingModule();
}
currentCancellationToken.ThrowIfCancellationRequested();
var hostInstanceId = GetHostInstanceId(localHost);
_logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost))
{
await localHost.StartAsync(currentCancellationToken);
}
if (!handlingError)
{
LastError = null;
if (!isOffline)
{
State = ScriptHostState.Running;
}
}
}
catch (OperationCanceledException)
{
GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id);
throw;
}
catch (Exception exc)
{
bool isActiveHost = ReferenceEquals(localHost, ActiveHost);
ILogger logger = GetHostLogger(localHost);
if (isActiveHost)
{
LastError = exc;
State = ScriptHostState.Error;
logger.ErrorOccuredDuringStartupOperation(activeOperation.Id, exc);
}
else
{
// Another host has been created before this host
// threw its startup exception. We want to make sure it
// doesn't control the state of the service.
logger.ErrorOccuredInactive(activeOperation.Id, exc);
}
attemptCount++;
if (ShutdownHostIfUnhealthy())
{
return;
}
if (isActiveHost)
{
// We don't want to return disposed services via the Services property, so
// set this to null before calling Orphan().
ActiveHost = null;
}
var orphanTask = Orphan(localHost)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
// Use the fallback logger now, as we cannot trust when the host
// logger will be disposed.
logger = _logger;
if (currentCancellationToken.IsCancellationRequested)
{
logger.CancellationRequested(activeOperation.Id);
currentCancellationToken.ThrowIfCancellationRequested();
}
var nextStartupAttemptMode = JobHostStartupMode.Normal;
if (exc is HostConfigurationException)
{
// Try starting the host without parsing host.json. This will start up a
// minimal host and allow the portal to see the error. Any modification will restart again.
nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
}
else if (exc is HostInitializationException)
{
nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
}
if (nextStartupAttemptMode != JobHostStartupMode.Normal)
{
logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id);
}
else
{
logger.LogDebug($"Will start a new host after delay.");
await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2), logger : logger);
if (currentCancellationToken.IsCancellationRequested)
{
logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started.");
currentCancellationToken.ThrowIfCancellationRequested();
}
logger.LogDebug("Starting new host after delay.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id);
}
}
}
