private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0,
JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null)
{
// Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations.
var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService))
{
try
{
await _hostStartSemaphore.WaitAsync();
// Now that we're inside the semaphore, set this task as completed. This prevents
// restarts from being invoked (via the PlaceholderSpecializationMiddleware) before
// the IHostedService has ever started.
_hostStartedSource.TrySetResult(true);
await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode);
}
finally
{
activeOperation.Dispose();
_hostStartSemaphore.Release();
}
}
}
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath);
State = isOffline ? ScriptHostState.Offline : State;
bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
// If we're in a non-transient error state or offline, skip host initialization
bool skipJobHostStartup = isOffline || hasNonTransientErrors;
_host = BuildHost(skipJobHostStartup, skipHostJsonConfiguration: startupMode == JobHostStartupMode.HandlingConfigurationParsingError);
var scriptHost = (ScriptHost)_host.Services.GetService <ScriptHost>();
if (scriptHost != null)
{
scriptHost.HostInitializing += OnHostInitializing;
}
LogInitialization(isOffline, attemptCount, ++_hostStartCount);
await _host.StartAsync(cancellationToken);
if (!startupMode.HasFlag(JobHostStartupMode.HandlingError))
{
LastError = null;
if (!isOffline)
{
State = ScriptHostState.Running;
}
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exc)
{
LastError = exc;
State = ScriptHostState.Error;
attemptCount++;
ILogger logger = GetHostLogger();
logger.LogError(exc, "A host error has occurred");
if (ShutdownHostIfUnhealthy())
{
return;
}
var orphanTask = Orphan(_host, logger)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
cancellationToken.ThrowIfCancellationRequested();
var nextStartupAttemptMode = JobHostStartupMode.Normal;
if (exc is HostConfigurationException)
{
// Try starting the host without parsing host.json. This will start up a
// minimal host and allow the portal to see the error. Any modification will restart again.
nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
}
else if (exc is HostInitializationException)
{
nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
}
if (nextStartupAttemptMode != JobHostStartupMode.Normal)
{
Task ignore = StartHostAsync(cancellationToken, attemptCount, nextStartupAttemptMode);
}
else
{
await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2))
.ContinueWith(t =>
{
cancellationToken.ThrowIfCancellationRequested();
return(StartHostAsync(cancellationToken, attemptCount));
});
}
}
}
/// <summary>
/// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore
/// before calling this method. Host starts and restarts must be synchronous to prevent orphaned
/// hosts or an incorrect ActiveHost.
/// </summary>
private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
IHost localHost = null;
var currentCancellationToken = activeOperation.CancellationTokenSource.Token;
_logger.StartupOperationStarting(activeOperation.Id);
try
{
currentCancellationToken.ThrowIfCancellationRequested();
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
bool isOffline = Utility.CheckAppOffline(_environment, _applicationHostOptions.CurrentValue.ScriptPath);
State = isOffline ? ScriptHostState.Offline : State;
bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError);
// If we're in a non-transient error state or offline, skip host initialization
bool skipJobHostStartup = isOffline || hasNonTransientErrors;
bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError;
_logger.Building(skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost))
{
localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration);
}
ActiveHost = localHost;
var scriptHost = (ScriptHost)ActiveHost.Services.GetService <ScriptHost>();
if (scriptHost != null)
{
scriptHost.HostInitializing += OnHostInitializing;
if (!handlingError)
{
// Services may be initialized, but we don't want set the state to Initialized as we're
// handling an error and want to retain the Error state.
scriptHost.HostInitialized += OnHostInitialized;
}
}
LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id);
if (!_scriptWebHostEnvironment.InStandbyMode)
{
// At this point we know that App Insights is initialized (if being used), so we
// can dispose this early request tracking module, which forces our new one to take over.
DisposeRequestTrackingModule();
}
currentCancellationToken.ThrowIfCancellationRequested();
var hostInstanceId = GetHostInstanceId(localHost);
_logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost))
{
await localHost.StartAsync(currentCancellationToken);
}
if (!handlingError)
{
LastError = null;
if (!isOffline)
{
State = ScriptHostState.Running;
}
}
}
catch (OperationCanceledException)
{
GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id);
throw;
}
catch (Exception exc)
{
bool isActiveHost = ReferenceEquals(localHost, ActiveHost);
ILogger logger = GetHostLogger(localHost);
if (isActiveHost)
{
LastError = exc;
State = ScriptHostState.Error;
logger.ErrorOccuredDuringStartupOperation(activeOperation.Id, exc);
}
else
{
// Another host has been created before this host
// threw its startup exception. We want to make sure it
// doesn't control the state of the service.
logger.ErrorOccuredInactive(activeOperation.Id, exc);
}
attemptCount++;
if (ShutdownHostIfUnhealthy())
{
return;
}
if (isActiveHost)
{
// We don't want to return disposed services via the Services property, so
// set this to null before calling Orphan().
ActiveHost = null;
}
var orphanTask = Orphan(localHost)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
// Use the fallback logger now, as we cannot trust when the host
// logger will be disposed.
logger = _logger;
if (currentCancellationToken.IsCancellationRequested)
{
logger.CancellationRequested(activeOperation.Id);
currentCancellationToken.ThrowIfCancellationRequested();
}
var nextStartupAttemptMode = JobHostStartupMode.Normal;
if (exc is HostConfigurationException)
{
// Try starting the host without parsing host.json. This will start up a
// minimal host and allow the portal to see the error. Any modification will restart again.
nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
}
else if (exc is HostInitializationException)
{
nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
}
if (nextStartupAttemptMode != JobHostStartupMode.Normal)
{
logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id);
}
else
{
logger.LogDebug($"Will start a new host after delay.");
await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2), logger : logger);
if (currentCancellationToken.IsCancellationRequested)
{
logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started.");
currentCancellationToken.ThrowIfCancellationRequested();
}
logger.LogDebug("Starting new host after delay.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id);
}
}
}
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
cancellationToken.ThrowIfCancellationRequested();
IHost localHost = null;
try
{
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath);
State = isOffline ? ScriptHostState.Offline : State;
bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
// If we're in a non-transient error state or offline, skip host initialization
bool skipJobHostStartup = isOffline || hasNonTransientErrors;
localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration: startupMode == JobHostStartupMode.HandlingConfigurationParsingError);
_host = localHost;
var scriptHost = (ScriptHost)_host.Services.GetService <ScriptHost>();
if (scriptHost != null)
{
scriptHost.HostInitializing += OnHostInitializing;
}
LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount);
if (!_scriptWebHostEnvironment.InStandbyMode)
{
// At this point we know that App Insights is initialized (if being used), so we
// can dispose this early request tracking module, which forces our new one to take over.
DisposeRequestTrackingModule();
}
await _host.StartAsync(cancellationToken);
if (!startupMode.HasFlag(JobHostStartupMode.HandlingError))
{
LastError = null;
if (!isOffline)
{
State = ScriptHostState.Running;
}
}
}
catch (OperationCanceledException)
{
GetHostLogger(localHost).LogDebug("Host startup was canceled.");
throw;
}
catch (Exception exc)
{
bool isActiveHost = ReferenceEquals(localHost, _host);
ILogger logger = GetHostLogger(localHost);
if (isActiveHost)
{
LastError = exc;
State = ScriptHostState.Error;
logger.LogError(exc, "A host error has occurred");
}
else
{
// Another host has been created before this host
// threw its startup exception. We want to make sure it
// doesn't control the state of the service.
logger.LogWarning(exc, "A host error has occurred on an inactive host");
}
attemptCount++;
if (ShutdownHostIfUnhealthy())
{
return;
}
var orphanTask = Orphan(localHost, logger)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
if (cancellationToken.IsCancellationRequested)
{
logger.LogDebug($"Cancellation requested. A new host will not be started.");
cancellationToken.ThrowIfCancellationRequested();
}
var nextStartupAttemptMode = JobHostStartupMode.Normal;
if (exc is HostConfigurationException)
{
// Try starting the host without parsing host.json. This will start up a
// minimal host and allow the portal to see the error. Any modification will restart again.
nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
}
else if (exc is HostInitializationException)
{
nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
}
if (nextStartupAttemptMode != JobHostStartupMode.Normal)
{
Task ignore = StartHostAsync(cancellationToken, attemptCount, nextStartupAttemptMode);
}
else
{
await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min : TimeSpan.FromSeconds(1), max : TimeSpan.FromMinutes(2))
.ContinueWith(t =>
{
cancellationToken.ThrowIfCancellationRequested();
return(StartHostAsync(cancellationToken, attemptCount));
});
}
}
}
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
try
{
await _hostStartSemaphore.WaitAsync();
// Now that we're inside the semaphore, set this task as completed. This prevents
// restarts from being invoked (via the PlaceholderSpecializationMiddleware) before
// the IHostedService has ever started.
_hostStartedSource.TrySetResult(true);
await UnsynchronizedStartHostAsync(cancellationToken, attemptCount, startupMode);
}
finally
{
_hostStartSemaphore.Release();
}
}
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
try
{
await _hostStartSemaphore.WaitAsync();
await UnsynchronizedStartHostAsync(cancellationToken, attemptCount, startupMode);
}
finally
{
_hostStartSemaphore.Release();
}
}
