public void RunAndBlock(CancellationToken cancellationToken = default(CancellationToken))
{
int consecutiveErrorCount = 0;
do
{
ScriptHost newInstance = null;
try
{
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
// Create a new host config, but keep the host id from existing one
_config.HostConfig = new JobHostConfiguration
{
HostId = _config.HostConfig.HostId
};
OnInitializeConfig(_config);
newInstance = _scriptHostFactory.Create(_environment, _settingsManager, _config);
_traceWriter = newInstance.TraceWriter;
_currentInstance = newInstance;
lock (_liveInstances)
{
_liveInstances.Add(newInstance);
}
OnHostCreated();
if (_traceWriter != null)
{
string message = string.Format("Starting Host (HostId={0}, Version={1}, ProcessId={2}, Debug={3}, Attempt={4})",
newInstance.ScriptConfig.HostConfig.HostId, ScriptHost.Version, Process.GetCurrentProcess().Id, newInstance.InDebugMode.ToString(), consecutiveErrorCount);
_traceWriter.Info(message);
}
newInstance.StartAsync(cancellationToken).GetAwaiter().GetResult();
// log any function initialization errors
LogErrors(newInstance);
OnHostStarted();
// only after ALL initialization is complete do we set the
// state to Running
State = ScriptHostState.Running;
LastError = null;
consecutiveErrorCount = 0;
_restartDelayTokenSource = null;
// Wait for a restart signal. This event will automatically reset.
// While we're restarting, it is possible for another restart to be
// signaled. That is fine - the restart will be processed immediately
// once we get to this line again. The important thing is that these
// restarts are only happening on a single thread.
WaitHandle.WaitAny(new WaitHandle[]
{
cancellationToken.WaitHandle,
_restartHostEvent,
_stopEvent
});
// Orphan the current host instance. We're stopping it, so it won't listen for any new functions
// it will finish any currently executing functions and then clean itself up.
// Spin around and create a new host instance.
#pragma warning disable 4014
Orphan(newInstance);
#pragma warning restore 4014
}
catch (Exception ex)
{
State = ScriptHostState.Error;
LastError = ex;
consecutiveErrorCount++;
// We need to keep the host running, so we catch and log any errors
// then restart the host
if (_traceWriter != null)
{
_traceWriter.Error("A ScriptHost error has occurred", ex);
}
// If a ScriptHost instance was created before the exception was thrown
// Orphan and cleanup that instance.
if (newInstance != null)
{
Orphan(newInstance, forceStop: true)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
}
// attempt restarts using an exponential backoff strategy
CreateRestartBackoffDelay(consecutiveErrorCount).GetAwaiter().GetResult();
}
}while (!_stopped && !cancellationToken.IsCancellationRequested);
}
public void RunAndBlock(CancellationToken cancellationToken = default(CancellationToken))
{
_consecutiveErrorCount = 0;
do
{
ScriptHost newInstance = null;
try
{
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
// Create a new host config, but keep the host id from existing one
_config.HostConfig = new JobHostConfiguration(_settingsManager.Configuration)
{
HostId = _config.HostConfig.HostId
};
OnInitializeConfig(_config);
newInstance = _scriptHostFactory.Create(_environment, EventManager, _settingsManager, _config, _loggerProviderFactory);
_currentInstance = newInstance;
lock (_liveInstances)
{
_liveInstances.Add(newInstance);
_hostStartCount++;
}
newInstance.HostInitializing += OnHostInitializing;
newInstance.HostInitialized += OnHostInitialized;
newInstance.HostStarted += OnHostStarted;
newInstance.Initialize();
newInstance.StartAsync(cancellationToken).GetAwaiter().GetResult();
// log any function initialization errors
LogErrors(newInstance);
LastError = null;
_consecutiveErrorCount = 0;
_restartDelayTokenSource = null;
// Wait for a restart signal. This event will automatically reset.
// While we're restarting, it is possible for another restart to be
// signaled. That is fine - the restart will be processed immediately
// once we get to this line again. The important thing is that these
// restarts are only happening on a single thread.
var waitHandles = new WaitHandle[] { cancellationToken.WaitHandle, _restartHostEvent, _stopEvent };
if (!waitHandles.Any(p => p.SafeWaitHandle.IsClosed))
{
WaitHandle.WaitAny(waitHandles);
}
// Orphan the current host instance. We're stopping it, so it won't listen for any new functions
// it will finish any currently executing functions and then clean itself up.
// Spin around and create a new host instance.
Task.Run(() => Orphan(newInstance)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously));
}
catch (Exception ex)
{
if (_disposed)
{
// In some cases during shutdown we'll be disposed and get
// some terminating exceptions. We want to just ignore these
// and stop immediately.
break;
}
State = ScriptHostState.Error;
LastError = ex;
_consecutiveErrorCount++;
// We need to keep the host running, so we catch and log any errors
// then restart the host
string message = "A ScriptHost error has occurred";
Instance?.Logger.LogError(0, ex, message);
if (ShutdownHostIfUnhealthy())
{
break;
}
// If a ScriptHost instance was created before the exception was thrown
// Orphan and cleanup that instance.
if (newInstance != null)
{
Task.Run(() => Orphan(newInstance, forceStop: true)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously));
}
// attempt restarts using an exponential backoff strategy
CreateRestartBackoffDelay(_consecutiveErrorCount).GetAwaiter().GetResult();
}
}while (!_stopped && !_disposed && !cancellationToken.IsCancellationRequested);
}
public void RunAndBlock(CancellationToken cancellationToken = default(CancellationToken))
{
// Start the host and restart it if requested. Host Restarts will happen when
// host level configuration files change
do
{
ScriptHost newInstance = null;
try
{
IsRunning = false;
// Create a new host config, but keep the host id from existing one
_config.HostConfig = new JobHostConfiguration
{
HostId = _config.HostConfig.HostId
};
OnInitializeConfig(_config.HostConfig);
newInstance = _scriptHostFactory.Create(_config);
_traceWriter = newInstance.TraceWriter;
newInstance.StartAsync(cancellationToken).GetAwaiter().GetResult();
// write any function initialization errors to the log file
LogErrors(newInstance);
lock (_liveInstances)
{
_liveInstances.Add(newInstance);
}
_currentInstance = newInstance;
OnHostStarted();
// only after ALL initialization is complete do we set this flag
IsRunning = true;
LastError = null;
// Wait for a restart signal. This event will automatically reset.
// While we're restarting, it is possible for another restart to be
// signaled. That is fine - the restart will be processed immediately
// once we get to this line again. The important thing is that these
// restarts are only happening on a single thread.
WaitHandle.WaitAny(new WaitHandle[]
{
cancellationToken.WaitHandle,
newInstance.RestartEvent,
_stopEvent
});
// Orphan the current host instance. We're stopping it, so it won't listen for any new functions
// it will finish any currently executing functions and then clean itself up.
// Spin around and create a new host instance.
Task taskIgnore = Orphan(newInstance);
}
catch (Exception ex)
{
IsRunning = false;
LastError = ex;
// We need to keep the host running, so we catch and log any errors
// then restart the host
if (_traceWriter != null)
{
_traceWriter.Error("A ScriptHost error has occurred", ex);
}
// If a ScriptHost instance was created before the exception was thrown
// Orphan and cleanup that instance.
if (newInstance != null)
{
Orphan(newInstance, forceStop: true)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
});
}
// Wait for a short period of time before restarting to
// avoid cases where a host level config error might cause
// a rapid restart cycle
Task.Delay(5000).GetAwaiter().GetResult();
}
}while (!_stopped && !cancellationToken.IsCancellationRequested);
}
public void RunAndBlock(CancellationToken cancellationToken = default(CancellationToken))
{
int consecutiveErrorCount = 0;
do
{
ScriptHost newInstance = null;
try
{
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
OnCreatingHost();
// Create a new host config, but keep the host id from existing one
_config.HostConfig = new JobHostConfiguration(_settingsManager.Configuration)
{
HostId = _config.HostConfig.HostId
};
OnInitializeConfig(_config);
newInstance = _scriptHostFactory.Create(_environment, EventManager, _settingsManager, _config, _loggerFactoryBuilder);
newInstance.HostInitialized += OnHostInitialized;
newInstance.HostStarted += OnHostStarted;
_traceWriter = newInstance.TraceWriter;
_logger = newInstance.Logger;
_currentInstance = newInstance;
lock (_liveInstances)
{
_liveInstances.Add(newInstance);
_hostStartCount++;
}
string extensionVersion = _settingsManager.GetSetting(EnvironmentSettingNames.FunctionsExtensionVersion);
string hostId = newInstance.ScriptConfig.HostConfig.HostId;
string message = $"Starting Host (HostId={hostId}, Version={ScriptHost.Version}, ProcessId={Process.GetCurrentProcess().Id}, Debug={newInstance.InDebugMode}, ConsecutiveErrors={consecutiveErrorCount}, StartupCount={_hostStartCount}, FunctionsExtensionVersion={extensionVersion})";
_traceWriter?.Info(message);
_logger?.LogInformation(message);
newInstance.StartAsync(cancellationToken).GetAwaiter().GetResult();
// log any function initialization errors
LogErrors(newInstance);
LastError = null;
consecutiveErrorCount = 0;
_restartDelayTokenSource = null;
// Wait for a restart signal. This event will automatically reset.
// While we're restarting, it is possible for another restart to be
// signaled. That is fine - the restart will be processed immediately
// once we get to this line again. The important thing is that these
// restarts are only happening on a single thread.
WaitHandle.WaitAny(new WaitHandle[]
{
cancellationToken.WaitHandle,
_restartHostEvent,
_stopEvent
});
// Orphan the current host instance. We're stopping it, so it won't listen for any new functions
// it will finish any currently executing functions and then clean itself up.
// Spin around and create a new host instance.
Task.Run(() => Orphan(newInstance)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously));
}
catch (Exception ex)
{
State = ScriptHostState.Error;
LastError = ex;
consecutiveErrorCount++;
// We need to keep the host running, so we catch and log any errors
// then restart the host
string message = "A ScriptHost error has occurred";
_traceWriter?.Error(message, ex);
_logger?.LogError(0, ex, message);
// If a ScriptHost instance was created before the exception was thrown
// Orphan and cleanup that instance.
if (newInstance != null)
{
Task.Run(() => Orphan(newInstance, forceStop: true)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously));
}
// attempt restarts using an exponential backoff strategy
CreateRestartBackoffDelay(consecutiveErrorCount).GetAwaiter().GetResult();
}
}while (!_stopped && !cancellationToken.IsCancellationRequested);
}