public IBackgroundDispatcher Create(BackgroundServerContext context, BackgroundProcessingServerOptions options)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
var execution = new BackgroundExecution(
context.StoppingToken,
new BackgroundExecutionOptions
{
Name = _process.GetType().Name,
RetryDelay = options.RetryDelay
});
return(new BackgroundDispatcherAsync(
execution,
ExecuteProcess,
Tuple.Create(_process, context, execution),
_taskScheduler(),
_maxConcurrency,
_ownsScheduler));
}
private IBackgroundDispatcher CreateHeartbeatProcess(BackgroundServerContext context, Action requestRestart)
{
return(new ServerHeartbeatProcess(_options.HeartbeatInterval, _options.ServerTimeout, requestRestart)
.UseBackgroundPool(threadCount: 1
#if !NETSTANDARD1_3
, thread => { thread.Priority = ThreadPriority.AboveNormal; }
#endif
)
.Create(context, _options));
}
private void StartDispatchers(BackgroundServerContext context, ICollection <IBackgroundDispatcher> dispatchers)
{
if (_dispatcherBuilders.Length == 0)
{
throw new InvalidOperationException("No dispatchers registered for the processing server.");
}
_logger.Info($"{GetServerTemplate(context.ServerId)} is starting the registered dispatchers: {String.Join(", ", _dispatcherBuilders.Select(builder => $"{builder}"))}...");
foreach (var dispatcherBuilder in _dispatcherBuilders)
{
dispatchers.Add(dispatcherBuilder.Create(context, _options));
}
_logger.Info($"{GetServerTemplate(context.ServerId)} all the dispatchers started");
}
private void CreateServer(BackgroundServerContext context)
{
_logger.Trace($"{GetServerTemplate(context.ServerId)} is announcing itself...");
var stopwatch = Stopwatch.StartNew();
using (var connection = _storage.GetConnection())
{
connection.AnnounceServer(context.ServerId, GetServerContext(_properties));
}
stopwatch.Stop();
ServerJobCancellationToken.AddServer(context.ServerId);
_logger.Info($"{GetServerTemplate(context.ServerId)} successfully announced in {stopwatch.Elapsed.TotalMilliseconds} ms");
}
private void WaitForDispatchers(
BackgroundServerContext context,
IReadOnlyList <IBackgroundDispatcher> dispatchers)
{
if (dispatchers.Count == 0)
{
return;
}
var waitTasks = new Task[dispatchers.Count];
for (var i = 0; i < dispatchers.Count; i++)
{
waitTasks[i] = dispatchers[i].WaitAsync(Timeout.InfiniteTimeSpan, CancellationToken.None);
}
var nonStopped = new List <IBackgroundDispatcher>();
try
{
Task.WaitAll(waitTasks, context.ShutdownToken);
}
catch (OperationCanceledException)
{
for (var i = 0; i < dispatchers.Count; i++)
{
if (waitTasks[i].Status != TaskStatus.RanToCompletion)
{
nonStopped.Add(dispatchers[i]);
}
}
}
if (nonStopped.Count > 0)
{
var nonStoppedNames = nonStopped.Select(dispatcher => $"{dispatcher.ToString()}").ToArray();
_logger.Warn($"{GetServerTemplate(context.ServerId)} stopped non-gracefully due to {String.Join(", ", nonStoppedNames)}. Outstanding work on those dispatchers could be aborted, and there can be delays in background processing. This server instance will be incorrectly shown as active for a while. To avoid non-graceful shutdowns, investigate what prevents from stopping gracefully and add CancellationToken support for those methods.");
}
else
{
_logger.Info($"{GetServerTemplate(context.ServerId)} All dispatchers stopped");
}
}
public IBackgroundDispatcher Create(BackgroundServerContext context, BackgroundProcessingServerOptions options)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
return(new BackgroundDispatcher(
new BackgroundExecution(context.StoppingToken, new BackgroundExecutionOptions
{
Name = _component.GetType().Name,
RetryDelay = options.RetryDelay
}),
ExecuteComponent,
Tuple.Create(_component, context),
_threadFactory));
}
private void ServerDelete(BackgroundServerContext context, Stopwatch stoppedAt)
{
try
{
_logger.Trace($"{GetServerTemplate(context.ServerId)} is reporting itself as stopped...");
ServerJobCancellationToken.RemoveServer(context.ServerId);
var stopwatch = Stopwatch.StartNew();
using (var connection = _storage.GetConnection())
{
connection.RemoveServer(context.ServerId);
}
stopwatch.Stop();
_logger.Info($"{GetServerTemplate(context.ServerId)} successfully reported itself as stopped in {stopwatch.Elapsed.TotalMilliseconds} ms");
_logger.Info($"{GetServerTemplate(context.ServerId)} has been stopped in total {stoppedAt?.Elapsed.TotalMilliseconds ?? 0} ms");
}
catch (Exception ex)
{
_logger.WarnException($"{GetServerTemplate(context.ServerId)} there was an exception, server may not be removed", ex);
}
}
public void Execute(Guid executionId, BackgroundExecution execution, CancellationToken stoppingToken,
CancellationToken stoppedToken, CancellationToken shutdownToken)
{
var serverId = GetServerId();
Stopwatch stoppedAt = null;
void HandleStopRestartSignal() => Interlocked.CompareExchange(ref stoppedAt, Stopwatch.StartNew(), null);
void HandleStoppingSignal() => _logger.Info($"{GetServerTemplate(serverId)} caught stopping signal...");
void HandleStoppedSignal() => _logger.Info($"{GetServerTemplate(serverId)} caught stopped signal...");
void HandleShutdownSignal() => _logger.Warn($"{GetServerTemplate(serverId)} caught shutdown signal...");
void HandleRestartSignal()
{
if (!stoppingToken.IsCancellationRequested)
{
_logger.Info($"{GetServerTemplate(serverId)} caught restart signal...");
}
}
//using (LogProvider.OpenMappedContext("ServerId", serverId.ToString()))
using (var restartCts = new CancellationTokenSource())
using (var restartStoppingCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, restartCts.Token))
using (var restartStoppedCts = CancellationTokenSource.CreateLinkedTokenSource(stoppedToken, restartCts.Token))
using (var restartShutdownCts = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken, restartCts.Token))
using (restartStoppingCts.Token.Register(HandleStopRestartSignal))
using (stoppingToken.Register(HandleStoppingSignal))
using (stoppedToken.Register(HandleStoppedSignal))
using (shutdownToken.Register(HandleShutdownSignal))
using (restartCts.Token.Register(HandleRestartSignal))
{
var context = new BackgroundServerContext(
serverId,
_storage,
_properties,
restartStoppingCts.Token,
restartStoppedCts.Token,
restartShutdownCts.Token);
var dispatchers = new List <IBackgroundDispatcher>();
CreateServer(context);
try
{
// ReSharper disable once AccessToDisposedClosure
using (var heartbeat = CreateHeartbeatProcess(context, () => restartCts.Cancel()))
{
StartDispatchers(context, dispatchers);
execution.NotifySucceeded();
WaitForDispatchers(context, dispatchers);
restartCts.Cancel();
// TODO Either modify the IBackgroundDispatcher.Wait method to handle CancellationToken
// or expose the WaitHandle property to not to perform sync-over-async and vice versa
// in 2.0.
heartbeat.WaitAsync(Timeout.InfiniteTimeSpan, shutdownToken).GetAwaiter().GetResult();
}
}
finally
{
DisposeDispatchers(dispatchers);
ServerDelete(context, stoppedAt);
}
}
}
private IBackgroundDispatcher CreateHeartbeatProcess(BackgroundServerContext context, Action requestRestart)
{
return(new ServerHeartbeatProcess(_options.HeartbeatInterval, _options.ServerTimeout, requestRestart)
.UseBackgroundPool(threadCount: 1)
.Create(context, _options));
}
