/// <summary>
/// Starts a worker that watches the dead man's switch.
/// When no notifications are received within the proper timeout period, the <see cref="CancellationToken" /> will be cancelled automatically.
/// You should pass this cancellation token to any worker that must be cancelled.
/// </summary>
/// <returns>A value indicating whether the dead man's switch triggered or not</returns>
public async Task WatchAsync(CancellationToken cancellationToken)
{
_logger.Debug("Watching dead man's switch");
while (!cancellationToken.IsCancellationRequested)
{
TimeSpan timeSinceLastNotification;
if (!_context.IsSuspended)
{
timeSinceLastNotification = TimeSpan.FromTicks(DateTime.UtcNow.Ticks - _context.LastNotifiedTicks);
;
if (timeSinceLastNotification > _options.Timeout)
{
_deadManSwitchTriggerer.Trigger();
return;
}
}
else
{
_logger.Debug("The dead man's switch is suspended. The worker will not be cancelled until the dead man's switch is resumed");
timeSinceLastNotification = TimeSpan.Zero;
}
var timeRemaining = _options.Timeout - timeSinceLastNotification;
await Task.Delay(timeRemaining, cancellationToken)
.ConfigureAwait(false);
}
_logger.Debug("Dead man switch watcher was canceled");
}
/// <inheritdoc />
public void Notify(string notification)
{
if (notification == null)
{
throw new ArgumentNullException(nameof(notification));
}
_logger.Debug("The dead man's switch received a notification: {Notification}", notification);
_context.AddNotification(new DeadManSwitchNotification(notification));
}
/// <inheritdoc />
public async ValueTask NotifyAsync(string notification, CancellationToken cancellationToken = default)
{
if (notification == null)
{
throw new ArgumentNullException(nameof(notification));
}
_logger.Debug("The dead man's switch received a notification: {Notification}", notification);
var enqueueStatus = _context.EnqueueStatusAsync(DeadManSwitchStatus.NotificationReceived, cancellationToken);
var addNotification = _context.AddNotificationAsync(new DeadManSwitchNotification(notification), cancellationToken);
await enqueueStatus.ConfigureAwait(false);
await addNotification.ConfigureAwait(false);
}
/// <inheritdoc />
public async Task RunAsync(IInfiniteDeadManSwitchWorker worker, DeadManSwitchOptions options, CancellationToken cancellationToken)
{
if (worker == null)
{
throw new ArgumentNullException(nameof(worker));
}
_logger.Trace("Starting infinite worker loop for {WorkerName} using a dead man's switch", worker.Name);
using (var deadManSwitchSession = _deadManSwitchSessionFactory.Create(options))
using (var watcherCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
var deadManSwitch = deadManSwitchSession.DeadManSwitch;
var deadManSwitchWatcher = deadManSwitchSession.DeadManSwitchWatcher;
var deadManSwitchContext = deadManSwitchSession.DeadManSwitchContext;
var watcherTask = Task.Factory.StartNew(() => deadManSwitchWatcher.WatchAsync(watcherCTS.Token), CancellationToken.None, TaskCreationOptions.LongRunning,
TaskScheduler.Default);
var iteration = 1;
while (!cancellationToken.IsCancellationRequested)
{
using (var workerCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, deadManSwitchContext.CancellationToken))
{
_logger.Trace("Beginning work iteration {Iteration} of infinite worker {WorkerName} using a dead man's switch", iteration, worker.Name);
var workerTask = Task.Run(() => worker.WorkAsync(deadManSwitch, workerCTS.Token), CancellationToken.None);
try
{
await workerTask.ConfigureAwait(false);
_logger.Debug("Worker {WorkerName} completed gracefully", worker.Name);
deadManSwitch.Notify("Worker task completed gracefully");
}
catch (OperationCanceledException)
{
_logger.Warning("Worker {WorkerName} was canceled", worker.Name);
// Restart watcher
await watcherTask.ConfigureAwait(false);
deadManSwitch.Notify("Worker task was canceled");
watcherTask = Task.Factory.StartNew(() => deadManSwitchWatcher.WatchAsync(watcherCTS.Token), CancellationToken.None, TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
}
iteration++;
}
_logger.Information("Cancellation requested, cleaning up infinite worker loop for {WorkerName}", worker.Name);
watcherCTS.Cancel();
await watcherTask.ConfigureAwait(false);
}
_logger.Trace("Infinite worker loop for {WorkerName} has stopped", worker.Name);
}
/// <summary>
/// Starts a worker that watches the dead man's switch.
/// When no notifications are received within the proper timeout period, the <see cref="CancellationToken"/> will be cancelled automatically.
/// You should pass this cancellation token to any worker that must be cancelled.
/// </summary>
/// <returns>A value indicating whether the dead man's switch triggered or not</returns>
public async ValueTask WatchAsync(CancellationToken cancellationToken)
{
_logger.Debug("Watching dead man's switch");
var status = DeadManSwitchStatus.NotificationReceived;
while (!cancellationToken.IsCancellationRequested)
{
if (status == DeadManSwitchStatus.Suspended)
{
_logger.Debug("The dead man's switch is suspended. The worker will not be cancelled until the dead man's switch is resumed");
// ignore any notifications and wait until the switch goes through the 'Resumed' status
while (status != DeadManSwitchStatus.Resumed)
{
try
{
status = await _context.DequeueStatusAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
_logger.Debug("Dead man switch was canceled while waiting to be resumed.");
return;
}
}
_logger.Debug("The dead man's switch is now resuming.");
}
using (var timeoutCTS = new CancellationTokenSource(_options.Timeout))
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCTS.Token))
{
try
{
status = await _context.DequeueStatusAsync(cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
if (cancellationToken.IsCancellationRequested)
{
_logger.Debug("Dead man switch watcher was canceled while waiting for the next notification");
return;
}
if (timeoutCTS.IsCancellationRequested)
{
await _deadManSwitchTriggerer.TriggerAsync(cancellationToken).ConfigureAwait(false);
return;
}
}
}
}
_logger.Debug("Dead man switch watcher was canceled");
}