private Task ProcessErrorHandler(ProcessErrorEventArgs args)
{
try
{
var eventHubsException = (args.Exception as EventHubsException);
eventHubsException?.TrackMetrics(Metrics);
if (eventHubsException == null)
{
Interlocked.Increment(ref Metrics.GeneralExceptions);
}
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
ErrorsObserved.Add(args.Exception);
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
return(Task.CompletedTask);
}
private Task ProcessEventHandler(string processorId, ProcessEventArgs args)
{
try
{
Interlocked.Increment(ref Metrics.TotalServiceOperations);
Interlocked.Increment(ref Metrics.EventHandlerCalls);
// If there was no event then there is nothing to do.
if (!args.HasEvent)
{
return(Task.CompletedTask);
}
// We're not expecting any events; capture it as unexpected.
Interlocked.Increment(ref Metrics.EventsRead);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, false), EventHubsException.FailureReason.GeneralError));
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
return(Task.CompletedTask);
}
public async Task Start(CancellationToken cancellationToken)
{
IsRunning = true;
using var process = Process.GetCurrentProcess();
using var publishCancellationSource = new CancellationTokenSource();
var publishingTasks = default(IEnumerable <Task>);
var runDuration = Stopwatch.StartNew();
try
{
// Begin publishing events in the background.
publishingTasks = Enumerable
.Range(0, Configuration.ProducerCount)
.Select(_ => Task.Run(() => new Publisher(Configuration, Metrics, ErrorsObserved).Start(publishCancellationSource.Token)))
.ToList();
// Update metrics.
while (!cancellationToken.IsCancellationRequested)
{
Metrics.UpdateEnvironmentStatistics(process);
Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// The run is ending. Clean up the outstanding background operations and
// complete the necessary metrics tracking.
try
{
publishCancellationSource.Cancel();
await Task.WhenAll(publishingTasks).ConfigureAwait(false);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
finally
{
runDuration.Stop();
IsRunning = false;
}
}
public async Task Start(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
var options = new EventHubProducerClientOptions
{
RetryOptions = new EventHubsRetryOptions
{
TryTimeout = Configuration.SendTimeout
}
};
var producer = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub, options);
await using (producer.ConfigureAwait(false))
{
while (!cancellationToken.IsCancellationRequested)
{
// Query partitions each iteration to allow any new partitions to be discovered. Dynamic upscaling of partitions
// is a near-term feature being added to the Event Hubs service.
var partitions = await producer.GetPartitionIdsAsync(cancellationToken).ConfigureAwait(false);
var selectedPartition = partitions[RandomNumberGenerator.Value.Next(0, partitions.Length)];
var batchEvents = new List <EventData>();
// Create the batch and generate a set of random events, keeping only those that were able to fit into the batch.
using var batch = await producer.CreateBatchAsync(new CreateBatchOptions { PartitionId = selectedPartition }).ConfigureAwait(false);
var events = EventGenerator.CreateEvents(
Configuration.PublishBatchSize,
Configuration.LargeMessageRandomFactorPercent,
Configuration.PublishingBodyMinBytes,
Configuration.PublishingBodyRegularMaxBytes,
currentSequence,
selectedPartition);
foreach (var currentEvent in events)
{
if (!batch.TryAdd(currentEvent))
{
break;
}
batchEvents.Add(currentEvent);
}
// Publish the events and report them, capturing any failures specific to the send operation.
try
{
if (batch.Count > 0)
{
await producer.SendAsync(batch, cancellationToken).ConfigureAwait(false);
foreach (var batchEvent in batchEvents)
{
batchEvent.Properties.TryGetValue(EventGenerator.IdPropertyName, out var id);
PublishedEvents.AddOrUpdate(id.ToString(), _ => batchEvent, (k, v) => batchEvent);
}
Interlocked.Add(ref Metrics.EventsPublished, batch.Count);
Interlocked.Increment(ref Metrics.TotalServiceOperations);
}
}
catch (TaskCanceledException)
{
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// Honor any requested delays to throttle publishing so that it doesn't overwhelm slower consumers.
if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
{
await Task.Delay(Configuration.PublishingDelay.Value).ConfigureAwait(false);
}
}
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.ProducerRestarted);
ErrorsObserved.Add(ex);
}
}
}
public async Task Start(CancellationToken cancellationToken)
{
IsRunning = true;
using var process = Process.GetCurrentProcess();
using var publishCancellationSource = new CancellationTokenSource();
using var processorCancellationSource = new CancellationTokenSource();
var publishingTask = default(Task);
var processorTasks = default(IEnumerable <Task>);
var runDuration = Stopwatch.StartNew();
try
{
// Determine the number of partitions in the Event Hub.
int partitionCount;
await using (var producerClient = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub))
{
partitionCount = (await producerClient.GetEventHubPropertiesAsync()).PartitionIds.Length;
}
// Begin publishing events in the background.
publishingTask = Task.Run(() => new Publisher(Configuration, Metrics, PublishedEvents, ErrorsObserved).Start(publishCancellationSource.Token));
// Start processing.
processorTasks = Enumerable
.Range(0, Configuration.ProcessorCount)
.Select(_ => Task.Run(() => new Processor(Configuration, Metrics, partitionCount, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
.ToList();
// Test for missing events and update metrics.
var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);
while (!cancellationToken.IsCancellationRequested)
{
Metrics.UpdateEnvironmentStatistics(process);
Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, eventDueInterval, Metrics);
await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// The run is ending. Clean up the outstanding background operations and
// complete the necessary metrics tracking.
try
{
var publishingStopped = DateTimeOffset.UtcNow;
publishCancellationSource.Cancel();
await publishingTask.ConfigureAwait(false);
// Wait a bit after publishing has completed before signaling for
// processing to be canceled, to allow the recently published
// events to be read.
await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false);
processorCancellationSource.Cancel();
await Task.WhenAll(processorTasks).ConfigureAwait(false);
// Wait a bit after processing has completed before and then perform
// the last bit of bookkeeping, scanning for missing and unexpected events.
await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
ScrubRecentlyPublishedEvents(PublishedEvents, UnexpectedEvents, ReadEvents, publishingStopped.Subtract(TimeSpan.FromMinutes(1)));
ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes), Metrics);
foreach (var unexpectedEvent in UnexpectedEvents.Values)
{
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(unexpectedEvent, false), EventHubsException.FailureReason.GeneralError));
}
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
finally
{
runDuration.Stop();
IsRunning = false;
}
}
private async Task ProcessEventHandler(string processorId, ProcessEventArgs args)
{
try
{
Interlocked.Increment(ref Metrics.TotalServiceOperations);
// If there was no event then there is nothing to do.
if (!args.HasEvent)
{
return;
}
// Determine if the event has an identifier and has been tracked as published.
var hasId = args.Data.Properties.TryGetValue(EventGenerator.IdPropertyName, out var id);
var eventId = (hasId) ? id?.ToString() : null;
var isTrackedEvent = PublishedEvents.TryRemove(eventId, out var publishedEvent);
// If the event has an id that has been seen before, track it as a duplicate processing and
// take no further action.
if ((hasId) && (ReadEvents.ContainsKey(eventId)))
{
Interlocked.Increment(ref Metrics.DuplicateEventsDiscarded);
return;
}
// Since this event wasn't a duplicate, consider it read.
Interlocked.Increment(ref Metrics.EventsRead);
// If there the event isn't a known and published event, then track it but take no
// further action.
if ((!hasId) || (!isTrackedEvent))
{
// If there was an id, then the event wasn't tracked. This is likely a race condition in tracking;
// allow for a small delay and then try to find the event again.
if (hasId)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
isTrackedEvent = PublishedEvents.TryRemove(eventId, out publishedEvent);
}
else
{
// If there wasn't an id then consider it an unexpected event failure.
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
return;
}
// If there was an id, but the event wasn't tracked as published or processed, cache it as an
// unexpected event for later consideration. If it cannot be cached, consider it a failure.
if ((!isTrackedEvent) && (!ReadEvents.ContainsKey(eventId)))
{
if (!UnexpectedEvents.TryAdd(eventId, args.Data))
{
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
}
return;
}
}
// It has been proven that the current event is expected; track it as having been read.
ReadEvents.TryAdd(eventId, 0);
// Validate the event against expectations.
if (!args.Data.IsEquivalentTo(publishedEvent))
{
if (!args.Data.Body.ToArray().SequenceEqual(publishedEvent.Body.ToArray()))
{
Interlocked.Increment(ref Metrics.InvalidBodies);
}
else
{
Interlocked.Increment(ref Metrics.InvalidProperties);
}
}
// Validate that the intended partition that was sent as a property matches the
// partition that the handler was triggered for.
if ((!publishedEvent.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var publishedPartition)) ||
(args.Partition.PartitionId != publishedPartition.ToString()))
{
Interlocked.Increment(ref Metrics.EventsFromWrongPartition);
}
// Validate that the sequence number that was sent as a property is greater than the last read
// sequence number for the partition; if there hasn't been an event for the partition yet, then
// there is no validation possible.
//
// Track the sequence number for future comparisons.
var currentSequence = default(object);
if ((LastReadPartitionSequence.TryGetValue(args.Partition.PartitionId, out var lastReadSequence)) &&
((!publishedEvent.Properties.TryGetValue(EventGenerator.SequencePropertyName, out currentSequence)) || (lastReadSequence >= (long)currentSequence)))
{
Interlocked.Increment(ref Metrics.EventsOutOfOrder);
}
var trackSequence = (currentSequence == default) ? -1 : (long)currentSequence;
LastReadPartitionSequence.AddOrUpdate(args.Partition.PartitionId, _ => trackSequence, (part, seq) => Math.Max(seq, trackSequence));
// Create a checkpoint every 100 events for the partition, just to follow expected patterns.
if (trackSequence % 100 == 0)
{
await args.UpdateCheckpointAsync(args.CancellationToken).ConfigureAwait(false);
}
// Mark the event as processed.
Interlocked.Increment(ref Metrics.EventsProcessed);
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
}
public async Task Start(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var options = new EventProcessorClientOptions
{
MaximumWaitTime = Configuration.ReadWaitTime,
RetryOptions = new EventHubsRetryOptions
{
TryTimeout = Configuration.ReadTimeout
}
};
var storageClient = default(BlobContainerClient);
var processor = default(EventProcessorClient);
try
{
storageClient = new BlobContainerClient(Configuration.StorageConnectionString, Configuration.BlobContainer);
processor = new EventProcessorClient(storageClient, EventHubConsumerClient.DefaultConsumerGroupName, Configuration.EventHubsConnectionString, Configuration.EventHub, options);
processor.ProcessEventAsync += ProcessEventHandler;
processor.ProcessErrorAsync += ProcessErrorHandler;
await processor.StartProcessingAsync(cancellationToken).ConfigureAwait(false);
await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.ProcessorRestarted);
ErrorsObserved.Add(ex);
}
finally
{
// Wait a short bit to allow for time processing the newly published events.
await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
// Constrain stopping the processor, just in case it has issues. It should not be allowed
// to hang, it should be abandoned so that processing can restart.
using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
try
{
if (processor != null)
{
await processor.StopProcessingAsync(cancellationSource.Token).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.ProcessorRestarted);
ErrorsObserved.Add(ex);
}
processor.ProcessEventAsync -= ProcessEventHandler;
processor.ProcessErrorAsync -= ProcessErrorHandler;
}
}
}
public async Task Start(CancellationToken cancellationToken)
{
var sendTasks = new List <Task>();
while (!cancellationToken.IsCancellationRequested)
{
using var backgroundCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
try
{
sendTasks.Clear();
var options = new EventHubProducerClientOptions
{
RetryOptions = new EventHubsRetryOptions
{
TryTimeout = Configuration.SendTimeout
}
};
var producer = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub, options);
await using (producer.ConfigureAwait(false))
{
// Create a set of background tasks to handle all but one of the concurrent sends. The final
// send will be performed directly.
if (Configuration.ConcurrentSends > 1)
{
for (var index = 0; index < Configuration.ConcurrentSends - 1; ++index)
{
sendTasks.Add(Task.Run(async() =>
{
while (!backgroundCancellationSource.Token.IsCancellationRequested)
{
await PerformSend(producer, backgroundCancellationSource.Token).ConfigureAwait(false);
if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
{
await Task.Delay(Configuration.PublishingDelay.Value, backgroundCancellationSource.Token).ConfigureAwait(false);
}
}
}));
}
}
// Perform one of the sends in the foreground, which will allow easier detection of a
// processor-level issue.
while (!cancellationToken.IsCancellationRequested)
{
try
{
await PerformSend(producer, cancellationToken).ConfigureAwait(false);
if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
{
await Task.Delay(Configuration.PublishingDelay.Value, cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
backgroundCancellationSource.Cancel();
await Task.WhenAll(sendTasks).ConfigureAwait(false);
}
}
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
// If this catch is hit, there's a problem with the producer itself; cancel the
// background sending and wait before allowing the producer to restart.
backgroundCancellationSource.Cancel();
await Task.WhenAll(sendTasks).ConfigureAwait(false);
Interlocked.Increment(ref Metrics.ProducerRestarted);
ErrorsObserved.Add(ex);
}
}
}
private async Task PerformSend(EventHubProducerClient producer,
CancellationToken cancellationToken)
{
// Create the batch and generate a set of random events, keeping only those that were able to fit into the batch.
// Because there is a side-effect of TryAdd in the statement, ensure that ToList is called to materialize the set
// or the batch will be empty at send.
using var batch = await producer.CreateBatchAsync().ConfigureAwait(false);
var batchEvents = new List <EventData>();
var events = EventGenerator.CreateEvents(
Configuration.PublishBatchSize,
Configuration.LargeMessageRandomFactorPercent,
Configuration.PublishingBodyMinBytes,
Configuration.PublishingBodyRegularMaxBytes,
currentSequence);
foreach (var currentEvent in events)
{
if (!batch.TryAdd(currentEvent))
{
break;
}
batchEvents.Add(currentEvent);
}
// Publish the events and report them, capturing any failures specific to the send operation.
try
{
if (batch.Count > 0)
{
Interlocked.Increment(ref Metrics.PublishAttempts);
await producer.SendAsync(batch, cancellationToken).ConfigureAwait(false);
Interlocked.Increment(ref Metrics.BatchesPublished);
Interlocked.Add(ref Metrics.EventsPublished, batch.Count);
Interlocked.Add(ref Metrics.TotalPublshedSizeBytes, batch.SizeInBytes);
}
}
catch (TaskCanceledException)
{
Interlocked.Decrement(ref Metrics.PublishAttempts);
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (OperationCanceledException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
Interlocked.Increment(ref Metrics.CanceledSendExceptions);
ErrorsObserved.Add(ex);
}
catch (TimeoutException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
Interlocked.Increment(ref Metrics.TimeoutExceptions);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.SendExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
}
public async Task Start(CancellationToken cancellationToken)
{
IsRunning = true;
using var process = Process.GetCurrentProcess();
using var publishCancellationSource = new CancellationTokenSource();
using var processorCancellationSource = new CancellationTokenSource();
var publishingTask = default(Task);
var processorTasks = default(IEnumerable <Task>);
var runDuration = Stopwatch.StartNew();
try
{
// Start processing.
processorTasks = Enumerable
.Range(0, Configuration.ProcessorCount)
.Select(_ => Task.Run(() => new Processor(Configuration, Metrics, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
.ToList();
// Test for missing events and update metrics.
var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);
while (!cancellationToken.IsCancellationRequested)
{
Metrics.UpdateEnvironmentStatistics(process);
Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// The run is ending. Clean up the outstanding background operations and
// complete the necessary metrics tracking.
try
{
publishCancellationSource.Cancel();
await publishingTask.ConfigureAwait(false);
// Wait a bit after publishing has completed before signaling for
// processing to be canceled, to allow any recently published
// events to be read.
await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
processorCancellationSource.Cancel();
await Task.WhenAll(processorTasks).ConfigureAwait(false);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
finally
{
runDuration.Stop();
IsRunning = false;
}
}