public async Task Start(CancellationToken cancellationToken)
{
IsRunning = true;
using var process = Process.GetCurrentProcess();
using var publishCancellationSource = new CancellationTokenSource();
using var processorCancellationSource = new CancellationTokenSource();
var publishingTask = default(Task);
var processorTasks = default(IEnumerable <Task>);
var runDuration = Stopwatch.StartNew();
try
{
// Begin publishing events in the background.
publishingTask = Task.Run(() => new Publisher(Configuration, Metrics, PublishedEvents, ErrorsObserved).Start(publishCancellationSource.Token));
// Start processing.
processorTasks = Enumerable
.Range(0, Configuration.ProcessorCount)
.Select(_ => Task.Run(() => new Processor(Configuration, Metrics, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
.ToList();
// Test for missing events and update metrics.
var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);
while (!cancellationToken.IsCancellationRequested)
{
Metrics.UpdateEnvironmentStatistics(process);
Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, eventDueInterval, Metrics);
await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// The run is ending. Clean up the outstanding background operations and
// complete the necessary metrics tracking.
try
{
publishCancellationSource.Cancel();
await publishingTask.ConfigureAwait(false);
// Wait a bit after publishing has completed before signaling for
// processing to be canceled, to allow the recently published
// events to be read.
await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false);
processorCancellationSource.Cancel();
await Task.WhenAll(processorTasks).ConfigureAwait(false);
// Wait a bit after processing has completed before and then perform
// the last bit of bookkeeping, scanning for missing and unexpected events.
await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes), Metrics);
foreach (var unexpectedEvent in UnexpectedEvents.Values)
{
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(unexpectedEvent, false), EventHubsException.FailureReason.GeneralError));
}
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
finally
{
runDuration.Stop();
IsRunning = false;
}
}
private static void ScanForUnreadEvents(ConcurrentDictionary <string, EventData> publishedEvents,
ConcurrentDictionary <string, EventData> unexpectedEvents,
ConcurrentDictionary <string, byte> readEvents,
ConcurrentBag <Exception> errorsObserved,
TimeSpan eventDueInterval,
Metrics metrics)
{
// An event is considered missing if it was published longer ago than the due time and
// still exists in the set of published events.
object publishDate;
var now = DateTimeOffset.UtcNow;
foreach (var publishedEvent in publishedEvents.ToList())
{
if ((publishedEvent.Value.Properties.TryGetValue(EventGenerator.PublishTimePropertyName, out publishDate)) &&
(((DateTimeOffset)publishDate).Add(eventDueInterval) < now) &&
(publishedEvents.TryRemove(publishedEvent.Key, out _)))
{
// Check to see if the event was read and tracked as unexpected. If so, perform basic validation
// and record it.
if (unexpectedEvents.TryRemove(publishedEvent.Key, out var readEvent))
{
Interlocked.Increment(ref metrics.EventsRead);
// Validate the event against expectations.
if (!readEvent.IsEquivalentTo(publishedEvent.Value))
{
if (!readEvent.Body.ToArray().SequenceEqual(publishedEvent.Value.Body.ToArray()))
{
Interlocked.Increment(ref metrics.InvalidBodies);
}
else
{
Interlocked.Increment(ref metrics.InvalidProperties);
}
}
if ((!publishedEvent.Value.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var publishedPartition)) ||
(!readEvent.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var readPartition)) ||
(readPartition.ToString() != publishedPartition.ToString()))
{
Interlocked.Increment(ref metrics.EventsFromWrongPartition);
}
Interlocked.Increment(ref metrics.EventsProcessed);
readEvents.TryAdd(publishedEvent.Key, 0);
}
else if (!readEvents.ContainsKey(publishedEvent.Key))
{
// The event wasn't read earlier and tracked as unexpected; it has not been seen. Track it as missing.
Interlocked.Increment(ref metrics.EventsNotReceived);
errorsObserved.Add(new EventHubsException(false, string.Empty, FormatMissingEvent(publishedEvent.Value, now), EventHubsException.FailureReason.GeneralError));
}
}
}
}
