private async Task ProcessEventHandler(string processorId, ProcessEventArgs args)
{
try
{
Interlocked.Increment(ref Metrics.TotalServiceOperations);
// If there was no event then there is nothing to do.
if (!args.HasEvent)
{
return;
}
// Determine if the event has an identifier and has been tracked as published.
var hasId = args.Data.Properties.TryGetValue(EventGenerator.IdPropertyName, out var id);
var eventId = (hasId) ? id?.ToString() : null;
var isTrackedEvent = PublishedEvents.TryRemove(eventId, out var publishedEvent);
// If the event has an id that has been seen before, track it as a duplicate processing and
// take no further action.
if ((hasId) && (ReadEvents.ContainsKey(eventId)))
{
Interlocked.Increment(ref Metrics.DuplicateEventsDiscarded);
return;
}
// Since this event wasn't a duplicate, consider it read.
Interlocked.Increment(ref Metrics.EventsRead);
// If there the event isn't a known and published event, then track it but take no
// further action.
if ((!hasId) || (!isTrackedEvent))
{
// If there was an id, then the event wasn't tracked. This is likely a race condition in tracking;
// allow for a small delay and then try to find the event again.
if (hasId)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
isTrackedEvent = PublishedEvents.TryRemove(eventId, out publishedEvent);
}
else
{
// If there wasn't an id then consider it an unexpected event failure.
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
return;
}
// If there was an id, but the event wasn't tracked as published or processed, cache it as an
// unexpected event for later consideration. If it cannot be cached, consider it a failure.
if ((!isTrackedEvent) && (!ReadEvents.ContainsKey(eventId)))
{
if (!UnexpectedEvents.TryAdd(eventId, args.Data))
{
Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
}
return;
}
}
// It has been proven that the current event is expected; track it as having been read.
ReadEvents.TryAdd(eventId, 0);
// Validate the event against expectations.
if (!args.Data.IsEquivalentTo(publishedEvent))
{
if (!args.Data.Body.ToArray().SequenceEqual(publishedEvent.Body.ToArray()))
{
Interlocked.Increment(ref Metrics.InvalidBodies);
}
else
{
Interlocked.Increment(ref Metrics.InvalidProperties);
}
}
// Validate that the intended partition that was sent as a property matches the
// partition that the handler was triggered for.
if ((!publishedEvent.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var publishedPartition)) ||
(args.Partition.PartitionId != publishedPartition.ToString()))
{
Interlocked.Increment(ref Metrics.EventsFromWrongPartition);
}
// Validate that the sequence number that was sent as a property is greater than the last read
// sequence number for the partition; if there hasn't been an event for the partition yet, then
// there is no validation possible.
//
// Track the sequence number for future comparisons.
var currentSequence = default(object);
if ((LastReadPartitionSequence.TryGetValue(args.Partition.PartitionId, out var lastReadSequence)) &&
((!publishedEvent.Properties.TryGetValue(EventGenerator.SequencePropertyName, out currentSequence)) || (lastReadSequence >= (long)currentSequence)))
{
Interlocked.Increment(ref Metrics.EventsOutOfOrder);
}
var trackSequence = (currentSequence == default) ? -1 : (long)currentSequence;
LastReadPartitionSequence.AddOrUpdate(args.Partition.PartitionId, _ => trackSequence, (part, seq) => Math.Max(seq, trackSequence));
// Create a checkpoint every 100 events for the partition, just to follow expected patterns.
if (trackSequence % 100 == 0)
{
await args.UpdateCheckpointAsync(args.CancellationToken).ConfigureAwait(false);
}
// Mark the event as processed.
Interlocked.Increment(ref Metrics.EventsProcessed);
}
catch (EventHubsException ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
ex.TrackMetrics(Metrics);
ErrorsObserved.Add(ex);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.ProcessingExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
}