private async Task ProcessEventsAsync(OperationContext context, List <EventData> messages)
{
// Tracking raw messages count.
Counters[ReceivedEventHubEventsCount].Add(messages.Count);
CacheActivityTracker.AddValue(CaSaaSActivityTrackingCounters.ReceivedEventHubMessages, messages.Count);
// Creating nested context for all the processing operations.
context = context.CreateNested(nameof(EventHubContentLocationEventStore));
if (messages.Count == 0)
{
// This probably does not actually occur, but just in case, ignore empty message batch.
// NOTE: We do this after logging to ensure we notice if the we are getting empty message batches.
return;
}
var state = new SharedEventProcessingState(context, this, messages);
if (_eventProcessingBlocks != null)
{
await context
.CreateOperation(Tracer, () => sendToActionBlockAsync(state))
.WithOptions(traceOperationStarted: false, endMessageFactory: r => $"TotalQueueSize={Interlocked.Read(ref _queueSize)}")
.RunAsync(caller: "SendToActionBlockAsync")
.TraceIfFailure(context);
}
else
{
await ProcessEventsCoreAsync(new ProcessEventsInput(state, messages, actionBlockIndex : -1, store : this), EventDataSerializer);
}
async Task <BoolResult> sendToActionBlockAsync(SharedEventProcessingState st)
{
// This local function "sends" a message into an action block based on the sender's hash code to process events in parallel from different machines.
// (keep in mind, that the data from the same machine should be processed sequentially, because events order matters).
// Then, it creates a local counter for each processing operation to track the results for the entire batch.
foreach (var messageGroup in messages.GroupBy(GetProcessingIndex))
{
int actionBlockIndex = messageGroup.Key;
var eventProcessingBlock = _eventProcessingBlocks ![actionBlockIndex];
var input = new ProcessEventsInput(st, messageGroup, actionBlockIndex, this);
var sendAsyncTask = eventProcessingBlock.SendAsync(input);
if (sendAsyncTask.Status == TaskStatus.WaitingForActivation)
{
// The action block is busy. It means that its most likely full.
Tracer.Debug(context, $"Action block {actionBlockIndex} is busy. Block's queue size={eventProcessingBlock.InputCount}.");
}
bool success = await sendAsyncTask;
if (!success)
{
// NOTE: This case should not actually occur.
// Complete the operation in case we couldn't send to the action block to prevent pending event queue from getting backlogged.
input.Complete();
return(new BoolResult("Failed to add message to an action block."));
}
}
private async Task ProcessEventsAsync(OperationContext context, List <EventData> messages)
{
// Creating nested context for all the processing operations.
context = context.CreateNested();
string asyncProcessing = _eventProcessingBlocks != null ? "on" : "off";
Tracer.Info(context, $"{Tracer.Name}: Received {messages.Count} events from Event Hub. Async processing is '{asyncProcessing}'.");
if (messages.Count == 0)
{
// This probably does not actually occur, but just in case, ignore empty message batch.
// NOTE: We do this after logging to ensure we notice if the we are getting empty message batches.
return;
}
var state = new SharedEventProcessingState(context, this, messages);
if (_eventProcessingBlocks != null)
{
// Creating nested context to correlate all the processing operations.
context = context.CreateNested();
await context.PerformOperationAsync(
Tracer,
() => sendToActionBlockAsync(),
traceOperationStarted : false).TraceIfFailure(context);
}
else
{
await ProcessEventsCoreAsync(new ProcessEventsInput(state, messages), EventDataSerializer, index : 0);
}
async Task <SendToActionBlockResult> sendToActionBlockAsync()
{
// This local function "sends" a message into an action block based on the sender's hash code to process events in parallel from different machines.
// (keep in mind, that the data from the same machine should be processed sequentially, because events order matters).
// Then, it creates a local counter for each processing operation to track the results for the entire batch.
var operationTasks = new List <Task <OperationCounters> >();
foreach (var messageGroup in messages.GroupBy(GetProcessingIndex))
{
var eventProcessingBlock = _eventProcessingBlocks[messageGroup.Key];
var input = new ProcessEventsInput(state, messageGroup);
bool success = await eventProcessingBlock.SendAsync(input);
if (!success)
{
// NOTE: This case should not actually occur.
// Complete the operation in case we couldn't send to the action block to prevent pending event queue from getting backlogged.
input.Complete();
return(new SendToActionBlockResult("Failed to add message to an action block."));
}
}
return(new SendToActionBlockResult(operationTasks));
}
}
private async Task ProcessEventsCoreAsync(ProcessEventsInput input, ContentLocationEventDataSerializer eventDataSerializer)
{
var context = input.Context;
var counters = input.EventStoreCounters;
await context.PerformOperationAsync(
Tracer,
async() =>
{
int filteredEvents = 0;
foreach (var message in input.Messages)
{
// Extracting information from the message
var foundEventFilter = message.Properties.TryGetValue(EventFilterKey, out var eventFilter);
message.Properties.TryGetValue(OperationIdKey, out var operationId);
var sender = TryGetMessageSender(message) ?? "Unknown sender";
var eventTimeUtc = message.SystemProperties.EnqueuedTimeUtc;
var eventProcessingDelay = DateTime.UtcNow - eventTimeUtc;
// Creating nested context with operationId as a guid. This helps to correlate operations on a worker and a master machines.
context = CreateNestedContext(context, operationId?.ToString());
Tracer.Debug(context, $"{Tracer.Name}.ReceivedEvent: ProcessingDelay={eventProcessingDelay}, Sender={sender}, OpId={operationId}, SeqNo={message.SystemProperties.SequenceNumber}, EQT={eventTimeUtc}, Filter={eventFilter}, Size={message.Body.Count}.");
Tracer.TrackMetric(context, EventProcessingDelayInSecondsMetricName, (long)eventProcessingDelay.TotalSeconds);
counters[ReceivedMessagesTotalSize].Add(message.Body.Count);
counters[ReceivedEventBatchCount].Increment();
if (!foundEventFilter || !string.Equals(eventFilter as string, _configuration.Epoch))
{
counters[FilteredEvents].Increment();
filteredEvents++;
continue;
}
// Deserializing a message
IReadOnlyList <ContentLocationEventData> eventDatas;
using (counters[Deserialization].Start())
{
eventDatas = eventDataSerializer.DeserializeEvents(message);
}
counters[ReceivedEventsCount].Add(eventDatas.Count);
// Dispatching deserialized events data
using (counters[DispatchEvents].Start())
{
foreach (var eventData in eventDatas)
{
// An event processor may fail to process the event, but we will save the sequence point anyway.
await DispatchAsync(context, eventData, counters);
}
}
_lastProcessedSequencePoint = new EventSequencePoint(message.SystemProperties.SequenceNumber);
}
Counters.Append(counters);
return(BoolResult.Success);
},
private async Task ProcessEventsAsync(OperationContext context, List <EventData> messages)
{
// Creating nested context for all the processing operations.
context = context.CreateNested();
var sw = Stopwatch.StartNew();
string asyncProcessing = _eventProcessingBlocks != null ? "on" : "off";
Tracer.Info(context, $"{Tracer.Name}: Received {messages.Count} events from Event Hub. Async processing is '{asyncProcessing}'.");
if (_eventProcessingBlocks != null)
{
// Creating nested context to correlate all the processing operations.
context = context.CreateNested();
SendToActionBlockResult result = await context.PerformOperationAsync(
Tracer,
() => sendToActionBlockAsync(),
traceOperationStarted : false).TraceIfFailure(context);
printOperationResultsAsynchronously(result);
}
else
{
await ProcessEventsCoreAsync(new ProcessEventsInput(context, messages, new OperationCounters(), processingFinishedTaskSource : null), EventDataSerializer);
}
void printOperationResultsAsynchronously(SendToActionBlockResult results)
{
if (results)
{
Task.WhenAll(results.Value).ContinueWith(
t =>
{
var eventStoreCounters = t.GetAwaiter().GetResult()
.Select(c => c.EventStoreCounters)
.Aggregate((collection, counterCollection) => collection + counterCollection);
int duration = (int)sw.ElapsedMilliseconds;
context.LogProcessEventsOverview(eventStoreCounters, duration);
}).IgnoreErrors();
}
}
async Task <SendToActionBlockResult> sendToActionBlockAsync()
{
// This local function "sends" a message into an action block based on the sender's hash code to process events in parallel from different machines.
// (keep in mind, that the data from the same machine should be processed sequentially, because events order matters).
// Then, it creates a local counter for each processing operation to track the results for the entire batch.
var operationTasks = new List <Task <OperationCounters> >();
foreach (var messageGroup in messages.GroupBy(GetProcessingIndex))
{
var eventProcessingBlock = _eventProcessingBlocks[messageGroup.Key];
var input = ProcessEventsInput.Create(context, messageGroup);
bool success = await eventProcessingBlock.SendAsync(input);
if (!success)
{
return(new SendToActionBlockResult("Failed to add message to an action block."));
}
Contract.Assert(input.ProcessingFinishedTaskSource != null);
operationTasks.Add(input.ProcessingFinishedTaskSource.Value.Task);
}
return(new SendToActionBlockResult(operationTasks));
}
}
protected override async Task ProcessEventsCoreAsync(ProcessEventsInput input, ContentLocationEventDataSerializer eventDataSerializer)
{
await Task.Delay(_configuration.Slowdown);
await base.ProcessEventsCoreAsync(input, eventDataSerializer);
}
