/// <summary>
/// Performs the actions needed to instrument a set of events.
/// </summary>
///
/// <param name="events">The events to instrument.</param>
///
private void InstrumentMessages(IEnumerable <EventData> events)
{
foreach (EventData eventData in events)
{
EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
}
}
/// <summary>
/// Performs the actions needed to instrument a set of events.
/// </summary>
///
/// <param name="events">The events to instrument.</param>
///
private void InstrumentMessages(IEnumerable <EventData> events)
{
foreach (EventData eventData in events)
{
EventDataInstrumentation.InstrumentEvent(eventData);
}
}
/// <summary>
/// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="events">The set of event data to send.</param>
/// <param name="options">The set of options to consider when sending this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
/// <seealso cref="SendAsync(EventData, CancellationToken)" />
/// <seealso cref="SendAsync(EventData, SendEventOptions, CancellationToken)" />
/// <seealso cref="SendAsync(IEnumerable{EventData}, CancellationToken)" />
/// <seealso cref="SendAsync(EventDataBatch, CancellationToken)" />
///
internal virtual async Task SendAsync(IEnumerable <EventData> events,
SendEventOptions options,
CancellationToken cancellationToken = default)
{
options ??= DefaultSendOptions;
Argument.AssertNotNull(events, nameof(events));
AssertSinglePartitionReference(options.PartitionId, options.PartitionKey);
int attempts = 0;
events = (events as IList <EventData>) ?? events.ToList();
InstrumentMessages(events);
var diagnosticIdentifiers = new List <string>();
foreach (var eventData in events)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out var identifier))
{
diagnosticIdentifiers.Add(identifier);
}
}
using DiagnosticScope scope = CreateDiagnosticScope(diagnosticIdentifiers);
var pooledProducer = PartitionProducerPool.GetPooledProducer(options.PartitionId, PartitionProducerLifespan);
while (!cancellationToken.IsCancellationRequested)
{
try
{
await using var _ = pooledProducer.ConfigureAwait(false);
await pooledProducer.TransportProducer.SendAsync(events, options, cancellationToken).ConfigureAwait(false);
return;
}
catch (EventHubsException eventHubException)
when(eventHubException.Reason == EventHubsException.FailureReason.ClientClosed && ShouldRecreateProducer(pooledProducer.TransportProducer, options.PartitionId))
{
if (++attempts >= MaximumCreateProducerAttempts)
{
scope.Failed(eventHubException);
throw;
}
pooledProducer = PartitionProducerPool.GetPooledProducer(options.PartitionId, PartitionProducerLifespan);
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
throw new TaskCanceledException();
}
/// <summary>
/// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="events">The set of event data to send.</param>
/// <param name="options">The set of options to consider when sending this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
/// <seealso cref="SendAsync(EventData, CancellationToken)" />
/// <seealso cref="SendAsync(EventData, SendEventOptions, CancellationToken)" />
/// <seealso cref="SendAsync(IEnumerable{EventData}, CancellationToken)" />
/// <seealso cref="SendAsync(EventDataBatch, CancellationToken)" />
///
internal virtual async Task SendAsync(IEnumerable <EventData> events,
SendEventOptions options,
CancellationToken cancellationToken = default)
{
options ??= DefaultSendOptions;
Argument.AssertNotNull(events, nameof(events));
AssertSinglePartitionReference(options.PartitionId, options.PartitionKey);
// Determine the transport producer to delegate the send operation to. Because sending to a specific
// partition requires a dedicated client, use (or create) that client if a partition was specified. Otherwise
// the default gateway producer can be used to request automatic routing from the Event Hubs service gateway.
TransportProducer activeProducer;
if (string.IsNullOrEmpty(options.PartitionId))
{
activeProducer = EventHubProducer;
}
else
{
// This assertion is intended as an additional check, not as a guarantee. There still exists a benign
// race condition where a transport producer may be created after the client has been closed; in this case
// the transport producer will be force-closed with the associated connection or, worst case, will close once
// its idle timeout period elapses.
Argument.AssertNotClosed(IsClosed, nameof(EventHubProducerClient));
activeProducer = PartitionProducers.GetOrAdd(options.PartitionId, id => Connection.CreateTransportProducer(id, RetryPolicy));
}
events = (events as IList <EventData>) ?? events.ToList();
InstrumentMessages(events);
var diagnosticIdentifiers = new List <string>();
foreach (var eventData in events)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out var identifier))
{
diagnosticIdentifiers.Add(identifier);
}
}
using DiagnosticScope scope = CreateDiagnosticScope(diagnosticIdentifiers);
try
{
await activeProducer.SendAsync(events, options, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
public bool TryAdd(EventData eventData)
{
bool instrumented = EventDataInstrumentation.InstrumentEvent(eventData);
bool added = InnerBatch.TryAdd(eventData);
if (!added && instrumented)
{
EventDataInstrumentation.ResetEvent(eventData);
}
return(added);
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
public bool TryAdd(EventData eventData)
{
bool instrumented = EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
bool added = InnerBatch.TryAdd(eventData);
if (!added && instrumented)
{
EventDataInstrumentation.ResetEvent(eventData);
}
return(added);
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
/// <remarks>
/// When an event is accepted into the batch, its content and state are frozen; any
/// changes made to the event will not be reflected in the batch nor will any state
/// transitions be reflected to the original instance.
/// </remarks>
///
/// <exception cref="InvalidOperationException">
/// When a batch is published, it will be locked for the duration of that operation. During this time,
/// no events may be added to the batch. Calling <c>TryAdd</c> while the batch is being published will
/// result in an <see cref="InvalidOperationException" /> until publishing has completed.
/// </exception>
///
public bool TryAdd(EventData eventData)
{
lock (SyncGuard)
{
AssertNotLocked();
eventData = eventData.Clone();
EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
var added = InnerBatch.TryAdd(eventData);
if ((added) && (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId)))
{
EventDiagnosticIdentifiers.Add(diagnosticId);
}
return(added);
}
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
public bool TryAdd(EventData eventData)
{
bool instrumented = EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
bool added = InnerBatch.TryAdd(eventData);
if (added)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
EventDiagnosticIdentifiers.Add(diagnosticId);
}
}
else if (instrumented)
{
EventDataInstrumentation.ResetEvent(eventData);
}
return(added);
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
/// <remarks>
/// When an event is accepted into the batch, its content and state are frozen; any
/// changes made to the event will not be reflected in the batch nor will any state
/// transitions be reflected to the original instance.
/// </remarks>
///
/// <exception cref="InvalidOperationException">
/// When a batch is published, it will be locked for the duration of that operation. During this time,
/// no events may be added to the batch. Calling <c>TryAdd</c> while the batch is being published will
/// result in an <see cref="InvalidOperationException" /> until publishing has completed.
/// </exception>
///
public bool TryAdd(EventData eventData)
{
lock (SyncGuard)
{
AssertNotLocked();
var messageScopeCreated = false;
var identifier = default(string);
try
{
(messageScopeCreated, identifier) = EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
var added = InnerBatch.TryAdd(eventData);
if ((added) && (identifier != null))
{
EventDiagnosticIdentifiers.Add(identifier);
}
return(added);
}
finally
{
// If a new message scope was added when instrumenting the instance, the identifier was
// added during this call. If so, remove it so that the source event is not modified; the
// instrumentation will have been captured by the batch's copy of the event, if it was accepted
// into the batch.
if ((messageScopeCreated) && (identifier != null))
{
EventDataInstrumentation.ResetEvent(eventData);
}
}
}
}
/// <summary>
/// Attempts to add an event to the batch, ensuring that the size
/// of the batch does not exceed its maximum.
/// </summary>
///
/// <param name="eventData">The event to attempt to add to the batch.</param>
///
/// <returns><c>true</c> if the event was added; otherwise, <c>false</c>.</returns>
///
public bool TryAdd(EventData eventData)
{
if (_locked)
{
throw new InvalidOperationException(Resources.EventBatchIsLocked);
}
bool instrumented = EventDataInstrumentation.InstrumentEvent(eventData, FullyQualifiedNamespace, EventHubName);
bool added = InnerBatch.TryAdd(eventData);
if (added)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
EventDiagnosticIdentifiers.Add(diagnosticId);
}
}
else if (instrumented)
{
EventDataInstrumentation.ResetEvent(eventData);
}
return(added);
}
/// <summary>
/// Starts running a task responsible for receiving and processing events in the context of a specified partition.
/// </summary>
///
/// <param name="partitionId">The identifier of the Event Hub partition the task is associated with. Events will be read only from this partition.</param>
/// <param name="startingPosition">The position within the partition where the task should begin reading events.</param>
/// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param>
/// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param>
/// <param name="trackLastEnqueuedEventInformation">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns>
///
protected virtual Task RunPartitionProcessingAsync(string partitionId,
EventPosition startingPosition,
TimeSpan?maximumReceiveWaitTime,
RetryOptions retryOptions,
bool trackLastEnqueuedEventInformation,
CancellationToken cancellationToken = default)
{
// TODO: should the retry options used here be the same for the abstract RetryPolicy property?
Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId));
Argument.AssertNotNull(retryOptions, nameof(retryOptions));
return(Task.Run(async() =>
{
// TODO: should we double check if a previous run already exists and close it? We have a race condition. Maybe we should throw in case another task exists.
var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var taskCancellationToken = cancellationSource.Token;
ActivePartitionProcessorTokenSources[partitionId] = cancellationSource;
// Context is set to default if operation fails. This shouldn't fail unless the user tries processing
// a partition they don't own.
PartitionContexts.TryGetValue(partitionId, out var context);
var options = new EventHubConsumerClientOptions
{
RetryOptions = retryOptions,
TrackLastEnqueuedEventInformation = trackLastEnqueuedEventInformation
};
await using var connection = CreateConnection();
await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, options))
{
await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, maximumReceiveWaitTime, taskCancellationToken))
{
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled &&
partitionEvent.Data != null &&
EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
diagnosticScope.Start();
try
{
await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
throw;
}
}
}
}));
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the event processor processing handlers.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
List <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = (await InnerConsumer.ReceiveAsync(MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false)).ToList();
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
// Small workaround to make sure we call ProcessEvent with EventData = null when no events have been received.
// The code is expected to get simpler when we start using the async enumerator internally to receive events.
if (receivedEvents.Count == 0)
{
receivedEvents.Add(null);
}
diagnosticScope.Start();
foreach (var eventData in receivedEvents)
{
try
{
var processorEvent = new EventProcessorEvent(Context, eventData, UpdateCheckpointAsync);
await ProcessEventAsync(processorEvent).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
unrecoverableException = eventProcessingException;
break;
}
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
throw eventHubException;
}
}
if (unrecoverableException != null)
{
throw unrecoverableException;
}
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
diagnosticScope.Start();
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
diagnosticScope.Failed(partitionProcessorException);
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (s_retryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
