/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
IEnumerable <EventData> receivedEvents = null;
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
}
catch (Exception exception)
{
await PartitionProcessor.ProcessErrorAsync(exception, cancellationToken).ConfigureAwait(false);
// Stop the pump if it's not a retryable exception.
if (RetryPolicy.CalculateRetryDelay(exception, 1) == null)
{
// StopAsync cannot be awaited in this method because it awaits RunningTask, so we would have a deadlock.
// For this reason, StopAsync starts to run concurrently with this task.
_ = StopAsync(PartitionProcessorCloseReason.EventHubException);
break;
}
}
await PartitionProcessor.ProcessEventsAsync(receivedEvents, cancellationToken).ConfigureAwait(false);
}
}
public void CalculateRetryDelayDoesNotRetryWhenAttemptsExceedTheMaximum(int retryAttempt)
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 5,
Delay = TimeSpan.FromSeconds(1),
MaximumDelay = TimeSpan.FromHours(1),
Mode = RetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), retryAttempt), Is.Null);
}
public void CalculateRetryDelayAllowsRetryForTransientExceptions()
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(1),
MaximumDelay = TimeSpan.FromSeconds(100),
Mode = RetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(new EventHubsException(true, null, null), 88), Is.Not.Null);
}
public void CalculateRetryDelayAllowsRetryForKnownRetriableExceptions(Exception exception)
{
var policy = new BasicRetryPolicy(new ServiceBusRetryOptions
{
MaxRetries = 99,
Delay = TimeSpan.FromSeconds(1),
MaxDelay = TimeSpan.FromSeconds(100),
Mode = ServiceBusRetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(exception, 88), Is.Not.Null);
}
public void CalculateRetryDelayDoesNotRetryWhenThereIsNoMaxRetries()
{
var policy = new BasicRetryPolicy(new ServiceBusRetryOptions
{
MaxRetries = 0,
Delay = TimeSpan.FromSeconds(1),
MaxDelay = TimeSpan.FromHours(1),
Mode = ServiceBusRetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), -1), Is.Null);
}
public void CalculateRetryDelayDoesNotRetryForNotKnownRetriableExceptions(Exception exception)
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(1),
MaximumDelay = TimeSpan.FromSeconds(100),
Mode = RetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(exception, 88), Is.Null);
}
public void CalculateRetryDelayRespectsMaximumDuration(int delaySeconds)
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(delaySeconds),
MaximumDelay = TimeSpan.FromSeconds(1),
Mode = RetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), 88), Is.EqualTo(policy.Options.MaximumDelay));
}
public void CalculateRetryDelayDoesNotRetryWhenThereIsNoMaximumDelay()
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(1),
MaximumDelay = TimeSpan.Zero,
Mode = RetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), 88), Is.Null);
}
public void CalculateRetryDelayDoesNotRetryWithAnActiveTransaction()
{
using var ts = new TransactionScope(TransactionScopeAsyncFlowOption.Suppress);
var exception = new ServiceBusException(true, "this is fake", "dummy", ServiceBusFailureReason.ServiceCommunicationProblem);
var policy = new BasicRetryPolicy(new ServiceBusRetryOptions
{
MaxRetries = 99,
Delay = TimeSpan.FromSeconds(1),
MaxDelay = TimeSpan.FromSeconds(100),
Mode = ServiceBusRetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(exception, 88), Is.Null);
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
public void CalculateRetryDelayDoesNotTrottleGeneralExceptions()
{
var delaySeconds = 1;
var exception = new EventHubsException(true, "dummy", EventHubsException.FailureReason.GeneralError);
var policy = new BasicRetryPolicy(new EventHubsRetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(delaySeconds),
MaximumDelay = TimeSpan.FromDays(1),
Mode = EventHubsRetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(exception, 1),
Is.AtLeast(TimeSpan.FromSeconds(delaySeconds)).And.AtMost(TimeSpan.FromSeconds(delaySeconds * 2)));
}
public void CalculateRetryDelayDoesNotOverlowTimespanMaximum()
{
// The fixed policy can't exceed the maximum due to limitations on
// the configured Delay and MaximumRetries; the exponential policy
// will overflow a TimeSpan on the 38th retry with maximum values if
// the calculation is uncapped.
var policy = new BasicRetryPolicy(new ServiceBusRetryOptions
{
MaxRetries = 100,
Delay = TimeSpan.FromMinutes(5),
MaxDelay = TimeSpan.MaxValue,
Mode = ServiceBusRetryMode.Exponential
});
Assert.That(policy.CalculateRetryDelay(new ServiceBusException(true, "transient", "fake", ServiceBusFailureReason.ServiceTimeout), 88), Is.EqualTo(TimeSpan.MaxValue));
}
public void CalculateRetryDelayUsesFixedMode(int iterations)
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(iterations),
MaximumDelay = TimeSpan.FromHours(72),
Mode = RetryMode.Fixed
});
var variance = TimeSpan.FromSeconds(policy.Options.Delay.TotalSeconds * policy.JitterFactor);
for (var index = 0; index < iterations; ++index)
{
Assert.That(policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), 88), Is.EqualTo(policy.Options.Delay).Within(variance), $"Iteration: { index } produced an unexpected delay.");
}
}
public void CalculateRetryDelayRespectsThrottleInterval()
{
var delaySeconds = 1;
var throttleException = new EventHubsException(true, "dummy", EventHubsException.FailureReason.ServiceBusy);
var policy = new BasicRetryPolicy(new EventHubsRetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromSeconds(delaySeconds),
MaximumDelay = TimeSpan.FromDays(1),
Mode = EventHubsRetryMode.Fixed
});
Assert.That(policy.CalculateRetryDelay(throttleException, 1),
Is.AtLeast(TimeSpan.FromSeconds(delaySeconds + policy.MinimumThrottleSeconds))
.And.AtMost(TimeSpan.FromSeconds((delaySeconds * 2) + policy.MaximumThrottleSeconds)));
}
public void CalculateRetryDelayUsesExponentialMode(int iterations)
{
var policy = new BasicRetryPolicy(new RetryOptions
{
MaximumRetries = 99,
Delay = TimeSpan.FromMilliseconds(15),
MaximumDelay = TimeSpan.FromHours(50000),
Mode = RetryMode.Exponential
});
var previousDelay = TimeSpan.Zero;
for (var index = 0; index < iterations; ++index)
{
var variance = TimeSpan.FromSeconds((policy.Options.Delay.TotalSeconds * index) * policy.JitterFactor);
var delay = policy.CalculateRetryDelay(Mock.Of <TimeoutException>(), index);
Assert.That(delay.HasValue, Is.True, $"Iteration: { index } did not have a value.");
Assert.That(delay.Value, Is.GreaterThan(previousDelay.Add(variance)), $"Iteration: { index } produced an unexpected delay.");
previousDelay = delay.Value;
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the event processor processing handlers.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
List <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = (await InnerConsumer.ReceiveAsync(MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false)).ToList();
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
// Small workaround to make sure we call ProcessEvent with EventData = null when no events have been received.
// The code is expected to get simpler when we start using the async enumerator internally to receive events.
if (receivedEvents.Count == 0)
{
receivedEvents.Add(null);
}
diagnosticScope.Start();
foreach (var eventData in receivedEvents)
{
try
{
var processorEvent = new EventProcessorEvent(Context, eventData, UpdateCheckpointAsync);
await ProcessEventAsync(processorEvent).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
unrecoverableException = eventProcessingException;
break;
}
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
throw eventHubException;
}
}
if (unrecoverableException != null)
{
throw unrecoverableException;
}
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
diagnosticScope.Start();
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
diagnosticScope.Failed(partitionProcessorException);
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (s_retryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
