public async Task <T> RunOperation <T>(Func <Task <T> > operation)
{
var failedAttemptCount = 0;
var stopWatch = Stopwatch.StartNew();
try
{
TimeSpan tryTimeout = RetryPolicy.CalculateTryTimeout(0);
while (!CancellationToken.IsCancellationRequested)
{
try
{
return(await operation().ConfigureAwait(false));
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EntityName);
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, mark the exception as active and break out of the loop.
++failedAttemptCount;
TimeSpan?retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount);
if (retryDelay.HasValue && !Scope.IsDisposed && !CancellationToken.IsCancellationRequested)
{
//EventHubsEventSource.Log.GetPropertiesError(EventHubName, activeEx.Message);
await Task.Delay(retryDelay.Value, CancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
stopWatch.Reset();
}
else if (ex is AmqpException)
{
throw activeEx;
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (Exception exception)
{
throw exception;
//TODO through correct exception throw AmqpExceptionHelper.GetClientException(exception);
}
finally
{
stopWatch.Stop();
//TODO log correct completion event ServiceBusEventSource.Log.PeekMessagesComplete(EntityName);
}
}
/// <summary>
/// Reads the set of partition publishing properties active for this producer at the time it was initialized.
/// </summary>
///
/// <param name="cancellationToken">The cancellation token to consider when creating the link.</param>
///
/// <returns>The set of <see cref="PartitionPublishingProperties" /> observed when the producer was initialized.</returns>
///
/// <remarks>
/// It is important to note that these properties are a snapshot of the service state at the time when the
/// producer was initialized; they do not necessarily represent the current state of the service.
/// </remarks>
///
public override async ValueTask <PartitionPublishingProperties> ReadInitializationPublishingPropertiesAsync(CancellationToken cancellationToken)
{
Argument.AssertNotClosed(_closed, nameof(AmqpProducer));
Argument.AssertNotClosed(ConnectionScope.IsDisposed, nameof(EventHubConnection));
// If the properties were already initialized, use them.
if (InitializedPartitionProperties != null)
{
return(InitializedPartitionProperties);
}
// Initialize the properties by forcing the link to be opened.
var failedAttemptCount = 0;
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
while ((!cancellationToken.IsCancellationRequested) && (!_closed))
{
try
{
if (!SendLink.TryGetOpenedObject(out _))
{
await SendLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout), cancellationToken).ConfigureAwait(false);
}
break;
}
catch (Exception ex)
{
++failedAttemptCount;
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
var activeEx = ex.TranslateServiceException(EventHubName);
var retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!cancellationToken.IsCancellationRequested))
{
await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else if (ex is AmqpException)
{
ExceptionDispatchInfo.Capture(activeEx).Throw();
}
else
{
throw;
}
}
}
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
return(InitializedPartitionProperties);
}
/// <summary>
/// Applies the checkpoint store's <see cref="RetryPolicy" /> to a specified function.
/// </summary>
///
/// <param name="functionToRetry">The function to which the retry policy should be applied.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
///
/// <returns>The value returned by the function to which the retry policy has been applied.</returns>
///
private async Task ApplyRetryPolicy(Func <CancellationToken, Task> functionToRetry,
CancellationToken cancellationToken)
{
TimeSpan?retryDelay;
var failedAttemptCount = 0;
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
var timeoutTokenSource = default(CancellationTokenSource);
var linkedTokenSource = default(CancellationTokenSource);
while (!cancellationToken.IsCancellationRequested)
{
try
{
timeoutTokenSource = new CancellationTokenSource(tryTimeout);
linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutTokenSource.Token);
await functionToRetry(linkedTokenSource.Token).ConfigureAwait(false);
return;
}
catch (Exception ex)
{
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, mark the exception as active and break out of the loop.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount);
if ((retryDelay.HasValue) && (!cancellationToken.IsCancellationRequested))
{
await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else
{
timeoutTokenSource?.Token.ThrowIfCancellationRequested <TimeoutException>();
throw;
}
}
finally
{
timeoutTokenSource?.Dispose();
linkedTokenSource?.Dispose();
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
/// <summary>
/// Sends an AMQP message that contains a batch of events to the associated Event Hub. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="messageFactory">A factory which can be used to produce an AMQP message containing the batch of events to be sent.</param>
/// <param name="partitionKey">The hashing key to use for influencing the partition to which events should be routed.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
protected virtual async Task SendAsync(Func <AmqpMessage> messageFactory,
string partitionKey,
CancellationToken cancellationToken)
{
var failedAttemptCount = 0;
var logPartition = PartitionId ?? partitionKey;
var retryDelay = default(TimeSpan?);
var messageHash = default(string);
var stopWatch = Stopwatch.StartNew();
SendingAmqpLink link;
try
{
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
while (!cancellationToken.IsCancellationRequested)
{
try
{
using AmqpMessage batchMessage = messageFactory();
messageHash = batchMessage.GetHashCode().ToString();
EventHubsEventSource.Log.EventPublishStart(EventHubName, logPartition, messageHash);
link = await SendLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
// Validate that the batch of messages is not too large to send. This is done after the link is created to ensure
// that the maximum message size is known, as it is dictated by the service using the link.
if (batchMessage.SerializedMessageSize > MaximumMessageSize)
{
throw new EventHubsException(EventHubName, string.Format(Resources.MessageSizeExceeded, messageHash, batchMessage.SerializedMessageSize, MaximumMessageSize), EventHubsException.FailureReason.MessageSizeExceeded);
}
// Attempt to send the message batch.
var deliveryTag = new ArraySegment <byte>(BitConverter.GetBytes(Interlocked.Increment(ref _deliveryCount)));
var outcome = await link.SendMessageAsync(batchMessage, deliveryTag, AmqpConstants.NullBinary, tryTimeout.CalculateRemaining(stopWatch.Elapsed)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
if (outcome.DescriptorCode != Accepted.Code)
{
throw AmqpError.CreateExceptionForError((outcome as Rejected)?.Error, EventHubName);
}
// The send operation should be considered successful; return to
// exit the retry loop.
return;
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EventHubName);
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!cancellationToken.IsCancellationRequested))
{
EventHubsEventSource.Log.EventPublishError(EventHubName, logPartition, messageHash, activeEx.Message);
await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
stopWatch.Reset();
}
else if (ex is AmqpException)
{
throw activeEx;
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (Exception ex)
{
EventHubsEventSource.Log.EventPublishError(EventHubName, logPartition, messageHash, ex.Message);
throw;
}
finally
{
stopWatch.Stop();
EventHubsEventSource.Log.EventPublishComplete(EventHubName, logPartition, messageHash);
}
}
/// <summary>
/// Creates a size-constraint batch to which <see cref="EventData" /> may be added using a try-based pattern. If an event would
/// exceed the maximum allowable size of the batch, the batch will not allow adding the event and signal that scenario using its
/// return value.
///
/// Because events that would violate the size constraint cannot be added, publishing a batch will not trigger an exception when
/// attempting to send the events to the Event Hubs service.
/// </summary>
///
/// <param name="options">The set of options to consider when creating this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>An <see cref="EventDataBatch" /> with the requested <paramref name="options"/>.</returns>
///
public override async ValueTask <TransportEventBatch> CreateBatchAsync(CreateBatchOptions options,
CancellationToken cancellationToken)
{
Argument.AssertNotNull(options, nameof(options));
Argument.AssertNotClosed(_closed, nameof(AmqpProducer));
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
// Ensure that maximum message size has been determined; this depends on the underlying
// AMQP link, so if not set, requesting the link will ensure that it is populated.
if (!MaximumMessageSize.HasValue)
{
var failedAttemptCount = 0;
var retryDelay = default(TimeSpan?);
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
while (!cancellationToken.IsCancellationRequested)
{
try
{
await SendLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout)).ConfigureAwait(false);
break;
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EventHubName);
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!cancellationToken.IsCancellationRequested))
{
await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else if (ex is AmqpException)
{
throw activeEx;
}
else
{
throw;
}
}
}
// If MaximumMessageSize has not been populated nor exception thrown
// by this point, then cancellation has been requested.
if (!MaximumMessageSize.HasValue)
{
throw new TaskCanceledException();
}
}
// Ensure that there was a maximum size populated; if none was provided,
// default to the maximum size allowed by the link.
options.MaximumSizeInBytes ??= MaximumMessageSize;
Argument.AssertInRange(options.MaximumSizeInBytes.Value, EventHubProducerClient.MinimumBatchSizeLimit, MaximumMessageSize.Value, nameof(options.MaximumSizeInBytes));
return(new AmqpEventBatch(MessageConverter, options));
}
/// <summary>
/// Receives a batch of <see cref="EventData" /> from the Event Hub partition.
/// </summary>
///
/// <param name="maximumMessageCount">The maximum number of messages to receive in this batch.</param>
/// <param name="maximumWaitTime">The maximum amount of time to wait to build up the requested message count for the batch; if not specified, the per-try timeout specified by the retry policy will be used.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The batch of <see cref="EventData" /> from the Event Hub partition this consumer is associated with. If no events are present, an empty set is returned.</returns>
///
public override async Task <IReadOnlyList <EventData> > ReceiveAsync(int maximumMessageCount,
TimeSpan?maximumWaitTime,
CancellationToken cancellationToken)
{
Argument.AssertNotClosed(_closed, nameof(AmqpConsumer));
Argument.AssertAtLeast(maximumMessageCount, 1, nameof(maximumMessageCount));
var receivedEventCount = 0;
var failedAttemptCount = 0;
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
var waitTime = (maximumWaitTime ?? tryTimeout);
var link = default(ReceivingAmqpLink);
var retryDelay = default(TimeSpan?);
var amqpMessages = default(IEnumerable <AmqpMessage>);
var receivedEvents = default(List <EventData>);
var lastReceivedEvent = default(EventData);
var stopWatch = ValueStopwatch.StartNew();
try
{
while ((!cancellationToken.IsCancellationRequested) && (!_closed))
{
try
{
// Creation of the link happens without explicit knowledge of the cancellation token
// used for this operation; validate the token state before attempting link creation and
// again after the operation completes to provide best efforts in respecting it.
EventHubsEventSource.Log.EventReceiveStart(EventHubName, ConsumerGroup, PartitionId);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
link = await ReceiveLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
var messagesReceived = await Task.Factory.FromAsync
(
(callback, state) => link.BeginReceiveMessages(maximumMessageCount, waitTime, callback, state),
(asyncResult) => link.EndReceiveMessages(asyncResult, out amqpMessages),
TaskCreationOptions.RunContinuationsAsynchronously
).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
// If event messages were received, then package them for consumption and
// return them.
if ((messagesReceived) && (amqpMessages != null))
{
receivedEvents ??= new List <EventData>();
foreach (AmqpMessage message in amqpMessages)
{
link.DisposeDelivery(message, true, AmqpConstants.AcceptedOutcome);
receivedEvents.Add(MessageConverter.CreateEventFromMessage(message));
message.Dispose();
}
receivedEventCount = receivedEvents.Count;
if (receivedEventCount > 0)
{
lastReceivedEvent = receivedEvents[receivedEventCount - 1];
if (lastReceivedEvent.Offset > long.MinValue)
{
CurrentEventPosition = EventPosition.FromOffset(lastReceivedEvent.Offset, false);
}
if (TrackLastEnqueuedEventProperties)
{
LastReceivedEvent = lastReceivedEvent;
}
}
return(receivedEvents);
}
// No events were available.
return(EmptyEventSet);
}
catch (EventHubsException ex) when(ex.Reason == EventHubsException.FailureReason.ServiceTimeout)
{
// Because the timeout specified with the request is intended to be the maximum
// amount of time to wait for events, a timeout isn't considered an error condition,
// rather a sign that no events were available in the requested period.
return(EmptyEventSet);
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EventHubName);
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!cancellationToken.IsCancellationRequested))
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, activeEx.Message);
await Task.Delay(UseMinimum(retryDelay.Value, waitTime.CalculateRemaining(stopWatch.GetElapsedTime())), cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else if (ex is AmqpException)
{
ExceptionDispatchInfo.Capture(activeEx).Throw();
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, ex.Message);
throw;
}
finally
{
EventHubsEventSource.Log.EventReceiveComplete(EventHubName, ConsumerGroup, PartitionId, receivedEventCount);
}
}
/// <summary>
/// Begins the background process responsible for receiving from the specified <see cref="TransportConsumer" />
/// and publishing to the requested <see cref="Channel{PartitionEvent}" />.
/// </summary>
///
/// <param name="transportConsumer">The consumer to use for receiving events.</param>
/// <param name="channel">The channel to which received events should be published.</param>
/// <param name="partitionContext">The context that represents the partition from which events being received.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to signal the request to cancel the background publishing.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private Task StartBackgroundChannelPublishingAsync(TransportConsumer transportConsumer,
Channel <PartitionEvent> channel,
PartitionContext partitionContext,
CancellationToken cancellationToken) =>
Task.Run(async() =>
{
var failedAttemptCount = 0;
var receivedItems = default(IEnumerable <EventData>);
var retryDelay = default(TimeSpan?);
var activeException = default(Exception);
while (!cancellationToken.IsCancellationRequested)
{
try
{
// Receive items in batches and then write them to the subscribed channels. The channels will naturally
// block if they reach their maximum queue size, so there is no need to throttle publishing.
receivedItems = await transportConsumer.ReceiveAsync(BackgroundPublishReceiveBatchSize, BackgroundPublishingWaitTime, cancellationToken).ConfigureAwait(false);
foreach (EventData item in receivedItems)
{
await channel.Writer.WriteAsync(new PartitionEvent(partitionContext, item), cancellationToken).ConfigureAwait(false);
}
failedAttemptCount = 0;
}
catch (TaskCanceledException ex)
{
// In the case that a task was canceled, if cancellation was requested, then publishing should
// is already terminating. Otherwise, something unexpected canceled the operation and it should
// be treated as an exception to ensure that the channels are marked final and consumers are made
// aware.
activeException = (cancellationToken.IsCancellationRequested) ? null : ex;
break;
}
catch (ConsumerDisconnectedException ex)
{
// If the consumer was disconnected, it is known to be unrecoverable; do not offer the chance to retry.
activeException = ex;
break;
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
channel.Writer.TryComplete(ex);
throw;
}
catch (Exception ex)
{
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, mark the exception as active and break out of the loop.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount);
if (retryDelay.HasValue)
{
await Task.Delay(retryDelay.Value).ConfigureAwait(false);
activeException = null;
}
else
{
activeException = ex;
break;
}
}
}
// Publishing has terminated; if there was an active exception, then take the necessary steps to mark publishing as aborted rather
// than completed normally.
if (activeException != null)
{
channel.Writer.TryComplete(activeException);
}
}, cancellationToken);
/// <summary>
/// Receives a batch of <see cref="EventData" /> from the Event Hub partition.
/// </summary>
///
/// <param name="maximumMessageCount">The maximum number of messages to receive in this batch.</param>
/// <param name="maximumWaitTime">The maximum amount of time to wait to build up the requested message count for the batch; if not specified, the per-try timeout specified by the retry policy will be used.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The batch of <see cref="EventData" /> from the Event Hub partition this consumer is associated with. If no events are present, an empty enumerable is returned.</returns>
///
public override async Task <IEnumerable <EventData> > ReceiveAsync(int maximumMessageCount,
TimeSpan?maximumWaitTime,
CancellationToken cancellationToken)
{
Argument.AssertNotClosed(_closed, nameof(AmqpConsumer));
Argument.AssertAtLeast(maximumMessageCount, 1, nameof(maximumMessageCount));
var receivedEventCount = 0;
var failedAttemptCount = 0;
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
var waitTime = (maximumWaitTime ?? tryTimeout);
var link = default(ReceivingAmqpLink);
var retryDelay = default(TimeSpan?);
var amqpMessages = default(IEnumerable <AmqpMessage>);
var receivedEvents = default(List <EventData>);
var stopWatch = Stopwatch.StartNew();
try
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
EventHubsEventSource.Log.EventReceiveStart(EventHubName, ConsumerGroup, PartitionId);
link = await ReceiveLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
var messagesReceived = await Task.Factory.FromAsync
(
(callback, state) => link.BeginReceiveMessages(maximumMessageCount, waitTime, callback, state),
(asyncResult) => link.EndReceiveMessages(asyncResult, out amqpMessages),
TaskCreationOptions.RunContinuationsAsynchronously
).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
// If event messages were received, then package them for consumption and
// return them.
if ((messagesReceived) && (amqpMessages != null))
{
receivedEvents ??= new List <EventData>();
foreach (AmqpMessage message in amqpMessages)
{
link.DisposeDelivery(message, true, AmqpConstants.AcceptedOutcome);
receivedEvents.Add(MessageConverter.CreateEventFromMessage(message));
message.Dispose();
}
receivedEventCount = receivedEvents.Count;
if ((TrackLastEnqueuedEventInformation) && (receivedEventCount > 0))
{
LastReceivedEvent = receivedEvents[receivedEventCount - 1];
}
return(receivedEvents);
}
// No events were available.
return(Enumerable.Empty <EventData>());
}
catch (EventHubsTimeoutException)
{
// Because the timeout specified with the request is intended to be the maximum
// amount of time to wait for events, a timeout isn't considered an error condition,
// rather a sign that no events were available in the requested period.
return(Enumerable.Empty <EventData>());
}
catch (AmqpException amqpException)
{
throw AmqpError.CreateExceptionForError(amqpException.Error, EventHubName);
}
catch (Exception ex)
{
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!cancellationToken.IsCancellationRequested))
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, ex.Message);
await Task.Delay(UseMinimum(retryDelay.Value, waitTime.CalculateRemaining(stopWatch.Elapsed)), cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (Exception ex)
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, ex.Message);
throw;
}
finally
{
stopWatch.Stop();
EventHubsEventSource.Log.EventReceiveComplete(EventHubName, ConsumerGroup, PartitionId, receivedEventCount);
}
}
/// <summary>
/// Sends an AMQP message that contains a batch of events to the associated Event Hub. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="messages">The set of AMQP messages to packaged in a batch envelope and sent.</param>
/// <param name="partitionKey">The hashing key to use for influencing the partition to which events should be routed.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <remarks>
/// Callers retain ownership of the <paramref name="messages" /> passed and hold responsibility for
/// ensuring that they are disposed.
/// </remarks>
///
protected virtual async Task SendAsync(IReadOnlyCollection <AmqpMessage> messages,
string partitionKey,
CancellationToken cancellationToken)
{
var failedAttemptCount = 0;
var logPartition = PartitionId ?? partitionKey;
var operationId = Guid.NewGuid().ToString("D", CultureInfo.InvariantCulture);
TimeSpan? retryDelay;
SendingAmqpLink link;
try
{
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
while (!cancellationToken.IsCancellationRequested)
{
EventHubsEventSource.Log.EventPublishStart(EventHubName, logPartition, operationId);
try
{
using AmqpMessage batchMessage = MessageConverter.CreateBatchFromMessages(messages);
if (!SendLink.TryGetOpenedObject(out link))
{
link = await SendLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout), cancellationToken).ConfigureAwait(false);
}
// Validate that the batch of messages is not too large to send. This is done after the link is created to ensure
// that the maximum message size is known, as it is dictated by the service using the link.
if (batchMessage.SerializedMessageSize > MaximumMessageSize)
{
throw new EventHubsException(EventHubName, string.Format(CultureInfo.CurrentCulture, Resources.MessageSizeExceeded, operationId, batchMessage.SerializedMessageSize, MaximumMessageSize), EventHubsException.FailureReason.MessageSizeExceeded);
}
// Attempt to send the message batch.
var deliveryTag = new ArraySegment <byte>(BitConverter.GetBytes(Interlocked.Increment(ref _deliveryCount)));
var outcome = await link.SendMessageAsync(batchMessage, deliveryTag, AmqpConstants.NullBinary, cancellationToken).ConfigureAwait(false);
if (outcome.DescriptorCode != Accepted.Code)
{
throw AmqpError.CreateExceptionForError((outcome as Rejected)?.Error, EventHubName);
}
// The send operation should be considered successful; return to
// exit the retry loop.
return;
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EventHubName);
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!_closed) && (!cancellationToken.IsCancellationRequested))
{
EventHubsEventSource.Log.EventPublishError(EventHubName, logPartition, operationId, activeEx.Message);
await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else if (ex is AmqpException)
{
ExceptionDispatchInfo.Capture(activeEx).Throw();
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
EventHubsEventSource.Log.EventPublishError(EventHubName, logPartition, operationId, ex.Message);
throw;
}
finally
{
EventHubsEventSource.Log.EventPublishComplete(EventHubName, logPartition, operationId, failedAttemptCount);
}
}
/// <summary>
/// Receives a batch of <see cref="EventData" /> from the Event Hub partition.
/// </summary>
///
/// <param name="maximumEventCount">The maximum number of messages to receive in this batch.</param>
/// <param name="maximumWaitTime">The maximum amount of time to wait for events to become available, if no events can be read from the prefetch queue. If not specified, the per-try timeout specified by the retry policy will be used.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The batch of <see cref="EventData" /> from the Event Hub partition this consumer is associated with. If no events are present, an empty set is returned.</returns>
///
/// <remarks>
/// When events are available in the prefetch queue, they will be used to form the batch as quickly as possible without waiting for additional events from the
/// Event Hubs service to try and meet the requested <paramref name="maximumEventCount" />. When no events are available in prefetch, the receiver will wait up
/// to the <paramref name="maximumWaitTime"/> for events to be read from the service. Once any events are available, they will be used to form the batch immediately.
/// </remarks>
///
public override async Task <IReadOnlyList <EventData> > ReceiveAsync(int maximumEventCount,
TimeSpan?maximumWaitTime,
CancellationToken cancellationToken)
{
Argument.AssertNotClosed(_closed, nameof(AmqpConsumer));
Argument.AssertNotClosed(ConnectionScope.IsDisposed, nameof(EventHubConnection));
Argument.AssertAtLeast(maximumEventCount, 1, nameof(maximumEventCount));
var receivedEventCount = 0;
var failedAttemptCount = 0;
var tryTimeout = RetryPolicy.CalculateTryTimeout(0);
var waitTime = (maximumWaitTime ?? tryTimeout);
var operationId = Guid.NewGuid().ToString("D", CultureInfo.InvariantCulture);
var link = default(ReceivingAmqpLink);
var retryDelay = default(TimeSpan?);
var receivedEvents = default(List <EventData>);
var lastReceivedEvent = default(EventData);
var stopWatch = ValueStopwatch.StartNew();
try
{
while ((!cancellationToken.IsCancellationRequested) && (!_closed))
{
try
{
// Creation of the link happens without explicit knowledge of the cancellation token
// used for this operation; validate the token state before attempting link creation and
// again after the operation completes to provide best efforts in respecting it.
EventHubsEventSource.Log.EventReceiveStart(EventHubName, ConsumerGroup, PartitionId, operationId);
link = await ReceiveLink.GetOrCreateAsync(UseMinimum(ConnectionScope.SessionTimeout, tryTimeout), cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
var messagesReceived = await link.ReceiveMessagesAsync(maximumEventCount, ReceiveBuildBatchInterval, waitTime, cancellationToken).ConfigureAwait(false);
// If no messages were received, then just return the empty set.
if (messagesReceived == null)
{
return(EmptyEventSet);
}
// If event messages were received, then package them for consumption and
// return them.
foreach (AmqpMessage message in messagesReceived)
{
receivedEvents ??= new List <EventData>();
link.DisposeDelivery(message, true, AmqpConstants.AcceptedOutcome);
receivedEvents.Add(MessageConverter.CreateEventFromMessage(message));
message.Dispose();
receivedEventCount = receivedEvents.Count;
}
if (receivedEventCount > 0)
{
lastReceivedEvent = receivedEvents[receivedEventCount - 1];
if (lastReceivedEvent.Offset > long.MinValue)
{
CurrentEventPosition = EventPosition.FromOffset(lastReceivedEvent.Offset, false);
}
if (TrackLastEnqueuedEventProperties)
{
LastReceivedEvent = lastReceivedEvent;
}
}
return(receivedEvents ?? EmptyEventSet);
}
catch (EventHubsException ex) when(ex.Reason == EventHubsException.FailureReason.ServiceTimeout)
{
// Because the timeout specified with the request is intended to be the maximum
// amount of time to wait for events, a timeout isn't considered an error condition,
// rather a sign that no events were available in the requested period.
return(EmptyEventSet);
}
catch (Exception ex)
{
Exception activeEx = ex.TranslateServiceException(EventHubName);
// If the partition was stolen determine the correct action to take for
// capturing it with respect to whether the consumer should be invalidated.
//
// In either case, it is a terminal exception and will not trigger a retry;
// allow the normal error handling flow to surface the exception.
if (ex.IsConsumerPartitionStolenException())
{
// If the consumer should be invalidated, capture the exception
// and force-close the link. This will ensure that the next operation
// will surface it.
if (InvalidateConsumerWhenPartitionStolen)
{
_activePartitionStolenException = ex;
CloseConsumerLink(link);
}
else
{
// If the consumer should not be invalidated, clear any previously captured exception to avoid
// surfacing the failure multiple times. If the link is stolen after this operation, it will
// be intercepted and handled as needed.
_activePartitionStolenException = null;
}
}
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, bubble the exception.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(activeEx, failedAttemptCount);
if ((retryDelay.HasValue) && (!ConnectionScope.IsDisposed) && (!_closed) && (!cancellationToken.IsCancellationRequested))
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, operationId, activeEx.Message);
await Task.Delay(UseMinimum(retryDelay.Value, waitTime.CalculateRemaining(stopWatch.GetElapsedTime())), cancellationToken).ConfigureAwait(false);
tryTimeout = RetryPolicy.CalculateTryTimeout(failedAttemptCount);
}
else if (ex is AmqpException)
{
ExceptionDispatchInfo.Capture(activeEx).Throw();
}
else
{
throw;
}
}
}
// If no value has been returned nor exception thrown by this point,
// then cancellation has been requested.
throw new TaskCanceledException();
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
EventHubsEventSource.Log.EventReceiveError(EventHubName, ConsumerGroup, PartitionId, operationId, ex.Message);
throw;
}
finally
{
EventHubsEventSource.Log.EventReceiveComplete(EventHubName, ConsumerGroup, PartitionId, operationId, failedAttemptCount, receivedEventCount);
}
}
/// <summary>
/// Begins the background process responsible for receiving from the partition
/// and publishing to all subscribed channels.
/// </summary>
///
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to signal the request to cancel the background publishing.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private Task StartBackgroundChannelPublishingAsync(CancellationToken cancellationToken) =>
Task.Run(async() =>
{
var failedAttemptCount = 0;
var publishTasks = new List <Task>();
var publishChannels = default(ICollection <Channel <EventData> >);
var receivedItems = default(IEnumerable <EventData>);
var retryDelay = default(TimeSpan?);
var activeException = default(Exception);
while (!cancellationToken.IsCancellationRequested)
{
try
{
// Receive items in batches and then write them to the subscribed channels. The channels will naturally
// block if they reach their maximum queue size, so there is no need to throttle publishing.
receivedItems = await InnerConsumer.ReceiveAsync(BackgroundPublishReceiveBatchSize, _backgroundPublishingWaitTime, cancellationToken).ConfigureAwait(false);
publishChannels = _activeChannels.Values;
foreach (EventData item in receivedItems)
{
foreach (Channel <EventData> channel in publishChannels)
{
publishTasks.Add(channel.Writer.WriteAsync(item, cancellationToken).AsTask());
}
}
await Task.WhenAll(publishTasks).ConfigureAwait(false);
publishTasks.Clear();
failedAttemptCount = 0;
}
catch (TaskCanceledException ex)
{
// In the case that a task was canceled, if cancellation was requested, then publishing should
// is already terminating. Otherwise, something unexpected canceled the operation and it should
// be treated as an exception to ensure that the channels are marked final and consumers are made
// aware.
activeException = (cancellationToken.IsCancellationRequested) ? null : ex;
break;
}
catch (ConsumerDisconnectedException ex)
{
// If the consumer was disconnected, it is known to be unrecoverable; do not offer the chance to retry.
activeException = ex;
break;
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
// These exceptions are known to be unrecoverable and there should be no attempts at further processing.
// The environment is in a bad state and is likely to fail.
//
// Attempt to clean up, which may or may not fail due to resource constraints,
// then let the exception bubble.
_isPublishingActive = false;
foreach (Channel <EventData> channel in _activeChannels.Values)
{
channel.Writer.TryComplete(ex);
}
throw;
}
catch (Exception ex)
{
// Determine if there should be a retry for the next attempt; if so enforce the delay but do not quit the loop.
// Otherwise, mark the exception as active and break out of the loop.
++failedAttemptCount;
retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount);
if (retryDelay.HasValue)
{
await Task.Delay(retryDelay.Value).ConfigureAwait(false);
activeException = null;
}
else
{
activeException = ex;
break;
}
}
}
// Publishing has terminated; if there was an active exception, then take the necessary steps to mark publishing as aborted rather
// than completed normally.
if (activeException != null)
{
await AbortBackgroundChannelPublishingAsync(activeException).ConfigureAwait(false);
}
}, cancellationToken);