internal async Task <EventPosition> GetInitialOffsetAsync() // throws InterruptedException, ExecutionException
{
Checkpoint startingCheckpoint = await this.host.CheckpointManager.GetCheckpointAsync(this.PartitionId).ConfigureAwait(false);
EventPosition eventPosition;
if (startingCheckpoint == null)
{
// No checkpoint was ever stored. Use the initialOffsetProvider instead.
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, this.PartitionId, "Calling user-provided initial offset provider");
eventPosition = this.host.EventProcessorOptions.InitialOffsetProvider(this.PartitionId);
ProcessorEventSource.Log.PartitionPumpInfo(
this.host.HostName,
this.PartitionId,
$"Initial Position Provider. Offset:{eventPosition.Offset}, SequenceNumber:{eventPosition.SequenceNumber}, DateTime:{eventPosition.EnqueuedTimeUtc}");
}
else
{
this.Offset = startingCheckpoint.Offset;
this.SequenceNumber = startingCheckpoint.SequenceNumber;
ProcessorEventSource.Log.PartitionPumpInfo(
this.host.HostName,
this.PartitionId, $"Retrieved starting offset/sequenceNumber: {this.Offset}/{this.SequenceNumber}");
eventPosition = EventPosition.FromOffset(this.Offset);
}
return(eventPosition);
}
private async Task <Tuple <EventPosition, long> > GetOffsetAndEpochAsync(IReliableDictionary <string, string> offsetDictionary, IReliableDictionary <string, long> epochDictionary)
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
EventPosition eventPosition;
if (offsetResult.HasValue)
{
eventPosition = EventPosition.FromOffset(offsetResult.Value);
}
else
{
// TODO: make this configurable behaviour
eventPosition = EventPosition.FromStart();
}
return(new Tuple <EventPosition, long>(eventPosition, newEpoch));
}
}
public void CreateConsumerInvokesTheTransportClient()
{
var transportClient = new ObservableTransportClientMock();
var client = new InjectableTransportClientMock(transportClient, "Endpoint=sb://not-real.servicebus.windows.net/;SharedAccessKeyName=DummyKey;SharedAccessKey=[not_real];EntityPath=fake");
var expectedPosition = EventPosition.FromOffset(65);
var expectedPartition = "2123";
var expectedConsumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var expectedRetryPolicy = new EventHubsRetryOptions {
MaximumRetries = 67
}.ToRetryPolicy();
var expectedTrackLastEnqueued = false;
var expectedPrefetch = 99U;
var expectedOwnerLevel = 123L;
client.CreateTransportConsumer(expectedConsumerGroup, expectedPartition, expectedPosition, expectedRetryPolicy, expectedTrackLastEnqueued, expectedOwnerLevel, expectedPrefetch);
(var actualConsumerGroup, var actualPartition, EventPosition actualPosition, var actualRetry, var actualTrackLastEnqueued, var actualOwnerLevel, var actualPrefetch) = transportClient.CreateConsumerCalledWith;
Assert.That(actualPartition, Is.EqualTo(expectedPartition), "The partition should have been passed.");
Assert.That(actualConsumerGroup, Is.EqualTo(expectedConsumerGroup), "The consumer groups should match.");
Assert.That(actualPosition.Offset, Is.EqualTo(expectedPosition.Offset), "The event position to receive should match.");
Assert.That(actualRetry, Is.SameAs(expectedRetryPolicy), "The retryPolicy should match.");
Assert.That(actualOwnerLevel, Is.EqualTo(expectedOwnerLevel), "The owner levels should match.");
Assert.That(actualPrefetch, Is.EqualTo(expectedPrefetch), "The prefetch counts should match.");
Assert.That(actualTrackLastEnqueued, Is.EqualTo(expectedTrackLastEnqueued), "The flag for tracking the last enqueued event should match.");
}
public void GetHashCodeReturnsDifferentValuesForDifferentMembers()
{
var first = EventPosition.FromOffset(12);
var second = EventPosition.FromSequenceNumber(123);
Assert.That(first.GetHashCode(), Is.Not.EqualTo(second.GetHashCode()));
}
public void IsEquivalentToDetectsOffset()
{
var first = EventPosition.FromOffset(42);
var second = EventPosition.FromOffset(1975);
Assert.That(first.IsEquivalentTo(second), Is.False);
}
public void CreateConsumerInvokesTheTransportClient()
{
var transportClient = new ObservableTransportClientMock();
var client = new InjectableTransportClientMock(transportClient, "Endpoint=sb://not-real.servicebus.windows.net/;SharedAccessKeyName=DummyKey;SharedAccessKey=[not_real];EntityPath=fake");
var expectedOptions = new EventHubConsumerOptions {
Retry = Retry.Default
};
var expectedPosition = EventPosition.FromOffset(65);
var expectedPartition = "2123";
var expectedConsumerGroup = EventHubConsumer.DefaultConsumerGroup;
client.CreateConsumer(expectedConsumerGroup, expectedPartition, expectedPosition, expectedOptions);
(var actualConsumerGroup, var actualPartition, var actualPosition, var actualOptions) = transportClient.CreateConsumerCalledWith;
Assert.That(actualPartition, Is.EqualTo(expectedPartition), "The partition should have been passed.");
Assert.That(actualConsumerGroup, Is.EqualTo(expectedConsumerGroup), "The consumer groups should match.");
Assert.That(actualPosition.Offset, Is.EqualTo(expectedPosition.Offset), "The event position to receive should match.");
Assert.That(actualOptions, Is.Not.Null, "The consumer options should have been set.");
Assert.That(actualPosition.Offset, Is.EqualTo(expectedPosition.Offset), "The event position to receive should match.");
Assert.That(actualOptions.OwnerLevel, Is.EqualTo(expectedOptions.OwnerLevel), "The owner levels should match.");
Assert.That(actualOptions.Identifier, Is.EqualTo(expectedOptions.Identifier), "The identifiers should match.");
Assert.That(actualOptions.PrefetchCount, Is.EqualTo(expectedOptions.PrefetchCount), "The prefetch counts should match.");
Assert.That(ExponentialRetry.HaveSameConfiguration((ExponentialRetry)actualOptions.Retry, (ExponentialRetry)expectedOptions.Retry), "The retries should match.");
Assert.That(actualOptions.MaximumReceiveWaitTimeOrDefault, Is.EqualTo(expectedOptions.MaximumReceiveWaitTimeOrDefault), "The wait times should match.");
}
public void CloneProducesACopy()
{
var options = new ReceiverOptions
{
ConsumerGroup = "custom$consumer",
BeginReceivingAt = EventPosition.FromOffset(65),
ExclusiveReceiverPriority = 99,
Retry = new ExponentialRetry(TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(5), 6),
DefaultMaximumReceiveWaitTime = TimeSpan.FromMinutes(65),
Identifier = "an_event_receiver"
};
var clone = options.Clone();
Assert.That(clone, Is.Not.Null, "The clone should not be null.");
Assert.That(clone.ConsumerGroup, Is.EqualTo(options.ConsumerGroup), "The consumer group of the clone should match.");
Assert.That(clone.BeginReceivingAt, Is.EqualTo(options.BeginReceivingAt), "The position to begin reading events of the clone should match.");
Assert.That(clone.ExclusiveReceiverPriority, Is.EqualTo(options.ExclusiveReceiverPriority), "The exclusive priority of the clone should match.");
Assert.That(clone.DefaultMaximumReceiveWaitTime, Is.EqualTo(options.DefaultMaximumReceiveWaitTime), "The default maximum wait time of the clone should match.");
Assert.That(clone.Identifier, Is.EqualTo(options.Identifier), "The identifier of the clone should match.");
Assert.That(ExponentialRetry.HaveSameConfiguration((ExponentialRetry)clone.Retry, (ExponentialRetry)options.Retry), "The retry of the clone should be considered equal.");
Assert.That(clone.Retry, Is.Not.SameAs(options.Retry), "The retry of the clone should be a copy, not the same instance.");
}
public void IsEventPositionEquivalentRecognizesSameOffsets()
{
var trackOnePosition = TrackOne.EventPosition.FromOffset("12", true);
var trackTwoPosition = EventPosition.FromOffset(12);
Assert.That(TrackOneComparer.IsEventPositionEquivalent(trackOnePosition, trackTwoPosition), Is.True, "The offset for track two is inclusive; the equivalent offset set as inclusive should match.");
}
async Task InitialOffsetProviderWithOffset()
{
// Send and receive single message so we can find out offset of the last message.
var partitions = await DiscoverEndOfStream();
TestUtility.Log("Discovered last event offsets on each partition as below:");
foreach (var p in partitions)
{
TestUtility.Log($"Partition {p.Key}: {p.Value.Item1}");
}
// Use a randomly generated container name so that initial offset provider will be respected.
var eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
TestUtility.EventHubsConnectionString,
TestUtility.StorageConnectionString,
Guid.NewGuid().ToString());
var processorOptions = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
InitialOffsetProvider = partitionId => EventPosition.FromOffset(partitions[partitionId].Item1),
MaxBatchSize = 100
};
var runResult = await this.RunGenericScenario(eventProcessorHost, processorOptions);
// We should have received only 1 event from each partition.
Assert.False(runResult.ReceivedEvents.Any(kvp => kvp.Value.Count != 1), "One of the partitions didn't return exactly 1 event");
}
public void IsEventPositionEquivalentDetectsDifferentOffsets()
{
var trackOnePosition = TrackOne.EventPosition.FromOffset("12", false);
var trackTwoPosition = EventPosition.FromOffset(12);
Assert.That(TrackOneComparer.IsEventPositionEquivalent(trackOnePosition, trackTwoPosition), Is.False, "The offset for track two is inclusive; even the same base offset with non-inclusive is not equivalent.");
}
public void ReceiveAsyncDetectsAnEmbeddedAmqpErrorForOperationCanceled()
{
var eventHub = "eventHubName";
var consumerGroup = "$DEFAULT";
var partition = "3";
var identifier = "cusTOM-1D";
var eventPosition = EventPosition.FromOffset(123);
var trackLastEnqueued = false;
var invalidateOnSteal = true;
var ownerLevel = 123L;
var tokenValue = "123ABC";
var retryPolicy = new BasicRetryPolicy(new EventHubsRetryOptions());
var embeddedException = new OperationCanceledException("", new AmqpException(new Error {
Condition = AmqpError.ArgumentError
}));
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <TokenCredential>();
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
mockCredential
.Setup(credential => credential.GetTokenAsync(It.IsAny <TokenRequestContext>(), It.Is <CancellationToken>(value => value == cancellationSource.Token)))
.Returns(new ValueTask <AccessToken>(new AccessToken(tokenValue, DateTimeOffset.MaxValue)));
mockScope
.Setup(scope => scope.OpenConsumerLinkAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <TimeSpan>(),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.IsAny <long?>(),
It.IsAny <bool>(),
It.IsAny <string>(),
It.IsAny <CancellationToken>()))
.Throws(embeddedException);
var consumer = new AmqpConsumer(eventHub, consumerGroup, partition, identifier, eventPosition, trackLastEnqueued, invalidateOnSteal, ownerLevel, null, null, mockScope.Object, Mock.Of <AmqpMessageConverter>(), retryPolicy);
Assert.That(async() => await consumer.ReceiveAsync(100, null, cancellationSource.Token), Throws.InstanceOf <OperationCanceledException>());
mockScope
.Verify(scope => scope.OpenConsumerLinkAsync(
It.Is <string>(value => value == consumerGroup),
It.Is <string>(value => value == partition),
It.Is <EventPosition>(value => value == eventPosition),
It.IsAny <TimeSpan>(),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.Is <long?>(value => value == ownerLevel),
It.Is <bool>(value => value == trackLastEnqueued),
It.Is <string>(value => value == identifier),
It.IsAny <CancellationToken>()),
Times.Once());
}
protected override async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(CancellationToken cancellationToken)
{
// First, we read information from the location that the EventHubs V5 SDK writes to.
Dictionary <string, EventProcessorCheckpoint> checkpoints = new Dictionary <string, EventProcessorCheckpoint>();
string checkpointBlobsPrefix = string.Format(CultureInfo.InvariantCulture, CheckpointPrefixFormat, FullyQualifiedNamespace.ToLowerInvariant(), EventHubName.ToLowerInvariant(), ConsumerGroup.ToLowerInvariant());
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: checkpointBlobsPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
if (long.TryParse(item.Metadata[OffsetMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long offset) &&
long.TryParse(item.Metadata[SequenceNumberMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long sequenceNumber))
{
string partitionId = item.Name.Substring(checkpointBlobsPrefix.Length);
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, sequenceNumber));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
// Check to see if there are any additional checkpoints in the older location that the V4 SDK would write to. If so, use them (this is helpful when moving from the V4 to V5 SDK,
// since it means we will not have to reprocess messages processed and checkpointed by the older SDK).
string legacyCheckpointAndOwnershipPrefix = $"{LegacyCheckpointStorageBlobPrefix}{ConsumerGroup}/";
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(prefix: legacyCheckpointAndOwnershipPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
string partitionId = item.Name.Substring(legacyCheckpointAndOwnershipPrefix.Length);
if (!checkpoints.ContainsKey(partitionId))
{
using MemoryStream checkpointStream = new MemoryStream();
await ContainerClient.GetBlobClient(item.Name).DownloadToAsync(checkpointStream, cancellationToken: cancellationToken).ConfigureAwait(false);
checkpointStream.Position = 0;
BlobPartitionLease lease = await JsonSerializer.DeserializeAsync <BlobPartitionLease>(checkpointStream, cancellationToken : cancellationToken).ConfigureAwait(false);
if (long.TryParse(lease.Offset, out long offset))
{
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, lease.SequenceNumber ?? 0));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
}
return(checkpoints.Values);
}
private PartitionReceiver CreateReceiver(ILogger log, Configuration configuration, string partition, string offset)
{
var epoch = DateTime.UtcNow.Ticks;
if (string.IsNullOrWhiteSpace(offset))
{
if (DefaultAge.HasValue)
{
var timestamp = DateTime.UtcNow.Subtract(DefaultAge.Value);
var position = EventPosition.FromEnqueuedTime(timestamp);
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from time {Timestamp}",
epoch, configuration.ConsumerGroup, partition, timestamp);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from time {Timestamp}",
configuration.ConsumerGroup, partition, timestamp);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
else
{
var position = DefaultPosition;
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from position {Position}",
epoch, configuration.ConsumerGroup, partition, position);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from offset {Position}",
configuration.ConsumerGroup, partition, position);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
}
else
{
var position = EventPosition.FromOffset(offset);
if (UseEpochReceiver)
{
log.Debug("Creating epoch {Epoch} receiver for {Consumer}#{Partition} from saved offset {Offset}",
epoch, configuration.ConsumerGroup, partition, offset);
return(configuration.Client.CreateEpochReceiver(configuration.ConsumerGroup, partition, position, epoch));
}
else
{
log.Debug("Creating receiver for {Consumer}#{Partition} from saved offset {Offset}",
configuration.ConsumerGroup, partition, offset);
return(configuration.Client.CreateReceiver(configuration.ConsumerGroup, partition, position));
}
}
}
/// <summary>
/// Determines the default start position for processing when no checkpoint is found
/// </summary>
/// <returns>An event position indicating the startup logic</returns>
public static EventPosition GetStartPosition()
{
EventPosition startPosition;
var startDefinition = Environment.GetEnvironmentVariable("StartPosition");
if (!bool.TryParse(Environment.GetEnvironmentVariable("StartInclusive"), out var startInclusive))
{
startInclusive = false;
}
if (string.IsNullOrWhiteSpace(startDefinition))
{
startDefinition = "Start";
}
startDefinition = startDefinition.Trim().ToLowerInvariant();
if (string.IsNullOrWhiteSpace(startDefinition))
{
startDefinition = "Start";
}
switch (startDefinition)
{
case "end":
startPosition = EventPosition.FromEnd();
break;
case "offset":
startPosition = EventPosition.FromOffset(Environment.GetEnvironmentVariable("StartOffset"), startInclusive);
break;
case "sequence":
if (!long.TryParse(Environment.GetEnvironmentVariable("StartSequence"), out var startSequence))
{
startSequence = -1;
}
startPosition = EventPosition.FromSequenceNumber(startSequence, startInclusive);
break;
case "time":
if (!int.TryParse(Environment.GetEnvironmentVariable("StartSeconds"), out var startSeconds))
{
startSeconds = 0;
}
startPosition = EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddSeconds(startSeconds * -1));
break;
default:
startPosition = EventPosition.FromStart();
break;
}
return(startPosition);
}
public void AnInstanceIsEqualToItself()
{
var first = EventPosition.FromOffset(12);
var second = first;
Assert.That(first.Equals((object)second), Is.True, "The default Equals comparison is incorrect.");
Assert.That(first.Equals(second), Is.True, "The IEquatable comparison is incorrect.");
Assert.That((first == second), Is.True, "The == operator comparison is incorrect.");
Assert.That((first != second), Is.False, "The != operator comparison is incorrect.");
}
public void BuildFilterExpressionEnsuresAnEventPositionIsFilterable()
{
// Unset all properties for the event position.
var position = EventPosition.FromOffset(1);
position.Offset = null;
Assert.That(() => AmqpFilter.BuildFilterExpression(position), Throws.ArgumentException);
}
public void DifferentOffsetsAreNotEqual()
{
var first = EventPosition.FromOffset(12);
var second = EventPosition.FromOffset(34);
Assert.That(first.Equals((object)second), Is.False, "The default Equals comparison is incorrect.");
Assert.That(first.Equals(second), Is.False, "The IEquatable comparison is incorrect.");
Assert.That((first == second), Is.False, "The == operator comparison is incorrect.");
Assert.That((first != second), Is.True, "The != operator comparison is incorrect.");
}
protected async Task <EventProcessorOptions> GetOptionsAsync(string connectionString)
{
var partitions = await DiscoverEndOfStream(connectionString);
return(new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(10),
MaxBatchSize = 100,
InitialOffsetProvider = pId => EventPosition.FromOffset(partitions[pId].Item1)
});
}
public void ReceiveAsyncAppliesTheRetryPolicyForAmqpErrors(EventHubsRetryOptions retryOptions)
{
var eventHub = "eventHubName";
var consumerGroup = "$DEFAULT";
var partition = "3";
var eventPosition = EventPosition.FromOffset(123);
var trackLastEnqueued = false;
var invalidateOnSteal = true;
var ownerLevel = 123L;
var tokenValue = "123ABC";
var retryPolicy = new BasicRetryPolicy(retryOptions);
var retriableException = AmqpError.CreateExceptionForError(new Error {
Condition = AmqpError.ServerBusyError
}, "dummy");
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <TokenCredential>();
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
mockCredential
.Setup(credential => credential.GetTokenAsync(It.IsAny <TokenRequestContext>(), It.Is <CancellationToken>(value => value == cancellationSource.Token)))
.Returns(new ValueTask <AccessToken>(new AccessToken(tokenValue, DateTimeOffset.MaxValue)));
mockScope
.Setup(scope => scope.OpenConsumerLinkAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.IsAny <long?>(),
It.IsAny <bool>(),
It.IsAny <CancellationToken>()))
.Throws(retriableException);
var consumer = new AmqpConsumer(eventHub, consumerGroup, partition, eventPosition, trackLastEnqueued, invalidateOnSteal, ownerLevel, null, null, mockScope.Object, Mock.Of <AmqpMessageConverter>(), retryPolicy);
Assert.That(async() => await consumer.ReceiveAsync(100, null, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
mockScope
.Verify(scope => scope.OpenConsumerLinkAsync(
It.Is <string>(value => value == consumerGroup),
It.Is <string>(value => value == partition),
It.Is <EventPosition>(value => value == eventPosition),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.Is <long?>(value => value == ownerLevel),
It.Is <bool>(value => value == trackLastEnqueued),
It.IsAny <CancellationToken>()),
Times.Exactly(1 + retryOptions.MaximumRetries));
}
public void BuildFilterExpressionHonorsInclusiveFlagForOffset(bool inclusive)
{
var comparison = (inclusive) ? ">=" : ">";
var position = EventPosition.FromOffset(1);
position.IsInclusive = inclusive;
var filter = AmqpFilter.BuildFilterExpression(position);
Assert.That(filter, Contains.Substring(comparison), "The comparison should be based on the inclusive flag.");
}
public async Task ReadPartitionFromOffset()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionFromOffset
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
PartitionProperties properties = await consumer.GetPartitionPropertiesAsync(firstPartition, cancellationSource.Token);
EventPosition startingPosition = EventPosition.FromOffset(properties.LastEnqueuedOffset);
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public EventHubReceiverProxy(EventHubPartitionSettings partitionSettings, string offset, ILogger logger)
{
var receiverOptions = new PartitionReceiverOptions();
if (partitionSettings.ReceiverOptions.PrefetchCount != null)
{
receiverOptions.PrefetchCount = partitionSettings.ReceiverOptions.PrefetchCount.Value;
}
receiverOptions.ConnectionOptions = new EventHubConnectionOptions {
TransportType = partitionSettings.Hub.EventHubsTransportType
};
var options = partitionSettings.Hub;
this.client = options.TokenCredential != null
? new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.FullyQualifiedNamespace, options.Path, options.TokenCredential, receiverOptions)
: new PartitionReceiver(options.ConsumerGroup, partitionSettings.Partition, GetEventPosition(), options.ConnectionString, options.Path, receiverOptions);
EventPosition GetEventPosition()
{
EventPosition eventPosition;
// If we have a starting offset, read from offset
if (offset != EventHubConstants.StartOfStream)
{
if (!long.TryParse(offset, out var longOffset))
{
throw new InvalidOperationException("Offset must be a number.");
}
logger.LogInformation("Starting to read from EventHub partition {0}-{1} at offset {2}", options.Path, partitionSettings.Partition, offset);
eventPosition = EventPosition.FromOffset(longOffset, true);
}
// else, if configured to start from now, start reading from most recent data
else if (partitionSettings.ReceiverOptions.StartFromNow)
{
eventPosition = EventPosition.Latest;
logger.LogInformation("Starting to read latest messages from EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition);
}
else
// else, start reading from begining of the partition
{
eventPosition = EventPosition.Earliest;
logger.LogInformation("Starting to read messages from begining of EventHub partition {0}-{1}.", options.Path, partitionSettings.Partition);
}
return(eventPosition);
}
}
/// <summary>
/// Provides test cases for the equality tests.
/// </summary>
///
public static IEnumerable <object[]> IsEquivalentToDetectsEqualEventPositionsCases()
{
var date = DateTimeOffset.Parse("1975-04-04T00:00:00Z");
yield return(new object[] { EventPosition.Earliest, EventPosition.Earliest });
yield return(new object[] { EventPosition.Latest, EventPosition.Latest });
yield return(new object[] { EventPosition.FromOffset(1975), EventPosition.FromOffset(1975) });
yield return(new object[] { EventPosition.FromSequenceNumber(42), EventPosition.FromSequenceNumber(42) });
yield return(new object[] { EventPosition.FromEnqueuedTime(date), EventPosition.FromEnqueuedTime(date) });
}
/// <summary>
/// Creates a checkpoint instance based on the blob name for a legacy checkpoint format.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace the checkpoint are associated with. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the specific Event Hub the checkpoint is associated with, relative to the Event Hubs namespace that contains it.</param>
/// <param name="consumerGroup">The name of the consumer group the checkpoint is associated with.</param>
/// <param name="partitionId">The partition id the specific checkpoint is associated with.</param>
/// <param name="blobName">The name of the blob that represents the checkpoint.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
///
/// <returns>A <see cref="EventProcessorCheckpoint"/> initialized with checkpoint properties if the checkpoint exists, otherwise <code>null</code>.</returns>
///
private async Task <EventProcessorCheckpoint> CreateLegacyCheckpoint(string fullyQualifiedNamespace,
string eventHubName,
string consumerGroup,
string blobName,
string partitionId,
CancellationToken cancellationToken)
{
var startingPosition = default(EventPosition?);
BlobBaseClient blobClient = ContainerClient.GetBlobClient(blobName);
using var memoryStream = new MemoryStream();
await blobClient.DownloadToAsync(memoryStream, cancellationToken).ConfigureAwait(false);
if (TryReadLegacyCheckpoint(
memoryStream.GetBuffer().AsSpan(0, (int)memoryStream.Length),
out long?offset,
out long?sequenceNumber))
{
if (offset.HasValue)
{
startingPosition = EventPosition.FromOffset(offset.Value, false);
}
else
{
// Skip checkpoints without an offset without logging an error.
return(null);
}
}
if (!startingPosition.HasValue)
{
InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup);
return(null);
}
return(new BlobStorageCheckpoint
{
FullyQualifiedNamespace = fullyQualifiedNamespace,
EventHubName = eventHubName,
ConsumerGroup = consumerGroup,
PartitionId = partitionId,
StartingPosition = startingPosition.Value,
Offset = offset,
SequenceNumber = sequenceNumber
});
}
public void BuildFilterExpressionPrefersOffset()
{
// Set all properties for the event position.
var offset = 1;
var position = EventPosition.FromOffset(offset);
position.SequenceNumber = 222;
position.EnqueuedTime = DateTimeOffset.Parse("2015-10-27T12:00:00Z");
var filter = AmqpFilter.BuildFilterExpression(position);
Assert.That(filter, Contains.Substring(AmqpFilter.OffsetName), "The offset should have precedence for filtering.");
Assert.That(filter, Contains.Substring(offset.ToString()), "The offset value should be present in the filter.");
}
public void ToStringReflectsTheState()
{
var inclusive = true;
var offset = 123;
var sequence = 778;
var enqueued = DateTimeOffset.Now.AddHours(1);
Assert.That(EventPosition.Earliest.ToString(), Contains.Substring(nameof(EventPosition.Earliest)), "Earliest should be represented.");
Assert.That(EventPosition.Latest.ToString(), Contains.Substring(nameof(EventPosition.Latest)), "Latest should be represented.");
Assert.That(EventPosition.FromOffset(offset).ToString(), Contains.Substring($"[{ offset }]"), "The offset should be represented.");
Assert.That(EventPosition.FromSequenceNumber(sequence).ToString(), Contains.Substring($"[{ sequence }]"), "The sequence should be represented.");
Assert.That(EventPosition.FromEnqueuedTime(enqueued).ToString(), Contains.Substring($"[{ enqueued }]"), "The enqueued time should be represented.");
Assert.That(EventPosition.FromOffset(offset, inclusive).ToString(), Contains.Substring($"[{ inclusive }]"), "The inclusive flag should be represented for the offset.");
Assert.That(EventPosition.FromSequenceNumber(sequence, inclusive).ToString(), Contains.Substring($"[{ inclusive }]"), "The inclusive flag should be represented for the sequence number.");
}
public void CreatePartitionReceiverCreatesDefaultWhenOptionsAreNotSet()
{
var clientOptions = new EventHubClientOptions
{
Retry = new ExponentialRetry(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), 5),
DefaultTimeout = TimeSpan.FromHours(24)
};
var expectedOptions = new ReceiverOptions
{
BeginReceivingAt = EventPosition.FromOffset(65),
ConsumerGroup = "SomeGroup",
ExclusiveReceiverPriority = 251,
Identifier = "Bob",
PrefetchCount = 600,
Retry = clientOptions.Retry,
DefaultMaximumReceiveWaitTime = clientOptions.DefaultTimeout
};
var expectedPartition = "56767";
var actualOptions = default(ReceiverOptions);
var actualPartition = default(string);
var connectionString = "Endpoint=value.com;SharedAccessKeyName=[value];SharedAccessKey=[value];EntityPath=[value]";
var mockClient = new Mock <EventHubClient>(connectionString, clientOptions)
{
CallBase = true
};
mockClient
.Protected()
.Setup <PartitionReceiver>("BuildPartitionReceiver", ItExpr.IsAny <ConnectionType>(), ItExpr.IsAny <string>(), ItExpr.IsAny <string>(), ItExpr.IsAny <ReceiverOptions>())
.Returns(Mock.Of <PartitionReceiver>())
.Callback <ConnectionType, string, string, ReceiverOptions>((type, path, partition, options) => { actualOptions = options; actualPartition = partition; });
mockClient.Object.CreatePartitionReceiver(expectedPartition, expectedOptions);
Assert.That(actualPartition, Is.EqualTo(expectedPartition), "The partition should match.");
Assert.That(actualOptions, Is.Not.Null, "The receiver options should have been set.");
Assert.That(actualOptions, Is.Not.SameAs(expectedOptions), "A clone of the options should have been made.");
Assert.That(actualOptions.BeginReceivingAt.Offset, Is.EqualTo(expectedOptions.BeginReceivingAt.Offset), "The beginning position to receive should match.");
Assert.That(actualOptions.ConsumerGroup, Is.EqualTo(expectedOptions.ConsumerGroup), "The consumer groups should match.");
Assert.That(actualOptions.ExclusiveReceiverPriority, Is.EqualTo(expectedOptions.ExclusiveReceiverPriority), "The exclusive priorities should match.");
Assert.That(actualOptions.Identifier, Is.EqualTo(expectedOptions.Identifier), "The identifiers should match.");
Assert.That(actualOptions.PrefetchCount, Is.EqualTo(expectedOptions.PrefetchCount), "The prefetch counts should match.");
Assert.That(actualOptions.Retry, Is.EqualTo(expectedOptions.Retry), "The retries should match.");
Assert.That(actualOptions.MaximumReceiveWaitTimeOrDefault, Is.EqualTo(expectedOptions.MaximumReceiveWaitTimeOrDefault), "The wait times should match.");
}
private async Task <PartitionReceiver> SetupWithCheckpointAsync(string consumerGroup, string partitionId)
{
string?offset = await _checkpointer.GetCheckpointAsync(partitionId);
EventPosition position;
if (string.IsNullOrEmpty(offset))
{
position = EventPosition.FromStart();
}
else
{
position = EventPosition.FromOffset(offset, false);
}
return(_client.CreateReceiver(consumerGroup, partitionId, position));
}
public void ReceiveAsyncRespectsTheRetryPolicy(RetryOptions retryOptions)
{
var eventHub = "eventHubName";
var consumerGroup = "$DEFAULT";
var partition = "3";
var eventPosition = EventPosition.FromOffset(123);
var options = new EventHubConsumerOptions {
Identifier = "OMG!"
};
var tokenValue = "123ABC";
var retryPolicy = new BasicRetryPolicy(retryOptions);
var retriableException = new EventHubsException(true, "Test");
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <TokenCredential>();
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
mockCredential
.Setup(credential => credential.GetTokenAsync(It.IsAny <TokenRequestContext>(), It.Is <CancellationToken>(value => value == cancellationSource.Token)))
.Returns(Task.FromResult(new AccessToken(tokenValue, DateTimeOffset.MaxValue)));
mockScope
.Setup(scope => scope.OpenConsumerLinkAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <EventHubConsumerOptions>(),
It.IsAny <TimeSpan>(),
It.IsAny <CancellationToken>()))
.Throws(retriableException);
var consumer = new AmqpEventHubConsumer(eventHub, consumerGroup, partition, eventPosition, options, mockScope.Object, Mock.Of <AmqpMessageConverter>(), retryPolicy, null);
Assert.That(async() => await consumer.ReceiveAsync(100, null, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
mockScope
.Verify(scope => scope.OpenConsumerLinkAsync(
It.Is <string>(value => value == consumerGroup),
It.Is <string>(value => value == partition),
It.Is <EventPosition>(value => value == eventPosition),
It.Is <EventHubConsumerOptions>(value => value == options),
It.IsAny <TimeSpan>(),
It.IsAny <CancellationToken>()),
Times.Exactly(1 + retryOptions.MaximumRetries));
}
public void ReceiveAsyncValidatesTheMaximumMessageCount(int count)
{
var eventHub = "eventHubName";
var consumerGroup = "$DEFAULT";
var partition = "3";
var eventPosition = EventPosition.FromOffset(123);
var retryPolicy = new BasicRetryPolicy(new EventHubsRetryOptions());
var retriableException = new EventHubsException(true, "Test");
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <TokenCredential>();
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
var consumer = new AmqpConsumer(eventHub, consumerGroup, partition, eventPosition, true, null, null, mockScope.Object, Mock.Of <AmqpMessageConverter>(), retryPolicy);
Assert.That(async() => await consumer.ReceiveAsync(count, null, cancellationSource.Token), Throws.InstanceOf <ArgumentException>());
}