/// <summary> /// Compares ownership information between two instances to determine if the /// instances represent the same ownership. /// </summary> /// /// <param name="instance">The instance that this method was invoked on.</param> /// <param name="other">The other partition ownership to consider.</param> /// /// <returns><c>true</c>, if the two ownership are structurally equivalent; otherwise, <c>false</c>.</returns> /// public static bool IsEquivalentTo(this PartitionOwnership instance, PartitionOwnership other) { // If the ownership are the same instance, they're equal. This should only happen // if both are null or they are the exact same instance. if (Object.ReferenceEquals(instance, other)) { return(true); } // If one or the other is null, then they cannot be equal, since we know that // they are not both null. if ((instance == null) || (other == null)) { return(false); } // If the contents of each attribute are equal, the instances are // equal. return ( instance.EventHubName == other.EventHubName && instance.ConsumerGroup == other.ConsumerGroup && instance.OwnerIdentifier == other.OwnerIdentifier && instance.PartitionId == other.PartitionId && instance.Offset == other.Offset && instance.SequenceNumber == other.SequenceNumber && instance.LastModifiedTime == other.LastModifiedTime && instance.ETag == other.ETag ); }
public async Task CheckReturnedEtagContainsSingleQuotes() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { // A regular expression used to capture strings enclosed in double quotes. Regex s_doubleQuotesExpression = new Regex("\"(.*)\"", RegexOptions.Compiled); var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); IEnumerable <PartitionOwnership> claimedOwnership = await checkpointStore.ClaimOwnershipAsync(ownershipList, default); IEnumerable <PartitionOwnership> storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default); var claimedOwnershipMatch = s_doubleQuotesExpression.Match(claimedOwnership.First().ETag); var storedOwnershipListMatch = s_doubleQuotesExpression.Match(storedOwnershipList.First().ETag); Assert.That(claimedOwnershipMatch.Success, Is.False); Assert.That(storedOwnershipListMatch.Success, Is.False); } }
public async Task OwnershipClaimFailsWhenETagExistsAndOwnershipDoesNotExist() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <PartitionOwnership>(); var eTaggyOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: "ETag" ); ownershipList.Add(eTaggyOwnership); await checkpointStore.ClaimOwnershipAsync(ownershipList, default); IEnumerable <PartitionOwnership> storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default); Assert.That(storedOwnershipList, Is.Not.Null.And.Empty); } }
public async Task OwnershipClaimSetsLastModifiedTimeAndETag() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await checkpointStore.ClaimOwnershipAsync(ownershipList, default); Assert.That(ownership.LastModifiedTime, Is.Not.Null); Assert.That(ownership.LastModifiedTime.Value, Is.GreaterThan(DateTimeOffset.UtcNow.Subtract(TimeSpan.FromSeconds(5)))); Assert.That(ownership.ETag, Is.Not.Null); } }
public async Task CheckpointUpdateUpdatesOwnershipInformation() { var partitionManager = new InMemoryPartitionManager(); var originalOwnership = new PartitionOwnership ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1))); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { originalOwnership }); // ETag must have been set by the partition manager. var originalLastModifiedTime = originalOwnership.LastModifiedTime; var originalETag = originalOwnership.ETag; await partitionManager.UpdateCheckpointAsync(new Checkpoint ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", 10, 20)); // Make sure the ownership has changed, even though the instance should be the same. var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership)); Assert.That(originalOwnership.Offset, Is.EqualTo(10)); Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(20)); Assert.That(originalOwnership.LastModifiedTime, Is.GreaterThan(originalLastModifiedTime)); Assert.That(originalOwnership.ETag, Is.Not.EqualTo(originalETag)); }
public async Task CheckpointUpdateFailsWhenOwnerChanges() { var partitionManager = new InMemoryPartitionManager(); var originalOwnership = new PartitionOwnership ("eventHubName", "consumerGroup", "ownerIdentifier1", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { originalOwnership }); // ETag must have been set by the partition manager. var originalLastModifiedTime = originalOwnership.LastModifiedTime; var originalETag = originalOwnership.ETag; await partitionManager.UpdateCheckpointAsync(new Checkpoint ("eventHubName", "consumerGroup", "ownerIdentifier2", "partitionId", 10, 20)); // Make sure the ownership hasn't changed. var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership)); Assert.That(originalOwnership.OwnerIdentifier, Is.EqualTo("ownerIdentifier1")); Assert.That(originalOwnership.Offset, Is.EqualTo(1)); Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(2)); Assert.That(originalOwnership.LastModifiedTime, Is.EqualTo(originalLastModifiedTime)); Assert.That(originalOwnership.ETag, Is.EqualTo(originalETag)); }
public async Task FirstOwnershipClaimSucceeds() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await checkpointStore.ClaimOwnershipAsync(ownershipList, default); IEnumerable <PartitionOwnership> storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default); Assert.That(storedOwnershipList, Is.Not.Null); Assert.That(storedOwnershipList.Count, Is.EqualTo(1)); Assert.That(storedOwnershipList.Single().IsEquivalentTo(ownership), Is.True); } }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var partitionManager = new BlobPartitionManager(containerClient); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { new MockPartitionOwnership("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", offset: 1), new MockPartitionOwnership("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId2", offset: 1) }); await partitionManager.UpdateCheckpointAsync(new MockCheckpoint ("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", 10, 20)); IEnumerable <PartitionOwnership> storedOwnershipList = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnershipList, Is.Not.Null); Assert.That(storedOwnershipList.Count, Is.EqualTo(2)); PartitionOwnership storedOwnership1 = storedOwnershipList.First(ownership => ownership.PartitionId == "partitionId1"); PartitionOwnership storedOwnership2 = storedOwnershipList.First(ownership => ownership.PartitionId == "partitionId2"); Assert.That(storedOwnership1, Is.Not.Null); Assert.That(storedOwnership1.Offset, Is.EqualTo(10)); Assert.That(storedOwnership2, Is.Not.Null); Assert.That(storedOwnership2.Offset, Is.EqualTo(1)); } }
public async Task OwnershipClaimSucceedsWhenETagIsValid() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await checkpointStore.ClaimOwnershipAsync(ownershipList); // ETag must have been set by the checkpoint store. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await checkpointStore.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnershipList, Is.Not.Null); Assert.That(storedOwnershipList.Count, Is.EqualTo(1)); Assert.That(storedOwnershipList.Single().IsEquivalentTo(secondOwnership), Is.True); } }
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace1", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); // ETag must have been set by the partition manager. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace2", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup"); IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup"); Assert.That(storedOwnership1, Is.Not.Null); Assert.That(storedOwnership1.Count, Is.EqualTo(1)); Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership)); Assert.That(storedOwnership2, Is.Not.Null); Assert.That(storedOwnership2.Count, Is.EqualTo(1)); Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership)); }
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces() { await using (var storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace1", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await checkpointStore.ClaimOwnershipAsync(ownershipList, default); // ETag must have been set by the checkpoint store. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace2", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); Assert.That(async() => await checkpointStore.ClaimOwnershipAsync(ownershipList, default), Throws.InstanceOf <RequestFailedException>()); var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup", default); Assert.That(storedOwnershipList, Is.Not.Null); Assert.That(storedOwnershipList.Count, Is.EqualTo(1)); Assert.That(storedOwnershipList.Single().IsEquivalentTo(firstOwnership), Is.True); } }
public async Task OwnershipClaimSucceedsWhenETagIsValid() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1 ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); // ETag must have been set by the partition manager. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 2, eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(secondOwnership)); }
public async Task CheckpointUpdateUpdatesOwnershipInformation() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var partitionManager = new BlobPartitionManager(containerClient); var originalOwnership = new MockPartitionOwnership ("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1, sequenceNumber: 2); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { originalOwnership }); // ETag must have been set by the partition manager. DateTimeOffset?originalLastModifiedTime = originalOwnership.LastModifiedTime; var originalETag = originalOwnership.ETag; // Give it a 1 second delay before updating the ownership information. We need to do this to ensure // we won't end up with the same 'LastModifiedTime'. In the InMemoryPartitionManager tests we have control // over it, but this time this property is entirely handled by the storage service. await Task.Delay(1000); await partitionManager.UpdateCheckpointAsync(new MockCheckpoint ("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", 10, 20)); // Make sure the ownership has changed. IEnumerable <PartitionOwnership> storedOwnershipList = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnershipList, Is.Not.Null); Assert.That(storedOwnershipList.Count, Is.EqualTo(1)); PartitionOwnership storedOwnership = storedOwnershipList.Single(); Assert.That(storedOwnership.Offset, Is.EqualTo(10)); Assert.That(storedOwnership.SequenceNumber, Is.EqualTo(20)); Assert.That(storedOwnership.LastModifiedTime, Is.GreaterThan(originalLastModifiedTime)); Assert.That(storedOwnership.ETag, Is.Not.EqualTo(originalETag)); } }
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag) { var partitionManager = new InMemoryPartitionManager(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1 ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 2, eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership)); }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { var partitionManager = new InMemoryPartitionManager(); var ownership1 = new PartitionOwnership ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", offset: 1); var ownership2 = new PartitionOwnership ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId2", offset: 1); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { ownership1, ownership2 }); await partitionManager.UpdateCheckpointAsync(new Checkpoint ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", 10, 20)); Assert.That(ownership1.Offset, Is.EqualTo(10)); Assert.That(ownership2.Offset, Is.EqualTo(1)); }
public async Task FirstOwnershipClaimSucceeds() { var partitionManager = new InMemoryPartitionManager(); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await partitionManager.ClaimOwnershipAsync(ownershipList); var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(ownership)); }
public async Task FirstOwnershipClaimSucceeds() { var storageManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await storageManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await storageManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(ownership)); }
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { // Send some events. EventData lastEvent; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); var partitionId = (await client.GetPartitionIdsAsync()).First(); await using (EventHubProducer producer = client.CreateProducer()) await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; for (int i = 0; i < dummyEventsCount; i++) { await producer.SendAsync(dummyEvent); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new InMemoryPartitionManager(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Any()) { partitionContext.UpdateCheckpointAsync(eventsList.Last()); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); PartitionOwnership ownership = ownershipEnumerable.Single(); Assert.That(ownership.Offset.HasValue, Is.True); Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset)); Assert.That(ownership.SequenceNumber.HasValue, Is.True); Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber)); } } }