public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the loadbalancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by loadbalancer. Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions)); }
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer1 = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); var loadbalancer2 = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the loadbalancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); // Stopping the loadbalancer should relinquish all partition ownership. await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None); completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // No partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0)); // Start loadbalancer2 so that the loadbalancer claims a random partition until none are left. // All partitions should be immediately claimable even though they were just claimed by the loadbalancer1. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // All partitions are owned by loadbalancer2. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); }
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace1", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); // ETag must have been set by the partition manager. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace2", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup"); IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup"); Assert.That(storedOwnership1, Is.Not.Null); Assert.That(storedOwnership1.Count, Is.EqualTo(1)); Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership)); Assert.That(storedOwnership2, Is.Not.Null); Assert.That(storedOwnership2.Count, Is.EqualTo(1)); Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership)); }
public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership() { var partitionManager = new MockCheckPointStorage(); IEnumerable <PartitionOwnership> ownership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(ownership, Is.Not.Null.And.Empty); }
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownershipCount = 5; for (int i = 0; i < ownershipCount; i++) { ownershipList.Add( new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", $"partitionId { i }" )); } await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount)); Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True); }
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag) { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership)); }
public async Task FirstOwnershipClaimSucceeds() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(ownership)); }
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions() { const int MinimumpartitionCount = 4; const int MaximumpartitionCount = 5; const int NumberOfPartitions = 12; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create more partitions owned by this loadbalancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create more partitions owned by a different loadbalancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create more partitions owned by a different loadbalancer above the MaximumPartitionCount. var loadbalancer3Id = Guid.NewGuid().ToString(); var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the partitionManager with the owned partitions. await partitionManager.ClaimOwnershipAsync(completeOwnership); // Get owned partitions. var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False); // Verify loadbalancer 3 has stealable partitions. Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount)); // Start the loadbalancer to steal owership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a loadbalancer owns > MaximumPartitionCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True); // Verify loadbalancer 3 now does not own > MaximumPartitionCount. Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount)); }
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount() { const int MinimumpartitionCount = 4; const int NumberOfPartitions = 13; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create partitions owned by this loadbalancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create partitions owned by a different loadbalancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create partitions owned by a different loadbalancer. var loadbalancer3Id = Guid.NewGuid().ToString(); var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the partitionManager with all partitions. await partitionManager.ClaimOwnershipAsync(completeOwnership); var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); // Get owned partitions. var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumpartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False); // Start the loadbalancer to claim owership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumpartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True); }
public async Task PartitionProcessorCanCreateACheckpoint() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { // Send some events. EventData lastEvent; await using (var producer = new EventHubProducerClient(connection)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < dummyEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { Assert.Fail("Convert to iterator."); //receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new MockCheckPointStorage(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, partitionManager, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { eventArgs.UpdateCheckpointAsync(); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); } } }