public async Task RunLoadBalancingAsyncDoesNotStealOwnershipAsRecovery() { const int NumberOfPartitions = 8; const int MinimumPartitionCount = 4; const int OrphanedPartitionCount = 2; var otherLoadBalancerIdentifier = Guid.NewGuid().ToString(); var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10)); // Ownership should start empty. var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership."); // Claim the minimum set of partitions for the "other" load balancer. completeOwnership = await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership(partitionIds.Take(MinimumPartitionCount), otherLoadBalancerIdentifier)); Assert.That(completeOwnership.Count(), Is.EqualTo(MinimumPartitionCount), "Storage should be tracking half the partitions as owned by another processor."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of any partitions."); // Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some // number of partitions, but the processor state does not reflect that ownership. // // Assign the processor ownership over half of the partitions in storage, but do not formally claim them. await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership( partitionIds.Skip(MinimumPartitionCount).Take(OrphanedPartitionCount), loadBalancer.OwnerIdentifier)); completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + MinimumPartitionCount), "Storage should be tracking half the partitions as owned by another processor as well as some orphans."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned or otherwise owned partitions."); // Run one load balancing cycle. At the end of the cycle, it should have claimed a random partition // and recovered ownership of the orphans. await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + MinimumPartitionCount + 1), "Storage should be tracking the orphaned partitions, other processor partitions, and one additional as owned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "The load balancer should have ownership of all orphaned partitions and one additional."); // Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled. var cycleCount = 0; while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4))) { await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); ++cycleCount; } // All partitions should be owned by load balancer. completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions)); }
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions() { const int MinimumpartitionCount = 4; const int MaximumpartitionCount = 5; const int NumberOfPartitions = 12; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10)); // Create more partitions owned by this load balancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create more partitions owned by a different load balancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create more partitions owned by a different load balancer above the MaximumPartitionCount. var loadbalancer3Id = Guid.NewGuid().ToString(); var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the storageManager with the owned partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); // Get owned partitions. var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False); // Verify load balancer 3 has stealable partitions. Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount)); // Start the load balancer to steal ownership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a load balancer owns > MaximumPartitionCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True); // Verify load balancer 3 now does not own > MaximumPartitionCount. Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount)); }
public async Task ProcessorClientBeginsWithTheNextEventAfterCheckpointing() { // Setup the environment. await using EventHubScope scope = await EventHubScope.CreateAsync(1); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(4)); // Send a set of events. var segmentEventCount = 25; var beforeCheckpointEvents = CreateEvents(segmentEventCount).ToList(); var afterCheckpointEvents = CreateEvents(segmentEventCount, segmentEventCount).ToList(); var sourceEvents = Enumerable.Concat(beforeCheckpointEvents, afterCheckpointEvents).ToList(); var checkpointEvent = beforeCheckpointEvents.Last(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the first half of the events and checkpoint. Func <ProcessEventArgs, Task> processedEventCallback = async args => { if (args.Data.IsEquivalentTo(checkpointEvent)) { await args.UpdateCheckpointAsync(cancellationSource.Token); } }; var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var beforeCheckpointProcessHandler = CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token, processedEventCallback); var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250) }; var storageManager = new InMemoryStorageManager(_ => {}); var processor = CreateProcessor(scope.ConsumerGroups.First(), connectionString, storageManager, options); processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += beforeCheckpointProcessHandler; await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled."); await processor.StopProcessingAsync(cancellationSource.Token); // Validate a checkpoint was created and that events were processed. var checkpoints = (await storageManager.ListCheckpointsAsync(processor.FullyQualifiedNamespace, processor.EventHubName, processor.ConsumerGroup, cancellationSource.Token))?.ToList(); Assert.That(checkpoints, Is.Not.Null, "A checkpoint should have been created."); Assert.That(checkpoints.Count, Is.EqualTo(1), "A single checkpoint should exist."); Assert.That(processedEvents.Count, Is.AtLeast(beforeCheckpointEvents.Count), "All events before the checkpoint should have been processed."); // Reset state and start the processor again; it should resume from the event following the checkpoint. processedEvents.Clear(); completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); processor.ProcessEventAsync -= beforeCheckpointProcessHandler; processor.ProcessEventAsync += CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token); await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled."); await processor.StopProcessingAsync(cancellationSource.Token); cancellationSource.Cancel(); foreach (var sourceEvent in afterCheckpointEvents) { var sourceId = sourceEvent.Properties[CustomIdProperty].ToString(); Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed."); Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event."); } }
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount() { const int MinimumPartitionCount = 4; const int NumberOfPartitions = 13; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10)); // Create partitions owned by this load balancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumPartitionCount); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create partitions owned by a different load balancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create partitions owned by a different load balancer. var loadbalancer3Id = Guid.NewGuid().ToString(); var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the storageManager with all partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); // Get owned partitions. var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumPartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False); // Start the load balancer to claim ownership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumPartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True); }
public async Task ProcessorClientCreatesOwnership() { // Setup the environment. var partitionCount = 2; var partitionIds = new HashSet <string>(); await using EventHubScope scope = await EventHubScope.CreateAsync(partitionCount); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(2)); // Discover the partitions. await using (var producer = new EventHubProducerClient(connectionString)) { foreach (var partitionId in (await producer.GetPartitionIdsAsync())) { partitionIds.Add(partitionId); } } // Send a set of events. var sourceEvents = CreateEvents(200).ToList(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the events. var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var storageManager = new InMemoryStorageManager(_ => {}); var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250) }; var processor = CreateProcessorWithIdentity(scope.ConsumerGroups.First(), scope.EventHubName, storageManager, options); processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token); await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled. { processedEvents.Count } events were processed."); await processor.StopProcessingAsync(cancellationSource.Token); cancellationSource.Cancel(); // Validate that events that were processed. var ownership = (await storageManager.ListOwnershipAsync(TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, scope.ConsumerGroups.First(), cancellationSource.Token))?.ToList(); Assert.That(ownership, Is.Not.Null, "The ownership list should have been returned."); Assert.That(ownership.Count, Is.AtLeast(1), "At least one partition should have been owned."); foreach (var partitionOwnership in ownership) { Assert.That(partitionIds.Contains(partitionOwnership.PartitionId), Is.True, $"The partition `{ partitionOwnership.PartitionId }` is not valid for the Event Hub."); Assert.That(partitionOwnership.OwnerIdentifier, Is.Empty, "Ownership should have bee relinquished when the processor was stopped."); } }