public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the load balancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by load balancer. Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions)); }
private async Task <IPartitionManager> BuildPartitionManagerAsync(ILeaseManager leaseManager) { this.leaseDocumentClient = this.leaseDocumentClient ?? this.leaseCollectionLocation.CreateDocumentClient(); DocumentCollection leaseCollection = await this.leaseDocumentClient.GetDocumentCollectionAsync(this.leaseCollectionLocation).ConfigureAwait(false); string leaseStoreCollectionLink = leaseCollection.SelfLink; string collectionSelfLink = this.feedCollectionLocation.GetCollectionSelfLink(); var factory = new CheckpointerObserverFactory(this.observerFactory, this.changeFeedProcessorOptions.CheckpointFrequency); var synchronizer = new PartitionSynchronizer(this.feedDocumentClient, collectionSelfLink, leaseManager, this.changeFeedProcessorOptions.DegreeOfParallelism, this.changeFeedProcessorOptions.QueryPartitionsMaxBatchSize); var leaseStore = new LeaseStore(this.leaseDocumentClient, this.leaseCollectionLocation, this.GetLeasePrefix(), leaseStoreCollectionLink); var bootstrapper = new Bootstrapper(synchronizer, leaseStore, this.lockTime, this.sleepTime); var partitionObserverFactory = new PartitionSupervisorFactory( factory, leaseManager, this.partitionProcessorFactory ?? new PartitionProcessorFactory(this.feedDocumentClient, this.changeFeedProcessorOptions, leaseManager, collectionSelfLink), this.changeFeedProcessorOptions); var partitionController = new PartitionController(leaseManager, partitionObserverFactory, synchronizer); if (this.loadBalancingStrategy == null) { this.loadBalancingStrategy = new EqualPartitionsBalancingStrategy(this.hostName, this.changeFeedProcessorOptions.MinPartitionCount, this.changeFeedProcessorOptions.MaxPartitionCount, this.changeFeedProcessorOptions.LeaseExpirationInterval); } var partitionLoadBalancer = new PartitionLoadBalancer(partitionController, leaseManager, this.loadBalancingStrategy, this.changeFeedProcessorOptions.LeaseAcquireInterval); return(new PartitionManager(bootstrapper, partitionController, partitionLoadBalancer)); }
public async Task AddLease_ThrowsException_LeaseAddingContinues() { FailingPartitionController controller = new FailingPartitionController(); // long acquire interval to ensure that only 1 load balancing iteration is performed in a test run var leaseAcquireInterval = TimeSpan.FromHours(1); var loadBalancer = new PartitionLoadBalancer(controller, this.leaseManager, this.strategy, leaseAcquireInterval); Mock.Get(this.strategy) .Setup(s => s.SelectLeasesToTake(It.IsAny <IEnumerable <ILease> >())) .Returns(new[] { Mock.Of <ILease>(), Mock.Of <ILease>() }); Mock.Get(this.leaseManager) .Setup(m => m.ListAllLeasesAsync()) .ReturnsAsync(new[] { Mock.Of <ILease>(), Mock.Of <ILease>() }); loadBalancer.Start(); await loadBalancer.StopAsync(); Mock.Get(this.strategy) .Verify(s => s.SelectLeasesToTake(It.IsAny <IEnumerable <ILease> >()), Times.Once); Mock.Get(this.leaseManager) .Verify(m => m.ListAllLeasesAsync(), Times.Once); Assert.Equal(2, controller.HitCount); }
internal MinimalProcessorMock(int eventBatchMaximumCount, string consumerGroup, string fullyQualifiedNamespace, string eventHubName, TokenCredential credential, EventProcessorOptions options, PartitionLoadBalancer loadBalancer) : base(eventBatchMaximumCount, consumerGroup, fullyQualifiedNamespace, eventHubName, credential, options, loadBalancer) { }
public async Task IsBalancedIsCorrectWithMultipleProcessorsAndAnUnevenDistribution() { const int MinimumPartitionCount = 4; const int NumberOfPartitions = 13; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); var completeOwnership = Enumerable.Empty <EventProcessorPartitionOwnership>(); // Create partitions owned by a different load balancer. var secondLoadBalancerId = Guid.NewGuid().ToString(); var secondLoadBalancerPartitions = Enumerable.Range(1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(secondLoadBalancerPartitions.Select(i => i.ToString()), secondLoadBalancerId)); // Create partitions owned by a different load balancer. var thirdLoadBalancerId = Guid.NewGuid().ToString(); var thirdLoadBalancerPartitions = Enumerable.Range(secondLoadBalancerPartitions.Max() + 1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(thirdLoadBalancerPartitions.Select(i => i.ToString()), thirdLoadBalancerId)); // Seed the storageManager with all partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); // Ensure that there is exactly one more than the minimum number of partitions available to be owned. var unownedPartitions = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); Assert.That(unownedPartitions.Count(), Is.EqualTo(MinimumPartitionCount + 1), $"There should be { MinimumPartitionCount + 1 } partitions left unowned."); // Run load balancing cycles until the load balancer believes that the state is balanced or the minimum count is quadrupled. var cycleCount = 0; while ((!loadBalancer.IsBalanced) && (cycleCount < (MinimumPartitionCount * 4))) { await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); ++cycleCount; } completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); unownedPartitions = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); Assert.That(unownedPartitions.Count(), Is.EqualTo(0), "There no partitions left unowned."); Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions), "All partitions should be owned."); Assert.That(loadBalancer.IsBalanced, Is.True, "The load balancer should believe the state is balanced when it owns the correct number of partitions."); Assert.That(cycleCount, Is.EqualTo(MinimumPartitionCount + 2), "The load balancer should have reached a balanced state once all partitions were owned and the next cycle claimed none."); }
public async Task RunLoadBalancingAsyncReclaimsOwnershipWhenRecovering() { const int NumberOfPartitions = 8; const int OrphanedPartitionCount = 4; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10)); // Ownership should start empty. var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership."); // Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some // number of partitions, but the processor state does not reflect that ownership. // // Assign the processor ownership over half of the partitions in storage, but do not formally claim them. var orphanedPartitions = partitionIds.Take(OrphanedPartitionCount); completeOwnership = await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership(orphanedPartitions, loadBalancer.OwnerIdentifier)); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking half the partitions as orphaned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned partitions."); // Run one load balancing cycle. At the end of the cycle, it should have claimed a random partition // and recovered ownership of the orphans. await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "Storage should be tracking the orphaned partitions and one additional as owned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "The load balancer should have ownership of all orphaned partitions and one additional."); // Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled. var cycleCount = 0; while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4))) { await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); ++cycleCount; } // All partitions should be owned by load balancer. completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions)); }
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer1 = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); var loadbalancer2 = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the load balancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); // Stopping the load balancer should relinquish all partition ownership. await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None); completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // No partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0)); // Start loadbalancer2 so that the load balancer claims a random partition until none are left. // All partitions should be immediately claimable even though they were just claimed by the loadbalancer1. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // All partitions are owned by loadbalancer2. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); }
private IPartitionManager BuildPartitionManager(ILeaseStoreManager leaseStoreManager) { string feedCollectionSelfLink = this.feedCollectionLocation.GetCollectionSelfLink(); var factory = new CheckpointerObserverFactory(this.observerFactory, this.changeFeedProcessorOptions.CheckpointFrequency); var synchronizer = new PartitionSynchronizer( this.feedDocumentClient, feedCollectionSelfLink, leaseStoreManager, leaseStoreManager, this.changeFeedProcessorOptions.DegreeOfParallelism, this.changeFeedProcessorOptions.QueryPartitionsMaxBatchSize); var bootstrapper = new Bootstrapper(synchronizer, leaseStoreManager, this.lockTime, this.sleepTime); var partitionSuperviserFactory = new PartitionSupervisorFactory( factory, leaseStoreManager, this.partitionProcessorFactory ?? new PartitionProcessorFactory(this.feedDocumentClient, this.changeFeedProcessorOptions, leaseStoreManager, feedCollectionSelfLink), this.changeFeedProcessorOptions); if (this.loadBalancingStrategy == null) { this.loadBalancingStrategy = new EqualPartitionsBalancingStrategy( this.HostName, this.changeFeedProcessorOptions.MinPartitionCount, this.changeFeedProcessorOptions.MaxPartitionCount, this.changeFeedProcessorOptions.LeaseExpirationInterval); } IPartitionController partitionController = new PartitionController(leaseStoreManager, leaseStoreManager, partitionSuperviserFactory, synchronizer); if (this.healthMonitor == null) { this.healthMonitor = new TraceHealthMonitor(); } partitionController = new HealthMonitoringPartitionControllerDecorator(partitionController, this.healthMonitor); var partitionLoadBalancer = new PartitionLoadBalancer( partitionController, leaseStoreManager, this.loadBalancingStrategy, this.changeFeedProcessorOptions.LeaseAcquireInterval); return(new PartitionManager(bootstrapper, partitionController, partitionLoadBalancer)); }
public async Task VerifiesEventProcessorLogs() { const int NumberOfPartitions = 4; const int MinimumpartitionCount = NumberOfPartitions / 2; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create more partitions owned by a different load balancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var completeOwnership = CreatePartitionOwnership(partitionIds.Skip(1), loadbalancer2Id); // Seed the storageManager with the owned partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); var mockLog = new Mock <PartitionLoadBalancerEventSource>(); loadbalancer.Logger = mockLog.Object; for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } await loadbalancer.RelinquishOwnershipAsync(CancellationToken.None); mockLog.Verify(m => m.RenewOwnershipStart(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.RenewOwnershipComplete(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.ClaimOwnershipStart(It.Is <string>(p => partitionIds.Contains(p)))); mockLog.Verify(m => m.MinimumPartitionsPerEventProcessor(MinimumpartitionCount)); mockLog.Verify(m => m.CurrentOwnershipCount(MinimumpartitionCount, loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.StealPartition(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.ShouldStealPartition(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.UnclaimedPartitions(It.Is <HashSet <string> >(set => set.Count == 0 || set.All(item => partitionIds.Contains(item))))); }
public async Task IsBalancedIsCorrectWithOneProcessor() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0), "No partitions should be owned."); // Start the load balancer so that it claims a random partition until none are left. for (var index = 0; index < NumberOfPartitions; ++index) { await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); Assert.That(loadBalancer.IsBalanced, Is.False, "The load balancer should not believe the state is balanced while partitions remain unclaimed."); } // The load balancer should not consider itself balanced until a cycle is run with no partitions claimed. Run one additional // cycle to satisfy that condition. Assert.That(loadBalancer.IsBalanced, Is.False, "The load balancer should not believe the state is balanced until no partition is claimed during a cycle."); await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // All partitions are owned by load balancer. completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions), "All partitions should be owned."); Assert.That(loadBalancer.IsBalanced, Is.True, "The load balancer should believe the state is balanced when it owns all partitions."); }
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions() { const int MinimumpartitionCount = 4; const int MaximumpartitionCount = 5; const int NumberOfPartitions = 12; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create more partitions owned by this load balancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create more partitions owned by a different load balancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create more partitions owned by a different load balancer above the MaximumPartitionCount. var loadbalancer3Id = Guid.NewGuid().ToString(); var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the storageManager with the owned partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); // Get owned partitions. var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False); // Verify load balancer 3 has stealable partitions. Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount)); // Start the load balancer to steal ownership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a load balancer owns > MaximumPartitionCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True); // Verify load balancer 3 now does not own > MaximumPartitionCount. Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount)); }
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount() { const int MinimumPartitionCount = 4; const int NumberOfPartitions = 13; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create partitions owned by this load balancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumPartitionCount); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create partitions owned by a different load balancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create partitions owned by a different load balancer. var loadbalancer3Id = Guid.NewGuid().ToString(); var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumPartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the storageManager with all partitions. await storageManager.ClaimOwnershipAsync(completeOwnership); var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); // Get owned partitions. var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumPartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False); // Start the load balancer to claim ownership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumPartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True); }
public async Task RunLoadBalancingAsyncReclaimsOwnershipWhenLeaseRenewalFails() { const int NumberOfPartitions = 8; const int OrphanedPartitionCount = 4; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var mockStorageManager = new Mock <InMemoryStorageManager>() { CallBase = true }; var loadBalancer = new PartitionLoadBalancer(mockStorageManager.Object, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10)); // Ownership should start empty. var completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership."); // Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some // number of partitions, but the processor state does not reflect that ownership. // // Assign the processor ownership over half of the partitions in storage, but do not formally claim them. var orphanedPartitions = partitionIds.Take(OrphanedPartitionCount); completeOwnership = await mockStorageManager.Object.ClaimOwnershipAsync(CreatePartitionOwnership(orphanedPartitions, loadBalancer.OwnerIdentifier)); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking half the partitions as orphaned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned partitions."); // Configure the Storage Manager to fail all claim attempts moving forward. mockStorageManager .Setup(sm => sm.ClaimOwnershipAsync(It.IsAny <IEnumerable <EventProcessorPartitionOwnership> >(), It.IsAny <CancellationToken>())) .ReturnsAsync(Enumerable.Empty <EventProcessorPartitionOwnership>()); // Run one load balancing cycle. At the end of the cycle, it should have recovered ownership of the orphans // but made no new claims. await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking the orphaned partitions as owned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount), "The load balancer should have ownership of all orphaned partitions and none additional."); // Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled. var cycleCount = 0; while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4))) { await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); ++cycleCount; } // Only the orphaned partitions should be owned by load balancer, other claims have failed. completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking the orphaned partitions as owned."); Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount), "The load balancer should have ownership of all orphaned partitions and none additional."); }
/// <summary> /// Initializes a new instance of the <see cref="MockEventProcessorClient" /> class. /// </summary> /// /// <param name="storageManager">The client responsible for interaction with durable storage, responsible for persisting checkpoints and load-balancing state.</param> /// <param name="consumerGroup">The name of the consumer group this processor is associated with. Events are read in the context of this group.</param> /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace to connect to. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param> /// <param name="eventHubName">The name of the specific Event Hub to associate the processor with.</param> /// <param name="connectionFactory">A factory used to provide new <see cref="EventHubConnection" /> instances.</param> /// <param name="clientOptions">The set of options to use for this processor.</param> /// <param name="fakePartitionProcessing"><c>true</c> if <see cref="RunPartitionProcessingAsync" /> should be overridden; otherwise, <c>false</c>.</param> /// <param name="numberOfPartitions">The amount of partitions the associated Event Hub has.</param> /// <param name="loadBalancer">The <see cref="PartitionLoadBalancer" /> used to manage partition load balance operations.</param> /// internal MockEventProcessorClient(StorageManager storageManager, string consumerGroup = "consumerGroup", string fullyQualifiedNamespace = "somehost.com", string eventHubName = "somehub", Func <EventHubConnection> connectionFactory = default, EventProcessorClientOptions clientOptions = default, bool fakePartitionProcessing = true, int numberOfPartitions = 3, PartitionLoadBalancer loadBalancer = default) : base(storageManager, consumerGroup, fullyQualifiedNamespace, eventHubName, connectionFactory, clientOptions, loadBalancer) { StorageManager = storageManager; var partitionIds = Enumerable .Range(1, numberOfPartitions) .Select(p => p.ToString()) .ToArray(); foreach (var partitionId in partitionIds) { EventPipeline[partitionId] = new ConcurrentQueue <EventData>(); } FakeRunPartitionProcessingAsync = fakePartitionProcessing; ProcessErrorAsync += eventArgs => { Exception[] newException = new Exception[] { eventArgs.Exception }; ExceptionCalls.AddOrUpdate( eventArgs.PartitionId, newException, (partitionId, value) => value.Concat(newException).ToArray()); return(Task.CompletedTask); }; ProcessEventAsync += eventArgs => { EventData[] newEvent = new EventData[] { eventArgs.Data }; ProcessEventCalls.AddOrUpdate( eventArgs.Partition.PartitionId, newEvent, (partitionId, value) => value.Concat(newEvent).ToArray()); return(Task.CompletedTask); }; PartitionInitializingAsync += eventArgs => { InitializeCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1); return(Task.CompletedTask); }; PartitionClosingAsync += eventArgs => { CloseCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1); StopReasons[eventArgs.PartitionId] = eventArgs.Reason; return(Task.CompletedTask); }; }