public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the load balancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by load balancer.
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions));
}
private async Task <IPartitionManager> BuildPartitionManagerAsync(ILeaseManager leaseManager)
{
this.leaseDocumentClient = this.leaseDocumentClient ?? this.leaseCollectionLocation.CreateDocumentClient();
DocumentCollection leaseCollection = await this.leaseDocumentClient.GetDocumentCollectionAsync(this.leaseCollectionLocation).ConfigureAwait(false);
string leaseStoreCollectionLink = leaseCollection.SelfLink;
string collectionSelfLink = this.feedCollectionLocation.GetCollectionSelfLink();
var factory = new CheckpointerObserverFactory(this.observerFactory, this.changeFeedProcessorOptions.CheckpointFrequency);
var synchronizer = new PartitionSynchronizer(this.feedDocumentClient, collectionSelfLink, leaseManager, this.changeFeedProcessorOptions.DegreeOfParallelism, this.changeFeedProcessorOptions.QueryPartitionsMaxBatchSize);
var leaseStore = new LeaseStore(this.leaseDocumentClient, this.leaseCollectionLocation, this.GetLeasePrefix(), leaseStoreCollectionLink);
var bootstrapper = new Bootstrapper(synchronizer, leaseStore, this.lockTime, this.sleepTime);
var partitionObserverFactory = new PartitionSupervisorFactory(
factory,
leaseManager,
this.partitionProcessorFactory ?? new PartitionProcessorFactory(this.feedDocumentClient, this.changeFeedProcessorOptions, leaseManager, collectionSelfLink),
this.changeFeedProcessorOptions);
var partitionController = new PartitionController(leaseManager, partitionObserverFactory, synchronizer);
if (this.loadBalancingStrategy == null)
{
this.loadBalancingStrategy = new EqualPartitionsBalancingStrategy(this.hostName, this.changeFeedProcessorOptions.MinPartitionCount, this.changeFeedProcessorOptions.MaxPartitionCount, this.changeFeedProcessorOptions.LeaseExpirationInterval);
}
var partitionLoadBalancer = new PartitionLoadBalancer(partitionController, leaseManager, this.loadBalancingStrategy, this.changeFeedProcessorOptions.LeaseAcquireInterval);
return(new PartitionManager(bootstrapper, partitionController, partitionLoadBalancer));
}
public async Task AddLease_ThrowsException_LeaseAddingContinues()
{
FailingPartitionController controller = new FailingPartitionController();
// long acquire interval to ensure that only 1 load balancing iteration is performed in a test run
var leaseAcquireInterval = TimeSpan.FromHours(1);
var loadBalancer = new PartitionLoadBalancer(controller, this.leaseManager, this.strategy, leaseAcquireInterval);
Mock.Get(this.strategy)
.Setup(s => s.SelectLeasesToTake(It.IsAny <IEnumerable <ILease> >()))
.Returns(new[] { Mock.Of <ILease>(), Mock.Of <ILease>() });
Mock.Get(this.leaseManager)
.Setup(m => m.ListAllLeasesAsync())
.ReturnsAsync(new[] { Mock.Of <ILease>(), Mock.Of <ILease>() });
loadBalancer.Start();
await loadBalancer.StopAsync();
Mock.Get(this.strategy)
.Verify(s => s.SelectLeasesToTake(It.IsAny <IEnumerable <ILease> >()), Times.Once);
Mock.Get(this.leaseManager)
.Verify(m => m.ListAllLeasesAsync(), Times.Once);
Assert.Equal(2, controller.HitCount);
}
internal MinimalProcessorMock(int eventBatchMaximumCount,
string consumerGroup,
string fullyQualifiedNamespace,
string eventHubName,
TokenCredential credential,
EventProcessorOptions options,
PartitionLoadBalancer loadBalancer) : base(eventBatchMaximumCount, consumerGroup, fullyQualifiedNamespace, eventHubName, credential, options, loadBalancer)
{
}
public async Task IsBalancedIsCorrectWithMultipleProcessorsAndAnUnevenDistribution()
{
const int MinimumPartitionCount = 4;
const int NumberOfPartitions = 13;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
var completeOwnership = Enumerable.Empty <EventProcessorPartitionOwnership>();
// Create partitions owned by a different load balancer.
var secondLoadBalancerId = Guid.NewGuid().ToString();
var secondLoadBalancerPartitions = Enumerable.Range(1, MinimumPartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(secondLoadBalancerPartitions.Select(i => i.ToString()), secondLoadBalancerId));
// Create partitions owned by a different load balancer.
var thirdLoadBalancerId = Guid.NewGuid().ToString();
var thirdLoadBalancerPartitions = Enumerable.Range(secondLoadBalancerPartitions.Max() + 1, MinimumPartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(thirdLoadBalancerPartitions.Select(i => i.ToString()), thirdLoadBalancerId));
// Seed the storageManager with all partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
// Ensure that there is exactly one more than the minimum number of partitions available to be owned.
var unownedPartitions = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));
Assert.That(unownedPartitions.Count(), Is.EqualTo(MinimumPartitionCount + 1), $"There should be { MinimumPartitionCount + 1 } partitions left unowned.");
// Run load balancing cycles until the load balancer believes that the state is balanced or the minimum count is quadrupled.
var cycleCount = 0;
while ((!loadBalancer.IsBalanced) && (cycleCount < (MinimumPartitionCount * 4)))
{
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
++cycleCount;
}
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
unownedPartitions = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));
Assert.That(unownedPartitions.Count(), Is.EqualTo(0), "There no partitions left unowned.");
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions), "All partitions should be owned.");
Assert.That(loadBalancer.IsBalanced, Is.True, "The load balancer should believe the state is balanced when it owns the correct number of partitions.");
Assert.That(cycleCount, Is.EqualTo(MinimumPartitionCount + 2), "The load balancer should have reached a balanced state once all partitions were owned and the next cycle claimed none.");
}
public async Task RunLoadBalancingAsyncReclaimsOwnershipWhenRecovering()
{
const int NumberOfPartitions = 8;
const int OrphanedPartitionCount = 4;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership.");
// Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some
// number of partitions, but the processor state does not reflect that ownership.
//
// Assign the processor ownership over half of the partitions in storage, but do not formally claim them.
var orphanedPartitions = partitionIds.Take(OrphanedPartitionCount);
completeOwnership = await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership(orphanedPartitions, loadBalancer.OwnerIdentifier));
Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking half the partitions as orphaned.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned partitions.");
// Run one load balancing cycle. At the end of the cycle, it should have claimed a random partition
// and recovered ownership of the orphans.
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "Storage should be tracking the orphaned partitions and one additional as owned.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "The load balancer should have ownership of all orphaned partitions and one additional.");
// Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled.
var cycleCount = 0;
while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4)))
{
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
++cycleCount;
}
// All partitions should be owned by load balancer.
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions));
}
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadbalancer1 = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
var loadbalancer2 = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the load balancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
// Stopping the load balancer should relinquish all partition ownership.
await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None);
completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// No partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0));
// Start loadbalancer2 so that the load balancer claims a random partition until none are left.
// All partitions should be immediately claimable even though they were just claimed by the loadbalancer1.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// All partitions are owned by loadbalancer2.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
}
private IPartitionManager BuildPartitionManager(ILeaseStoreManager leaseStoreManager)
{
string feedCollectionSelfLink = this.feedCollectionLocation.GetCollectionSelfLink();
var factory = new CheckpointerObserverFactory(this.observerFactory, this.changeFeedProcessorOptions.CheckpointFrequency);
var synchronizer = new PartitionSynchronizer(
this.feedDocumentClient,
feedCollectionSelfLink,
leaseStoreManager,
leaseStoreManager,
this.changeFeedProcessorOptions.DegreeOfParallelism,
this.changeFeedProcessorOptions.QueryPartitionsMaxBatchSize);
var bootstrapper = new Bootstrapper(synchronizer, leaseStoreManager, this.lockTime, this.sleepTime);
var partitionSuperviserFactory = new PartitionSupervisorFactory(
factory,
leaseStoreManager,
this.partitionProcessorFactory ?? new PartitionProcessorFactory(this.feedDocumentClient, this.changeFeedProcessorOptions, leaseStoreManager, feedCollectionSelfLink),
this.changeFeedProcessorOptions);
if (this.loadBalancingStrategy == null)
{
this.loadBalancingStrategy = new EqualPartitionsBalancingStrategy(
this.HostName,
this.changeFeedProcessorOptions.MinPartitionCount,
this.changeFeedProcessorOptions.MaxPartitionCount,
this.changeFeedProcessorOptions.LeaseExpirationInterval);
}
IPartitionController partitionController = new PartitionController(leaseStoreManager, leaseStoreManager, partitionSuperviserFactory, synchronizer);
if (this.healthMonitor == null)
{
this.healthMonitor = new TraceHealthMonitor();
}
partitionController = new HealthMonitoringPartitionControllerDecorator(partitionController, this.healthMonitor);
var partitionLoadBalancer = new PartitionLoadBalancer(
partitionController,
leaseStoreManager,
this.loadBalancingStrategy,
this.changeFeedProcessorOptions.LeaseAcquireInterval);
return(new PartitionManager(bootstrapper, partitionController, partitionLoadBalancer));
}
public async Task VerifiesEventProcessorLogs()
{
const int NumberOfPartitions = 4;
const int MinimumpartitionCount = NumberOfPartitions / 2;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var completeOwnership = CreatePartitionOwnership(partitionIds.Skip(1), loadbalancer2Id);
// Seed the storageManager with the owned partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
var mockLog = new Mock <PartitionLoadBalancerEventSource>();
loadbalancer.Logger = mockLog.Object;
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
await loadbalancer.RelinquishOwnershipAsync(CancellationToken.None);
mockLog.Verify(m => m.RenewOwnershipStart(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.RenewOwnershipComplete(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ClaimOwnershipStart(It.Is <string>(p => partitionIds.Contains(p))));
mockLog.Verify(m => m.MinimumPartitionsPerEventProcessor(MinimumpartitionCount));
mockLog.Verify(m => m.CurrentOwnershipCount(MinimumpartitionCount, loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.StealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ShouldStealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.UnclaimedPartitions(It.Is <HashSet <string> >(set => set.Count == 0 || set.All(item => partitionIds.Contains(item)))));
}
public async Task IsBalancedIsCorrectWithOneProcessor()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadBalancer = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0), "No partitions should be owned.");
// Start the load balancer so that it claims a random partition until none are left.
for (var index = 0; index < NumberOfPartitions; ++index)
{
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
Assert.That(loadBalancer.IsBalanced, Is.False, "The load balancer should not believe the state is balanced while partitions remain unclaimed.");
}
// The load balancer should not consider itself balanced until a cycle is run with no partitions claimed. Run one additional
// cycle to satisfy that condition.
Assert.That(loadBalancer.IsBalanced, Is.False, "The load balancer should not believe the state is balanced until no partition is claimed during a cycle.");
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// All partitions are owned by load balancer.
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions), "All partitions should be owned.");
Assert.That(loadBalancer.IsBalanced, Is.True, "The load balancer should believe the state is balanced when it owns all partitions.");
}
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions()
{
const int MinimumpartitionCount = 4;
const int MaximumpartitionCount = 5;
const int NumberOfPartitions = 12;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by this load balancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create more partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create more partitions owned by a different load balancer above the MaximumPartitionCount.
var loadbalancer3Id = Guid.NewGuid().ToString();
var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the storageManager with the owned partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
// Get owned partitions.
var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False);
// Verify load balancer 3 has stealable partitions.
Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount));
// Start the load balancer to steal ownership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a load balancer owns > MaximumPartitionCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True);
// Verify load balancer 3 now does not own > MaximumPartitionCount.
Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount));
}
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount()
{
const int MinimumPartitionCount = 4;
const int NumberOfPartitions = 13;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create partitions owned by this load balancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumPartitionCount);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumPartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create partitions owned by a different load balancer.
var loadbalancer3Id = Guid.NewGuid().ToString();
var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumPartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the storageManager with all partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));
// Get owned partitions.
var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumPartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False);
// Start the load balancer to claim ownership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumPartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True);
}
public async Task RunLoadBalancingAsyncReclaimsOwnershipWhenLeaseRenewalFails()
{
const int NumberOfPartitions = 8;
const int OrphanedPartitionCount = 4;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var mockStorageManager = new Mock <InMemoryStorageManager>()
{
CallBase = true
};
var loadBalancer = new PartitionLoadBalancer(mockStorageManager.Object, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10));
// Ownership should start empty.
var completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership.");
// Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some
// number of partitions, but the processor state does not reflect that ownership.
//
// Assign the processor ownership over half of the partitions in storage, but do not formally claim them.
var orphanedPartitions = partitionIds.Take(OrphanedPartitionCount);
completeOwnership = await mockStorageManager.Object.ClaimOwnershipAsync(CreatePartitionOwnership(orphanedPartitions, loadBalancer.OwnerIdentifier));
Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking half the partitions as orphaned.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned partitions.");
// Configure the Storage Manager to fail all claim attempts moving forward.
mockStorageManager
.Setup(sm => sm.ClaimOwnershipAsync(It.IsAny <IEnumerable <EventProcessorPartitionOwnership> >(), It.IsAny <CancellationToken>()))
.ReturnsAsync(Enumerable.Empty <EventProcessorPartitionOwnership>());
// Run one load balancing cycle. At the end of the cycle, it should have recovered ownership of the orphans
// but made no new claims.
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking the orphaned partitions as owned.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount), "The load balancer should have ownership of all orphaned partitions and none additional.");
// Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled.
var cycleCount = 0;
while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4)))
{
await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
++cycleCount;
}
// Only the orphaned partitions should be owned by load balancer, other claims have failed.
completeOwnership = await mockStorageManager.Object.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount), "Storage should be tracking the orphaned partitions as owned.");
Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount), "The load balancer should have ownership of all orphaned partitions and none additional.");
}
/// <summary>
/// Initializes a new instance of the <see cref="MockEventProcessorClient" /> class.
/// </summary>
///
/// <param name="storageManager">The client responsible for interaction with durable storage, responsible for persisting checkpoints and load-balancing state.</param>
/// <param name="consumerGroup">The name of the consumer group this processor is associated with. Events are read in the context of this group.</param>
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace to connect to. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the specific Event Hub to associate the processor with.</param>
/// <param name="connectionFactory">A factory used to provide new <see cref="EventHubConnection" /> instances.</param>
/// <param name="clientOptions">The set of options to use for this processor.</param>
/// <param name="fakePartitionProcessing"><c>true</c> if <see cref="RunPartitionProcessingAsync" /> should be overridden; otherwise, <c>false</c>.</param>
/// <param name="numberOfPartitions">The amount of partitions the associated Event Hub has.</param>
/// <param name="loadBalancer">The <see cref="PartitionLoadBalancer" /> used to manage partition load balance operations.</param>
///
internal MockEventProcessorClient(StorageManager storageManager,
string consumerGroup = "consumerGroup",
string fullyQualifiedNamespace = "somehost.com",
string eventHubName = "somehub",
Func <EventHubConnection> connectionFactory = default,
EventProcessorClientOptions clientOptions = default,
bool fakePartitionProcessing = true,
int numberOfPartitions = 3,
PartitionLoadBalancer loadBalancer = default) : base(storageManager, consumerGroup, fullyQualifiedNamespace, eventHubName, connectionFactory, clientOptions, loadBalancer)
{
StorageManager = storageManager;
var partitionIds = Enumerable
.Range(1, numberOfPartitions)
.Select(p => p.ToString())
.ToArray();
foreach (var partitionId in partitionIds)
{
EventPipeline[partitionId] = new ConcurrentQueue <EventData>();
}
FakeRunPartitionProcessingAsync = fakePartitionProcessing;
ProcessErrorAsync += eventArgs =>
{
Exception[] newException = new Exception[] { eventArgs.Exception };
ExceptionCalls.AddOrUpdate(
eventArgs.PartitionId,
newException,
(partitionId, value) => value.Concat(newException).ToArray());
return(Task.CompletedTask);
};
ProcessEventAsync += eventArgs =>
{
EventData[] newEvent = new EventData[] { eventArgs.Data };
ProcessEventCalls.AddOrUpdate(
eventArgs.Partition.PartitionId,
newEvent,
(partitionId, value) => value.Concat(newEvent).ToArray());
return(Task.CompletedTask);
};
PartitionInitializingAsync += eventArgs =>
{
InitializeCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1);
return(Task.CompletedTask);
};
PartitionClosingAsync += eventArgs =>
{
CloseCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1);
StopReasons[eventArgs.PartitionId] = eventArgs.Reason;
return(Task.CompletedTask);
};
}
