public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the load balancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by load balancer.
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions));
}
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer1 = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
var loadbalancer2 = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the load balancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
// Stopping the load balancer should relinquish all partition ownership.
await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None);
completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// No partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0));
// Start loadbalancer2 so that the load balancer claims a random partition until none are left.
// All partitions should be immediately claimable even though they were just claimed by the loadbalancer1.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await storageManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// All partitions are owned by loadbalancer2.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
}
public async Task VerifiesEventProcessorLogs()
{
const int NumberOfPartitions = 4;
const int MinimumpartitionCount = NumberOfPartitions / 2;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var completeOwnership = CreatePartitionOwnership(partitionIds.Skip(1), loadbalancer2Id);
// Seed the storageManager with the owned partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
var mockLog = new Mock <PartitionLoadBalancerEventSource>();
loadbalancer.Logger = mockLog.Object;
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
await loadbalancer.RelinquishOwnershipAsync(CancellationToken.None);
mockLog.Verify(m => m.RenewOwnershipStart(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.RenewOwnershipComplete(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ClaimOwnershipStart(It.Is <string>(p => partitionIds.Contains(p))));
mockLog.Verify(m => m.MinimumPartitionsPerEventProcessor(MinimumpartitionCount));
mockLog.Verify(m => m.CurrentOwnershipCount(MinimumpartitionCount, loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.StealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ShouldStealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.UnclaimedPartitions(It.Is <HashSet <string> >(p => p.Overlaps(partitionIds))));
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to a handler delegate for processing using code that you
// provide.
//
// These handler delegates are invoked for each event or error that occurs during operation of the event processor. For
// a given partition, only a single event will be dispatched for processing at a time so that the order of events within a
// partition is preserved. Partitions, however, are processed concurrently. As a result, your handlers are potentially processing
// multiple events or errors at any given time.
// A partition manager may create checkpoints and list/claim partition ownership. A developer may implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of a MockCheckPointStorage to get started with using the `EventProcessor` but in production,
// you should choose an implementation of the PartitionManager interface that will
// store the checkpoints and partition ownership to a persistent store instead.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
var partitionManager = new MockCheckPointStorage();
// It's also possible to specify custom options upon event processor creation. We don't want to wait
// more than 1 second for every set of events.
var eventProcessorOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessorClient(partitionManager, EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName, eventProcessorOptions);
int totalEventsCount = 0;
int partitionsBeingProcessedCount = 0;
eventProcessor.PartitionInitializingAsync += (eventArgs) =>
{
// This is the last piece of code guaranteed to run before event processing, so all initialization
// must be done by the moment this method returns.
// We want to receive events from the latest available position so older events don't interfere with our sample.
eventArgs.DefaultStartingPosition = EventPosition.Latest;
Interlocked.Increment(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ eventArgs.PartitionId }': partition processing has started.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.PartitionClosingAsync += (eventArgs) =>
{
// The code to be run just before stopping processing events for a partition. This is the right place to dispose
// of objects that will no longer be used.
Interlocked.Decrement(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ eventArgs.Partition.PartitionId }': partition processing has stopped. Reason: { eventArgs.Reason }.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessEventAsync += (eventArgs) =>
{
// Here the user can specify what to do with the event received from the event processor. We are counting how
// many events were received across all partitions so we can check whether all sent events were received.
//
// It's important to notice that this method is called even when no event is received after the maximum wait time, which
// can be specified by the user in the event processor options. In this case, the received event is null.
if (eventArgs.Data != null)
{
Interlocked.Increment(ref totalEventsCount);
Console.WriteLine($"\tPartition '{ eventArgs.Partition.PartitionId }': event received.");
}
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessErrorAsync += (eventArgs) =>
{
// Any exception which occurs as a result of the event processor itself will be passed to
// this delegate so it may be handled. The processor will continue to process events if
// it is able to unless this handler explicitly requests that it stop doing so.
//
// It is important to note that this does not include exceptions during event processing; those
// are considered responsibility of the developer implementing the event processing handler. It
// is, therefore, highly encouraged that best practices for exception handling practices are
// followed with that delegate.
//
// This piece of code is not supposed to be reached by this sample. If the following message has been printed
// to the Console, then something unexpected has happened.
Console.WriteLine($"\tPartition '{ eventArgs.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartProcessingAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. This may take some time
// as there's a 10 seconds interval between claims. To be sure that we do not block forever in case the event processor
// fails, we will specify a fairly long time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
var partitionsCount = (await producerClient.GetPartitionIdsAsync()).Length;
while (partitionsBeingProcessedCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// The processor may take some time to connect to the Event Hubs service. Let's wait 1 second before sending
// events so we don't end up missing events.
await Task.Delay(1000);
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
int amountOfSets = 10;
int eventsPerSet = 2;
int expectedAmountOfEvents = amountOfSets * eventsPerSet;
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the second event.")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the first event.")));
await producerClient.SendAsync(eventBatch);
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopProcessingAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions()
{
const int MinimumpartitionCount = 4;
const int MaximumpartitionCount = 5;
const int NumberOfPartitions = 12;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by this load balancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create more partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create more partitions owned by a different load balancer above the MaximumPartitionCount.
var loadbalancer3Id = Guid.NewGuid().ToString();
var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the storageManager with the owned partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
// Get owned partitions.
var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False);
// Verify load balancer 3 has stealable partitions.
Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount));
// Start the load balancer to steal ownership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a load balancer owns > MaximumPartitionCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True);
// Verify load balancer 3 now does not own > MaximumPartitionCount.
Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount));
}
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount()
{
const int MinimumpartitionCount = 4;
const int NumberOfPartitions = 13;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var storageManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create partitions owned by this load balancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create partitions owned by a different load balancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create partitions owned by a different load balancer.
var loadbalancer3Id = Guid.NewGuid().ToString();
var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the storageManager with all partitions.
await storageManager.ClaimOwnershipAsync(completeOwnership);
var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));
// Get owned partitions.
var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumpartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False);
// Start the load balancer to claim ownership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumpartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True);
}
