Beispiel #1
0
        public async Task RunLoadBalancingAsyncDoesNotStealOwnershipAsRecovery()
        {
            const int NumberOfPartitions     = 8;
            const int MinimumPartitionCount  = 4;
            const int OrphanedPartitionCount = 2;

            var otherLoadBalancerIdentifier = Guid.NewGuid().ToString();
            var partitionIds   = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
            var storageManager = new InMemoryStorageManager((s) => Console.WriteLine(s));
            var loadBalancer   = new PartitionLoadBalancer(storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10));

            // Ownership should start empty.

            var completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);

            Assert.That(completeOwnership.Count(), Is.EqualTo(0), "Storage should be tracking no ownership to start.");
            Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should start with no ownership.");

            // Claim the minimum set of partitions for the "other" load balancer.

            completeOwnership = await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership(partitionIds.Take(MinimumPartitionCount), otherLoadBalancerIdentifier));

            Assert.That(completeOwnership.Count(), Is.EqualTo(MinimumPartitionCount), "Storage should be tracking half the partitions as owned by another processor.");
            Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of any partitions.");

            // Mimic the state of a processor when recovering from a crash; storage says that the processor has ownership of some
            // number of partitions, but the processor state does not reflect that ownership.
            //
            // Assign the processor ownership over half of the partitions in storage, but do not formally claim them.

            await storageManager.ClaimOwnershipAsync(CreatePartitionOwnership( partitionIds.Skip(MinimumPartitionCount).Take(OrphanedPartitionCount), loadBalancer.OwnerIdentifier));

            completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);

            Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + MinimumPartitionCount), "Storage should be tracking half the partitions as owned by another processor as well as some orphans.");
            Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(0), "The load balancer should have no ownership of orphaned or otherwise owned partitions.");

            // Run one load balancing cycle.  At the end of the cycle, it should have claimed a random partition
            // and recovered ownership of the orphans.

            await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);

            completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);

            Assert.That(completeOwnership.Count(), Is.EqualTo(OrphanedPartitionCount + MinimumPartitionCount + 1), "Storage should be tracking the orphaned partitions, other processor partitions, and one additional as owned.");
            Assert.That(loadBalancer.OwnedPartitionIds.Count(), Is.EqualTo(OrphanedPartitionCount + 1), "The load balancer should have ownership of all orphaned partitions and one additional.");

            // Run load balancing cycles until the load balancer believes that the state is balanced or the partition count is quadrupled.

            var cycleCount = 0;

            while ((!loadBalancer.IsBalanced) && (cycleCount < (NumberOfPartitions * 4)))
            {
                await loadBalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);

                ++cycleCount;
            }

            // All partitions should be owned by load balancer.

            completeOwnership = await storageManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);

            Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions));
        }
Beispiel #2
0
        public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions()
        {
            const int MinimumpartitionCount = 4;
            const int MaximumpartitionCount = 5;
            const int NumberOfPartitions    = 12;
            var       partitionIds          = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
            var       storageManager        = new InMemoryStorageManager((s) => Console.WriteLine(s));
            var       loadbalancer          = new PartitionLoadBalancer(
                storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10));

            // Create more partitions owned by this load balancer.

            var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1);
            var completeOwnership         = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);

            // Create more partitions owned by a different load balancer.

            var loadbalancer2Id           = Guid.NewGuid().ToString();
            var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);

            completeOwnership = completeOwnership
                                .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));

            // Create more partitions owned by a different load balancer above the MaximumPartitionCount.

            var loadbalancer3Id       = Guid.NewGuid().ToString();
            var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount);

            completeOwnership = completeOwnership
                                .Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id));

            // Seed the storageManager with the owned partitions.

            await storageManager.ClaimOwnershipAsync(completeOwnership);

            // Get owned partitions.

            var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);

            var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
            var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);

            // Verify owned partitionIds match the owned partitions.

            Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False);

            // Verify load balancer 3 has stealable partitions.

            Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount));

            // Start the load balancer to steal ownership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a load balancer owns > MaximumPartitionCount.

            await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);

            // Get owned partitions.

            totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);

            ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
            ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);

            // Verify that we took ownership of the additional partition.

            Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True);

            // Verify load balancer 3 now does not own > MaximumPartitionCount.

            Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount));
        }
        public async Task ProcessorClientBeginsWithTheNextEventAfterCheckpointing()
        {
            // Setup the environment.

            await using EventHubScope scope = await EventHubScope.CreateAsync(1);

            var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromMinutes(4));

            // Send a set of events.

            var segmentEventCount      = 25;
            var beforeCheckpointEvents = CreateEvents(segmentEventCount).ToList();
            var afterCheckpointEvents  = CreateEvents(segmentEventCount, segmentEventCount).ToList();
            var sourceEvents           = Enumerable.Concat(beforeCheckpointEvents, afterCheckpointEvents).ToList();
            var checkpointEvent        = beforeCheckpointEvents.Last();
            var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the first half of the events and checkpoint.

            Func <ProcessEventArgs, Task> processedEventCallback = async args =>
            {
                if (args.Data.IsEquivalentTo(checkpointEvent))
                {
                    await args.UpdateCheckpointAsync(cancellationSource.Token);
                }
            };

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var beforeCheckpointProcessHandler = CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token, processedEventCallback);
            var options = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250)
            };
            var storageManager = new InMemoryStorageManager(_ => {});
            var processor      = CreateProcessor(scope.ConsumerGroups.First(), connectionString, storageManager, options);

            processor.ProcessErrorAsync += CreateAssertingErrorHandler();
            processor.ProcessEventAsync += beforeCheckpointProcessHandler;

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            // Validate a checkpoint was created and that events were processed.

            var checkpoints = (await storageManager.ListCheckpointsAsync(processor.FullyQualifiedNamespace, processor.EventHubName, processor.ConsumerGroup, cancellationSource.Token))?.ToList();

            Assert.That(checkpoints, Is.Not.Null, "A checkpoint should have been created.");
            Assert.That(checkpoints.Count, Is.EqualTo(1), "A single checkpoint should exist.");
            Assert.That(processedEvents.Count, Is.AtLeast(beforeCheckpointEvents.Count), "All events before the checkpoint should have been processed.");

            // Reset state and start the processor again; it should resume from the event following the checkpoint.

            processedEvents.Clear();
            completionSource             = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            processor.ProcessEventAsync -= beforeCheckpointProcessHandler;
            processor.ProcessEventAsync += CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token);

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            cancellationSource.Cancel();

            foreach (var sourceEvent in afterCheckpointEvents)
            {
                var sourceId = sourceEvent.Properties[CustomIdProperty].ToString();
                Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed.");
                Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event.");
            }
        }
Beispiel #4
0
        public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount()
        {
            const int MinimumPartitionCount = 4;
            const int NumberOfPartitions    = 13;
            var       partitionIds          = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
            var       storageManager        = new InMemoryStorageManager((s) => Console.WriteLine(s));
            var       loadbalancer          = new PartitionLoadBalancer(
                storageManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10));

            // Create partitions owned by this load balancer.

            var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumPartitionCount);
            var completeOwnership         = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);

            // Create partitions owned by a different load balancer.

            var loadbalancer2Id           = Guid.NewGuid().ToString();
            var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumPartitionCount);

            completeOwnership = completeOwnership
                                .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));

            // Create partitions owned by a different load balancer.

            var loadbalancer3Id           = Guid.NewGuid().ToString();
            var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumPartitionCount);

            completeOwnership = completeOwnership
                                .Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id));

            // Seed the storageManager with all partitions.

            await storageManager.ClaimOwnershipAsync(completeOwnership);

            var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));

            // Get owned partitions.

            var totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);

            var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);

            // Verify owned partitionIds match the owned partitions.

            Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumPartitionCount));
            Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False);

            // Start the load balancer to claim ownership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount.

            await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);

            // Get owned partitions.

            totalOwnedPartitions = await storageManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);

            ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);

            // Verify that we took ownership of the additional partition.

            Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumPartitionCount));
            Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True);
        }
        public async Task ProcessorClientCreatesOwnership()
        {
            // Setup the environment.

            var partitionCount = 2;
            var partitionIds   = new HashSet <string>();

            await using EventHubScope scope = await EventHubScope.CreateAsync(partitionCount);

            var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromMinutes(2));

            // Discover the partitions.

            await using (var producer = new EventHubProducerClient(connectionString))
            {
                foreach (var partitionId in (await producer.GetPartitionIdsAsync()))
                {
                    partitionIds.Add(partitionId);
                }
            }

            // Send a set of events.

            var sourceEvents = CreateEvents(200).ToList();
            var sentCount    = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the events.

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var storageManager   = new InMemoryStorageManager(_ => {});
            var options          = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250)
            };
            var processor = CreateProcessorWithIdentity(scope.ConsumerGroups.First(), scope.EventHubName, storageManager, options);

            processor.ProcessErrorAsync += CreateAssertingErrorHandler();
            processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token);

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled.  { processedEvents.Count } events were processed.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            cancellationSource.Cancel();

            // Validate that events that were processed.

            var ownership = (await storageManager.ListOwnershipAsync(TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, scope.ConsumerGroups.First(), cancellationSource.Token))?.ToList();

            Assert.That(ownership, Is.Not.Null, "The ownership list should have been returned.");
            Assert.That(ownership.Count, Is.AtLeast(1), "At least one partition should have been owned.");

            foreach (var partitionOwnership in ownership)
            {
                Assert.That(partitionIds.Contains(partitionOwnership.PartitionId), Is.True, $"The partition `{ partitionOwnership.PartitionId }` is not valid for the Event Hub.");
                Assert.That(partitionOwnership.OwnerIdentifier, Is.Empty, "Ownership should have bee relinquished when the processor was stopped.");
            }
        }