public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership() { var partitionManager = new MockCheckPointStorage(); IEnumerable <PartitionOwnership> ownership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(ownership, Is.Not.Null.And.Empty); }
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownershipCount = 5; for (int i = 0; i < ownershipCount; i++) { ownershipList.Add( new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", $"partitionId { i }" )); } await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount)); Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True); }
public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the loadbalancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by loadbalancer. Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions)); }
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing() { string fullyQualifiedNamespace = "namespace"; string eventHubName = "eventHub"; var mockStorage = new MockCheckPointStorage(); var mockConsumer = new Mock<EventHubConsumerClient>("cg", Mock.Of<EventHubConnection>(), default); var mockProcessor = new Mock<EventProcessorClient>(mockStorage, "cg", fullyQualifiedNamespace, eventHubName, Mock.Of<Func<EventHubConnection>>(), default, default) { CallBase = true };
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately() { const int NumberOfPartitions = 3; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer1 = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); var loadbalancer2 = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Ownership should start empty. var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); Assert.That(completeOwnership.Count(), Is.EqualTo(0)); // Start the loadbalancer so that it claims a random partition until none are left. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup); // All partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); // Stopping the loadbalancer should relinquish all partition ownership. await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None); completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // No partitions are owned by loadbalancer1. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0)); // Start loadbalancer2 so that the loadbalancer claims a random partition until none are left. // All partitions should be immediately claimable even though they were just claimed by the loadbalancer1. for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup); // All partitions are owned by loadbalancer2. Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions)); }
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownershipCount = 5; for (int i = 0; i < ownershipCount; i++) { ownershipList.Add( new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", $"{ i }" )); } await partitionManager.ClaimOwnershipAsync(ownershipList); // The ETags must have been set by the partition manager. var eTags = ownershipList.Select(ownership => ownership.ETag).ToList(); ownershipList.Clear(); // Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful // claims (rounded down). var expectedClaimedCount = ownershipCount / 2; for (int i = 0; i < ownershipCount; i++) { ownershipList.Add( new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", $"{ i }", eTag: i % 2 == 1 ? eTags[i] : null )); } IEnumerable <PartitionOwnership> claimedOwnership = await partitionManager.ClaimOwnershipAsync(ownershipList); Assert.That(claimedOwnership, Is.Not.Null); Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount)); Assert.That(claimedOwnership.OrderBy(ownership => int.Parse(ownership.PartitionId)).SequenceEqual(ownershipList.Where(ownership => int.Parse(ownership.PartitionId) % 2 == 1)), Is.True); }
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace1", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); // ETag must have been set by the partition manager. var eTag = firstOwnership.ETag; ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace2", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup"); IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup"); Assert.That(storedOwnership1, Is.Not.Null); Assert.That(storedOwnership1.Count, Is.EqualTo(1)); Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership)); Assert.That(storedOwnership2, Is.Not.Null); Assert.That(storedOwnership2.Count, Is.EqualTo(1)); Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership)); }
public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces() { var partitionManager = new MockCheckPointStorage(); await partitionManager.UpdateCheckpointAsync(new Checkpoint ("namespace1", "eventHubName", "consumerGroup", "partitionId", 10, 20)); await partitionManager.UpdateCheckpointAsync(new Checkpoint ("namespace2", "eventHubName", "consumerGroup", "partitionId", 10, 20)); IEnumerable <Checkpoint> storedCheckpointsList1 = await partitionManager.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup"); IEnumerable <Checkpoint> storedCheckpointsList2 = await partitionManager.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup"); Assert.That(storedCheckpointsList1, Is.Not.Null); Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1)); Assert.That(storedCheckpointsList2, Is.Not.Null); Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1)); }
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag) { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var firstOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(firstOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); ownershipList.Clear(); var secondOwnership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", eTag: eTag ); ownershipList.Add(secondOwnership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership)); }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { var partitionManager = new MockCheckPointStorage(); await partitionManager.UpdateCheckpointAsync(new Checkpoint ("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20)); await partitionManager.UpdateCheckpointAsync(new Checkpoint ("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20)); IEnumerable <Checkpoint> storedCheckpointsList = await partitionManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedCheckpointsList, Is.Not.Null); Assert.That(storedCheckpointsList.Count, Is.EqualTo(2)); Checkpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1"); Checkpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2"); Assert.That(storedCheckpoint1, Is.Not.Null); Assert.That(storedCheckpoint2, Is.Not.Null); }
public async Task VerifiesEventProcessorLogs() { const int NumberOfPartitions = 4; const int MinimumpartitionCount = NumberOfPartitions / 2; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create more partitions owned by a different loadbalancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var completeOwnership = CreatePartitionOwnership(partitionIds.Skip(1), loadbalancer2Id); // Seed the partitionManager with the owned partitions. await partitionManager.ClaimOwnershipAsync(completeOwnership); var mockLog = new Mock <PartitionLoadBalancerEventSource>(); loadbalancer.Logger = mockLog.Object; for (int i = 0; i < NumberOfPartitions; i++) { await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); } await loadbalancer.RelinquishOwnershipAsync(CancellationToken.None); mockLog.Verify(m => m.RenewOwnershipStart(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.RenewOwnershipComplete(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.ClaimOwnershipStart(It.Is <string>(p => partitionIds.Contains(p)))); mockLog.Verify(m => m.MinimumPartitionsPerEventProcessor(MinimumpartitionCount)); mockLog.Verify(m => m.CurrentOwnershipCount(MinimumpartitionCount, loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.StealPartition(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.ShouldStealPartition(loadbalancer.OwnerIdentifier)); mockLog.Verify(m => m.UnclaimedPartitions(It.Is <HashSet <string> >(p => p.Overlaps(partitionIds)))); }
public async Task FirstOwnershipClaimSucceeds() { var partitionManager = new MockCheckPointStorage(); var ownershipList = new List <PartitionOwnership>(); var ownership = new PartitionOwnership ( "namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId" ); ownershipList.Add(ownership); await partitionManager.ClaimOwnershipAsync(ownershipList); IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedOwnership, Is.Not.Null); Assert.That(storedOwnership.Count, Is.EqualTo(1)); Assert.That(storedOwnership.Single(), Is.EqualTo(ownership)); }
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions() { const int MinimumpartitionCount = 4; const int MaximumpartitionCount = 5; const int NumberOfPartitions = 12; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create more partitions owned by this loadbalancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create more partitions owned by a different loadbalancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create more partitions owned by a different loadbalancer above the MaximumPartitionCount. var loadbalancer3Id = Guid.NewGuid().ToString(); var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the partitionManager with the owned partitions. await partitionManager.ClaimOwnershipAsync(completeOwnership); // Get owned partitions. var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False); // Verify loadbalancer 3 has stealable partitions. Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount)); // Start the loadbalancer to steal owership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a loadbalancer owns > MaximumPartitionCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True); // Verify loadbalancer 3 now does not own > MaximumPartitionCount. Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount)); }
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount() { const int MinimumpartitionCount = 4; const int NumberOfPartitions = 13; var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray(); var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s)); var loadbalancer = new PartitionLoadBalancer( partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1)); // Create partitions owned by this loadbalancer. var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount); var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier); // Create partitions owned by a different loadbalancer. var loadbalancer2Id = Guid.NewGuid().ToString(); var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id)); // Create partitions owned by a different loadbalancer. var loadbalancer3Id = Guid.NewGuid().ToString(); var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumpartitionCount); completeOwnership = completeOwnership .Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id)); // Seed the partitionManager with all partitions. await partitionManager.ClaimOwnershipAsync(completeOwnership); var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId)); // Get owned partitions. var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify owned partitionIds match the owned partitions. Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumpartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False); // Start the loadbalancer to claim owership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount. await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None); // Get owned partitions. totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup); ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier); // Verify that we took ownership of the additional partition. Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumpartitionCount)); Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True); }
public async Task PartitionProcessorCanCreateACheckpoint() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { // Send some events. EventData lastEvent; await using (var producer = new EventHubProducerClient(connection)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < dummyEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { Assert.Fail("Convert to iterator."); //receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new MockCheckPointStorage(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, partitionManager, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { eventArgs.UpdateCheckpointAsync(); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); } } }
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing() { var mockStorage = new MockCheckPointStorage(); var mockConsumer = new Mock <EventHubConsumerClient>("cg", Mock.Of <EventHubConnection>(), default); var mockProcessor = new Mock <EventProcessorClient>(mockStorage, "cg", "ns", "eh", Mock.Of <Func <EventHubConnection> >(), default, default)
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing() { var mockStorage = new MockCheckPointStorage(); var mockConsumer = new Mock <EventHubConsumerClient>("cg", Mock.Of <EventHubConnection>(), default); var mockProcessor = new Mock <EventProcessorClient>(mockStorage, "cg", "ns", "eh", Mock.Of <Func <EventHubConnection> >(), default) { CallBase = true }; using ClientDiagnosticListener listener = new ClientDiagnosticListener(DiagnosticSourceName); var completionSource = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously); var processEventCalls = 0; mockConsumer .Setup(consumer => consumer.ReadEventsFromPartitionAsync( It.IsAny <string>(), It.IsAny <EventPosition>(), It.IsAny <ReadEventOptions>(), It.IsAny <CancellationToken>())) .Returns <string, EventPosition, ReadEventOptions, CancellationToken>((partitionId, position, options, token) => { async IAsyncEnumerable <PartitionEvent> mockPartitionEventEnumerable() { var context = new MockPartitionContext(partitionId); yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>()) { Properties = { { "Diagnostic-Id", "id" } } })); yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>()) { Properties = { { "Diagnostic-Id", "id2" } } })); while (!completionSource.Task.IsCompleted && !token.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(1)); yield return(new PartitionEvent()); } yield break; }; return(mockPartitionEventEnumerable()); }); mockProcessor .Setup(processor => processor.CreateConsumer( It.IsAny <string>(), It.IsAny <EventHubConnection>(), It.IsAny <EventHubConsumerClientOptions>())) .Returns(mockConsumer.Object); mockProcessor.Object.ProcessEventAsync += eventArgs => { if (++processEventCalls == 2) { completionSource.SetResult(null); } return(Task.CompletedTask); }; // RunPartitionProcessingAsync does not invoke the error handler, but we are setting it here in case // this fact changes in the future. mockProcessor.Object.ProcessErrorAsync += eventArgs => Task.CompletedTask; // Start processing and wait for the consumer to be invoked. Set a cancellation for backup to ensure // that the test completes deterministically. using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(15)); using var partitionProcessingTask = Task.Run(() => mockProcessor.Object.RunPartitionProcessingAsync("pid", default, cancellationSource.Token));
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { string partitionId; int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; long?checkpointedSequenceNumber = default; await using (var producer = new EventHubProducerClient(connectionString)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connection)) { partitionId = (await consumer.GetPartitionIdsAsync()).First(); // Send a few dummy events. We are not expecting to receive these. var dummyEventsCount = 30; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < dummyEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { Assert.Fail("Convert to iterator"); //receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber; // Send the events we expect to receive. using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < expectedEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } } // Create a partition manager and add an ownership with a checkpoint in it. var partitionManager = new MockCheckPointStorage(); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId, lastModifiedTime: DateTimeOffset.UtcNow) }); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, partitionManager, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { Interlocked.Increment(ref receivedEventsCount); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }