public async Task EventProcessorCanReceiveFromCheckpointedEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { string partitionId; int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; long?checkpointedSequenceNumber = default; await using (var producer = new EventHubProducerClient(connectionString)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connection)) { partitionId = (await consumer.GetPartitionIdsAsync()).First(); // Send a few dummy events. We are not expecting to receive these. var dummyEventsCount = 30; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < dummyEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { Assert.Fail("Convert to iterator"); //receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber; // Send the events we expect to receive. using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < expectedEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } } // Create a partition manager and add an ownership with a checkpoint in it. var partitionManager = new MockCheckPointStorage(); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId, lastModifiedTime: DateTimeOffset.UtcNow) }); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, partitionManager, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { Interlocked.Increment(ref receivedEventsCount); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { allReceivedEvents.AddOrUpdate ( eventArgs.Partition.PartitionId, partitionId => new List <EventData>() { eventArgs.Data }, (partitionId, list) => { list.Add(eventArgs.Data); return(list); } ); } } ); eventProcessorManager.AddEventProcessors(1); // Send some events. string[] partitionIds; var expectedEvents = new Dictionary <string, List <EventData> >(); await using (var producer = new EventHubProducerClient(connection)) { partitionIds = await producer.GetPartitionIdsAsync(); foreach (var partitionId in partitionIds) { // Send a similar set of events for every partition. using (var partitionBatch = await producer.CreateBatchAsync(new CreateBatchOptions { PartitionId = partitionId })) { expectedEvents[partitionId] = new List <EventData> { new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")), new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come.")) }; foreach (var expectedEvent in expectedEvents[partitionId]) { partitionBatch.TryAdd(expectedEvent); } await producer.SendAsync(partitionBatch); } } } // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Make sure we received every event with the correct partition context, // in the order they were sent. foreach (var partitionId in partitionIds) { Assert.That(allReceivedEvents.TryGetValue(partitionId, out List <EventData> partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received."); Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match."); var index = 0; foreach (EventData receivedEvent in partitionReceivedEvents) { Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events."); ++index; } } Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count())); } } }
public async Task EventProcessorCanStartAgainAfterStopping() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { int receivedEventsCount = 0; // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { Interlocked.Increment(ref receivedEventsCount); } } ); eventProcessorManager.AddEventProcessors(1); // Send some events. var expectedEventsCount = 20; await using (var producer = new EventHubProducerClient(connection)) { using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < expectedEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } } // We'll start and stop the event processors twice. This way, we can assert they will behave // the same way both times, reprocessing all events in the second run. for (int i = 0; i < 2; i++) { receivedEventsCount = 0; // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }."); } } } }
public async Task ProducerSendsEventsInTheSameSetToTheSamePartition() { var partitions = 10; await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { var eventBatch = Enumerable .Range(0, 30) .Select(index => new EventData(Encoding.UTF8.GetBytes("I'm getting used to this amount of messages"))) .ToList(); var partitionIds = await client.GetPartitionIdsAsync(); var partitionsCount = 0; var receivedEventsCount = 0; var consumers = new List <EventHubConsumer>(); try { for (var index = 0; index < partitions; index++) { consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest)); // Initiate an operation to force the consumer to connect and set its position at the // end of the event stream. await consumers[index].ReceiveAsync(1, TimeSpan.Zero); } // Send the batch of events. await producer.SendAsync(eventBatch); // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. foreach (EventHubConsumer consumer in consumers) { var receivedEvents = new List <EventData>(); var index = 0; while (++index < ReceiveRetryLimit) { receivedEvents.AddRange(await consumer.ReceiveAsync(eventBatch.Count + 10, TimeSpan.FromMilliseconds(25))); } if (receivedEvents.Count > 0) { partitionsCount++; receivedEventsCount += receivedEvents.Count; } } } finally { foreach (EventHubConsumer consumer in consumers) { consumer.Close(); } } Assert.That(partitionsCount, Is.EqualTo(1)); Assert.That(receivedEventsCount, Is.EqualTo(eventBatch.Count)); } } }
public async Task ProducerSendsEventsWithTheSamePartitionHashKeyToTheSamePartition() { var partitions = 10; var partitionKey = "some123key-!d"; await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { var batches = 5; var partitionIds = await client.GetPartitionIdsAsync(); var partitionsCount = 0; var receivedEventsCount = 0; var consumers = new List <EventHubConsumer>(); try { for (var index = 0; index < partitions; index++) { consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest)); // Initiate an operation to force the consumer to connect and set its position at the // end of the event stream. await consumers[index].ReceiveAsync(1, TimeSpan.Zero); } // Send the batches of events. var batchOptions = new SendOptions { PartitionKey = partitionKey }; for (var index = 0; index < batches; index++) { await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes($"Just a few messages ({ index })")), batchOptions); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. foreach (EventHubConsumer consumer in consumers) { var receivedEvents = new List <EventData>(); var index = 0; while (++index < ReceiveRetryLimit) { receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25))); } if (receivedEvents.Count > 0) { partitionsCount++; receivedEventsCount += receivedEvents.Count; foreach (EventData receivedEvent in receivedEvents) { Assert.That(receivedEvent.PartitionKey, Is.EqualTo(partitionKey)); } } } } finally { foreach (EventHubConsumer consumer in consumers) { consumer.Close(); } } Assert.That(partitionsCount, Is.EqualTo(1)); Assert.That(receivedEventsCount, Is.EqualTo(batches)); } } }
public async Task EventProcessorWaitsMaximumWaitTimeForEvents(int maximumWaitTimeInSecs) { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { var timestamps = new ConcurrentDictionary <string, List <DateTimeOffset> >(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, clientOptions: new EventProcessorClientOptions { MaximumWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs) }, onInitialize: eventArgs => timestamps.TryAdd(eventArgs.PartitionId, new List <DateTimeOffset> { DateTimeOffset.UtcNow }), onProcessEvent: eventArgs => timestamps.AddOrUpdate ( // The key already exists, so the 'addValue' factory will never be called. eventArgs.Partition.PartitionId, partitionId => null, (partitionId, list) => { list.Add(DateTimeOffset.UtcNow); return(list); } ) ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. foreach (KeyValuePair <string, List <DateTimeOffset> > kvp in timestamps) { var partitionId = kvp.Key; List <DateTimeOffset> partitionTimestamps = kvp.Value; Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more time stamp samples were expected."); for (int index = 1; index < partitionTimestamps.Count; index++) { var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds; Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short."); Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long."); ++index; } } } } }
public async Task EventsCanBeReadByMultipleProcessorClients() { // Setup the environment. await using EventHubScope scope = await EventHubScope.CreateAsync(4); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(6)); // Send a set of events. var sourceEvents = EventGenerator.CreateEvents(500).ToList(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the events. var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); // Create multiple processors using the same handlers; events may not appear in the same order, but each event // should be read once. var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(500) }; var processors = new[] { CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options), CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options) }; foreach (var processor in processors) { processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token); } await Task.WhenAll(processors.Select(processor => processor.StartProcessingAsync(cancellationSource.Token))); // Allow the processors to complete, while respecting the test timeout. await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled. { processedEvents.Count } events were processed."); await Task.WhenAll(processors.Select(processor => processor.StopProcessingAsync(cancellationSource.Token))); cancellationSource.Cancel(); // Validate the events that were processed. foreach (var sourceEvent in sourceEvents) { var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString(); Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed."); Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event."); } }
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { // Send some events. EventData lastEvent; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); var partitionId = (await client.GetPartitionIdsAsync()).First(); await using (EventHubProducer producer = client.CreateProducer()) await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; for (int i = 0; i < dummyEventsCount; i++) { await producer.SendAsync(dummyEvent); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new InMemoryPartitionManager(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Any()) { partitionContext.UpdateCheckpointAsync(eventsList.Last()); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); PartitionOwnership ownership = ownershipEnumerable.Single(); Assert.That(ownership.Offset.HasValue, Is.True); Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset)); Assert.That(ownership.SequenceNumber.HasValue, Is.True); Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber)); } } }
public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); DateTimeOffset enqueuedTime; await using (EventHubProducer producer = client.CreateProducer()) { // Send a few dummy events. We are not expecting to receive these. for (int i = 0; i < 30; i++) { await producer.SendAsync(dummyEvent); } // Wait a reasonable amount of time so the events are able to reach the service. await Task.Delay(1000); // Send the events we expect to receive. enqueuedTime = DateTimeOffset.UtcNow; for (int i = 0; i < expectedEventsCount; i++) { await producer.SendAsync(dummyEvent); } } // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, options: new EventProcessorOptions { InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime) }, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Count > 0) { Interlocked.Add(ref receivedEventsCount, eventsList.Count); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }
public async Task EventProcessorCanStartAgainAfterStopping() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { int receivedEventsCount = 0; // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Count > 0) { Interlocked.Add(ref receivedEventsCount, eventsList.Count); } } ); eventProcessorManager.AddEventProcessors(1); // Send some events. var expectedEventsCount = 20; await using (EventHubProducer producer = client.CreateProducer()) { var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); for (int i = 0; i < expectedEventsCount; i++) { await producer.SendAsync(dummyEvent); } } // We'll start and stop the event processors twice. This way, we can assert they will behave // the same way both times, reprocessing all events in the second run. for (int i = 0; i < 2; i++) { receivedEventsCount = 0; // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }."); } } } }
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); long?checkpointedSequenceNumber = default; var partitionId = (await client.GetPartitionIdsAsync()).First(); await using (EventHubProducer producer = client.CreateProducer()) await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest)) { // Send a few dummy events. We are not expecting to receive these. var dummyEventsCount = 30; for (int i = 0; i < dummyEventsCount; i++) { await producer.SendAsync(dummyEvent); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber; // Send the events we expect to receive. for (int i = 0; i < expectedEventsCount; i++) { await producer.SendAsync(dummyEvent); } } // Create a partition manager and add an ownership with a checkpoint in it. var partitionManager = new InMemoryPartitionManager(); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId, sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow) }); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Count > 0) { Interlocked.Add(ref receivedEventsCount, eventsList.Count); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }
public async Task ProcessorClientBeginsWithTheNextEventAfterCheckpointing() { // Setup the environment. await using EventHubScope scope = await EventHubScope.CreateAsync(1); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(4)); // Send a set of events. var segmentEventCount = 25; var beforeCheckpointEvents = EventGenerator.CreateEvents(segmentEventCount).ToList(); var afterCheckpointEvents = EventGenerator.CreateEvents(segmentEventCount).ToList(); var sourceEvents = Enumerable.Concat(beforeCheckpointEvents, afterCheckpointEvents).ToList(); var checkpointEvent = beforeCheckpointEvents.Last(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the first half of the events and checkpoint. Func <ProcessEventArgs, Task> processedEventCallback = async args => { if (args.Data.IsEquivalentTo(checkpointEvent)) { await args.UpdateCheckpointAsync(cancellationSource.Token); } }; var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var beforeCheckpointProcessHandler = CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token, processedEventCallback); var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250) }; var storageManager = new InMemoryStorageManager(_ => {}); var processor = CreateProcessor(scope.ConsumerGroups.First(), connectionString, storageManager, options); processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += beforeCheckpointProcessHandler; await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled."); await processor.StopProcessingAsync(cancellationSource.Token); // Validate a checkpoint was created and that events were processed. var checkpoints = (await storageManager.ListCheckpointsAsync(processor.FullyQualifiedNamespace, processor.EventHubName, processor.ConsumerGroup, cancellationSource.Token))?.ToList(); Assert.That(checkpoints, Is.Not.Null, "A checkpoint should have been created."); Assert.That(checkpoints.Count, Is.EqualTo(1), "A single checkpoint should exist."); Assert.That(processedEvents.Count, Is.AtLeast(beforeCheckpointEvents.Count), "All events before the checkpoint should have been processed."); // Reset state and start the processor again; it should resume from the event following the checkpoint. processedEvents.Clear(); completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); processor.ProcessEventAsync -= beforeCheckpointProcessHandler; processor.ProcessEventAsync += CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token); await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled."); await processor.StopProcessingAsync(cancellationSource.Token); cancellationSource.Cancel(); foreach (var sourceEvent in afterCheckpointEvents) { var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString(); Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed."); Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event."); } }
public async Task ProcessorClientCanStartFromAnInitialPosition() { // Setup the environment. await using EventHubScope scope = await EventHubScope.CreateAsync(1); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(4)); // Send a set of events. var sourceEvents = EventGenerator.CreateEvents(25).ToList(); var lastSourceEvent = sourceEvents.Last(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Read the initial set back, marking the offset and sequence number of the last event in the initial set. var startingOffset = 0L; await using (var consumer = new EventHubConsumerClient(scope.ConsumerGroups.First(), connectionString)) { await foreach (var partitionEvent in consumer.ReadEventsAsync(new ReadEventOptions { MaximumWaitTime = null }, cancellationSource.Token)) { if (partitionEvent.Data.IsEquivalentTo(lastSourceEvent)) { startingOffset = partitionEvent.Data.Offset; break; } } } // Send the second set of events to be read by the processor. sourceEvents = EventGenerator.CreateEvents(20).ToList(); sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the second set of events. var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250) }; var processor = CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options); processor.PartitionInitializingAsync += args => { args.DefaultStartingPosition = EventPosition.FromOffset(startingOffset, false); return(Task.CompletedTask); }; processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token); await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled."); await processor.StopProcessingAsync(cancellationSource.Token); cancellationSource.Cancel(); // Validate the events that were processed. foreach (var sourceEvent in sourceEvents) { var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString(); Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed."); Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event."); } }
public async Task ProcessorClientCreatesOwnership() { // Setup the environment. var partitionCount = 2; var partitionIds = new HashSet <string>(); await using EventHubScope scope = await EventHubScope.CreateAsync(partitionCount); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromMinutes(2)); // Discover the partitions. await using (var producer = new EventHubProducerClient(connectionString)) { foreach (var partitionId in (await producer.GetPartitionIdsAsync())) { partitionIds.Add(partitionId); } } // Send a set of events. var sourceEvents = EventGenerator.CreateEvents(200).ToList(); var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token); Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent."); // Attempt to read back the events. var processedEvents = new ConcurrentDictionary <string, EventData>(); var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var storageManager = new InMemoryStorageManager(_ => {}); var options = new EventProcessorOptions { LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250) }; var processor = CreateProcessorWithIdentity(scope.ConsumerGroups.First(), scope.EventHubName, storageManager, options); processor.ProcessErrorAsync += CreateAssertingErrorHandler(); processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token); await processor.StartProcessingAsync(cancellationSource.Token); await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token)); Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled. { processedEvents.Count } events were processed."); await processor.StopProcessingAsync(cancellationSource.Token); cancellationSource.Cancel(); // Validate that events that were processed. var ownership = (await storageManager.ListOwnershipAsync(TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, scope.ConsumerGroups.First(), cancellationSource.Token))?.ToList(); Assert.That(ownership, Is.Not.Null, "The ownership list should have been returned."); Assert.That(ownership.Count, Is.AtLeast(1), "At least one partition should have been owned."); foreach (var partitionOwnership in ownership) { Assert.That(partitionIds.Contains(partitionOwnership.PartitionId), Is.True, $"The partition `{ partitionOwnership.PartitionId }` is not valid for the Event Hub."); Assert.That(partitionOwnership.OwnerIdentifier, Is.Empty, "Ownership should have bee relinquished when the processor was stopped."); } }
public async Task PartitionProcessorCanCreateACheckpoint() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { // Send some events. EventData lastEvent; await using (var producer = new EventHubProducerClient(connection)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < dummyEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { Assert.Fail("Convert to iterator."); //receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new MockCheckPointStorage(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, partitionManager, onProcessEvent: eventArgs => { if (eventArgs.Data != null) { eventArgs.UpdateCheckpointAsync(); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); } } }
public async Task EventProcessorCannotReceiveMoreThanMaximumMessageCountMessagesAtATime(int maximumMessageCount) { var partitions = 2; await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { var unexpectedMessageCount = -1; // Send some events. await using (EventHubProducer producer = client.CreateProducer()) { var eventSet = Enumerable .Range(0, 20 * maximumMessageCount) .Select(index => new EventData(new byte[10])) .ToList(); // Send one set per partition. for (int i = 0; i < partitions; i++) { await producer.SendAsync(eventSet); } } // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumer.DefaultConsumerGroupName, client, options: new EventProcessorOptions { MaximumMessageCount = maximumMessageCount }, onProcessEvents: (partitionContext, events, cancellationToken) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); // In case we find a message count greater than the allowed amount, we only store the first // occurrence and ignore the subsequent ones. if (eventsList.Count > maximumMessageCount) { Interlocked.CompareExchange(ref unexpectedMessageCount, eventsList.Count, -1); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(unexpectedMessageCount, Is.EqualTo(-1), $"A set of { unexpectedMessageCount } events was received."); } } }
public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; DateTimeOffset enqueuedTime; await using (var producer = new EventHubProducerClient(connection)) { // Send a few dummy events. We are not expecting to receive these. using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < 30; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } // Wait a reasonable amount of time so the events are able to reach the service. await Task.Delay(1000); // Send the events we expect to receive. enqueuedTime = DateTimeOffset.UtcNow; using (var dummyBatch = await producer.CreateBatchAsync()) { for (int i = 0; i < expectedEventsCount; i++) { dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy."))); } await producer.SendAsync(dummyBatch); } } // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString, onInitialize: eventArgs => eventArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(enqueuedTime), onProcessEvent: eventArgs => { if (eventArgs.Data != null) { Interlocked.Increment(ref receivedEventsCount); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }
public async Task ProducerDoesNotSendToSpecificPartitionWhenPartitionIdIsNotSpecified(bool nullPartition) { var partitions = 10; await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) { var producerOptions = new EventHubProducerOptions { }; if (nullPartition) { producerOptions.PartitionId = null; } await using (EventHubProducer producer = client.CreateProducer(producerOptions)) { var batches = 30; var partitionIds = await client.GetPartitionIdsAsync(); var partitionsCount = 0; var consumers = new List <EventHubConsumer>(); try { for (var index = 0; index < partitions; index++) { consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest)); // Initiate an operation to force the consumer to connect and set its position at the // end of the event stream. await consumers[index].ReceiveAsync(1, TimeSpan.Zero); } // Send the batches of events. for (var index = 0; index < batches; index++) { await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("It's not healthy to send so many messages"))); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. foreach (EventHubConsumer consumer in consumers) { var receivedEvents = new List <EventData>(); var index = 0; while (++index < ReceiveRetryLimit) { receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25))); } if (receivedEvents.Count > 0) { partitionsCount++; } } } finally { await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync())); } Assert.That(partitionsCount, Is.GreaterThan(1)); } } } }
public async Task LoadBalancingIsEnforcedWhenDistributionIsUneven() { var partitions = 10; await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { ConcurrentDictionary <string, int> ownedPartitionsCount = new ConcurrentDictionary <string, int>(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connectionString // TODO: fix test. OwnerIdentifier is not accessible anymore. // onInitialize: eventArgs => // ownedPartitionsCount.AddOrUpdate(eventArgs.Context.OwnerIdentifier, 1, (ownerId, value) => value + 1), // onStop: eventArgs => // ownedPartitionsCount.AddOrUpdate(eventArgs.Context.OwnerIdentifier, 0, (ownerId, value) => value - 1) ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize. await eventProcessorManager.WaitStabilization(); // Assert all partitions have been claimed. Assert.That(ownedPartitionsCount.ToArray().Single().Value, Is.EqualTo(partitions)); // Insert a new event processor into the manager so it can start stealing partitions. eventProcessorManager.AddEventProcessors(1); await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize. await eventProcessorManager.WaitStabilization(); // Take a snapshot of the current partition balancing status. IEnumerable <int> ownedPartitionsCountSnapshot = ownedPartitionsCount.ToArray().Select(kvp => kvp.Value); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. var minimumOwnedPartitionsCount = partitions / 2; var maximumOwnedPartitionsCount = minimumOwnedPartitionsCount + 1; foreach (var count in ownedPartitionsCountSnapshot) { Assert.That(count, Is.InRange(minimumOwnedPartitionsCount, maximumOwnedPartitionsCount)); } Assert.That(ownedPartitionsCountSnapshot.Sum(), Is.EqualTo(partitions)); } } }
public async Task ReceiveCanReadOneEventBatch() { await using (var scope = await EventHubScope.CreateAsync(4)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); var eventBatch = new[] { new EventData(Encoding.UTF8.GetBytes("One")), new EventData(Encoding.UTF8.GetBytes("Two")), new EventData(Encoding.UTF8.GetBytes("Three")) }; var receiverOptions = new EventReceiverOptions { BeginReceivingAt = EventPosition.NewEventsOnly }; await using (var client = new EventHubClient(connectionString)) { var partition = (await client.GetPartitionIdsAsync()).First(); await using (var sender = client.CreateSender(new EventSenderOptions { PartitionId = partition })) await using (var receiver = client.CreateReceiver(partition, receiverOptions)) { // Initiate an operation to force the receiver to connect and set its position at the // end of the event stream. Assert.That(async() => await receiver.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing); // Send the batch of events. await sender.SendAsync(eventBatch); // Recieve and validate the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < eventBatch.Length) && (++index < 3)) { receivedEvents.AddRange(await receiver.ReceiveAsync(eventBatch.Length + 10, TimeSpan.FromMilliseconds(25))); } index = 0; Assert.That(receivedEvents, Is.Not.Empty, "There should have been a set of events received."); foreach (var receivedEvent in receivedEvents) { Assert.That(receivedEvent.IsEquivalentTo(eventBatch[index]), Is.True, $"The received event at index: { index } did not match the sent batch."); ++index; } Assert.That(index, Is.EqualTo(eventBatch.Length), "The number of received events did not match the batch size."); } } } }