public async Task ConnectionTransportCanRetrievePartitionProperties(EventHubsTransportType transportType)
        {
            var partitionCount = 4;

            await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
            {
                var options = new EventHubConnectionOptions();

                var credential = new SharedAccessCredential
                                 (
                    new SharedAccessSignature
                    (
                        $"{ options.TransportType.GetUriScheme() }://{ EventHubsTestEnvironment.Instance.FullyQualifiedNamespace }/{ scope.EventHubName }".ToLowerInvariant(),
                        EventHubsTestEnvironment.Instance.SharedAccessKeyName,
                        EventHubsTestEnvironment.Instance.SharedAccessKey,
                        TimeSpan.FromHours(4)
                    )
                                 );

                await using (var connection = new TestConnectionWithRetryPolicy(EventHubsTestEnvironment.Instance.FullyQualifiedNamespace, scope.EventHubName, credential, new EventHubConnectionOptions {
                    TransportType = transportType
                }))
                {
                    var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(20));
                    var properties   = await connection.GetPropertiesAsync();

                    var partition           = properties.PartitionIds.First();
                    var partitionProperties = await connection.GetPartitionPropertiesAsync(partition, cancellation.Token);

                    Assert.That(partitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
                    Assert.That(partitionProperties.Id, Is.EqualTo(partition), "The partition identifier should match.");
                    Assert.That(partitionProperties.EventHubName, Is.EqualTo(scope.EventHubName).Using((IEqualityComparer <string>)StringComparer.InvariantCultureIgnoreCase), "The Event Hub path should match.");
                    Assert.That(partitionProperties.BeginningSequenceNumber, Is.Not.EqualTo(default(long)), "The beginning sequence number should have been populated.");
                    Assert.That(partitionProperties.LastEnqueuedSequenceNumber, Is.Not.EqualTo(default(long)), "The last sequence number should have been populated.");
                    Assert.That(partitionProperties.LastEnqueuedOffset, Is.Not.EqualTo(default(long)), "The last offset should have been populated.");
                }
            }
        }
示例#2
0
        public async Task ConnectionCanConnectToEventHubsUsingArguments()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var options = new EventHubConnectionOptions();

                var credential = new SharedAccessSignatureCredential
                                 (
                    new SharedAccessSignature
                    (
                        $"{ options.TransportType.GetUriScheme() }://{ TestEnvironment.FullyQualifiedNamespace }/{ scope.EventHubName }".ToLowerInvariant(),
                        TestEnvironment.SharedAccessKeyName,
                        TestEnvironment.SharedAccessKey,
                        TimeSpan.FromHours(4)
                    )
                                 );

                await using (var connection = new TestConnectionWithTransport(TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, credential))
                {
                    Assert.That(() => connection.GetPropertiesAsync(), Throws.Nothing);
                }
            }
        }
        public async Task ReceiverWithOptionsCanReceive()
        {
            await using (var scope = await EventHubScope.CreateAsync(4))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                var options = new EventReceiverOptions
                {
                    ConsumerGroup    = "$Default",
                    BeginReceivingAt = EventPosition.NewEventsOnly
                };

                await using (var client = new EventHubClient(connectionString))
                {
                    var partition = (await client.GetPartitionIdsAsync()).First();

                    await using (var receiver = client.CreateReceiver(partition, options))
                    {
                        Assert.That(async() => await receiver.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
                    }
                }
            }
        }
        public async Task ProducerCanSendLargeBatch()
        {
            await using (var scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString, new EventHubClientOptions {
                    DefaultTimeout = TimeSpan.FromMinutes(2)
                }))
                    await using (var producer = client.CreateProducer())
                    {
                        // Actual limit is 1046520 for a single event.
                        var events = new[]
                        {
                            new EventData(new byte[1000000 / 3]),
                            new EventData(new byte[1000000 / 3]),
                            new EventData(new byte[1000000 / 3])
                        };

                        Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
                    }
            }
        }
示例#5
0
        public async Task ConnectionTransportCannotRetrieveMetadataWhenClosed()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var connection = new TestConnectionWithTransport(connectionString))
                {
                    var partition = (await connection.GetPartitionIdsAsync()).First();

                    Assert.That(async() => await connection.GetPropertiesAsync(), Throws.Nothing);
                    Assert.That(async() => await connection.GetPartitionPropertiesAsync(partition), Throws.Nothing);

                    await connection.CloseAsync();

                    await Task.Delay(TimeSpan.FromSeconds(5));

                    Assert.That(async() => await connection.GetPartitionIdsAsync(), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
                    Assert.That(async() => await connection.GetPropertiesAsync(), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
                    Assert.That(async() => await connection.GetPartitionPropertiesAsync(partition), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
                }
            }
        }
示例#6
0
        public async Task ProducerUpdatesPropertiesAfterPublishingBatches()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);
                var options          = new EventHubProducerClientOptions {
                    EnableIdempotentPartitions = true
                };

                await using var producer = new EventHubProducerClient(connectionString, options);

                var cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

                var partition = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).First();
                var initialPartitionProperties = await producer.GetPartitionPublishingPropertiesAsync(partition);

                var batchOptions = new CreateBatchOptions {
                    PartitionId = partition
                };

                using var batch = await producer.CreateBatchAsync(batchOptions, cancellationSource.Token);

                batch.TryAdd(EventGenerator.CreateEvents(1).First());
                batch.TryAdd(EventGenerator.CreateEvents(1).First());
                batch.TryAdd(EventGenerator.CreateEvents(1).First());

                await producer.SendAsync(batch, cancellationSource.Token);

                var updatedPartitionProperties = await producer.GetPartitionPublishingPropertiesAsync(partition);

                Assert.That(updatedPartitionProperties.IsIdempotentPublishingEnabled, Is.True, "Idempotent publishing should be enabled.");
                Assert.That(updatedPartitionProperties.ProducerGroupId, Is.EqualTo(initialPartitionProperties.ProducerGroupId), "The producer group identifier should not have changed.");
                Assert.That(updatedPartitionProperties.OwnerLevel, Is.EqualTo(initialPartitionProperties.OwnerLevel), "The owner level should not have changed.");
                Assert.That(updatedPartitionProperties.LastPublishedSequenceNumber, Is.GreaterThan(initialPartitionProperties.LastPublishedSequenceNumber), "The last published sequence number should have increased.");
            }
        }
        public async Task PartitionReceiverCanRetrievePartitionProperties(EventHubsTransportType transportType)
        {
            var partitionCount = 1;

            await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
            {
                var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));

                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
                var receiverOptions  = new PartitionReceiverOptions {
                    ConnectionOptions = new EventHubConnectionOptions {
                        TransportType = transportType
                    }
                };

                var partitionId = default(string);

                await using (var producer = new EventHubProducerClient(connectionString))
                {
                    partitionId = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).First();
                }

                await using (var receiver = new PartitionReceiver(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connectionString, receiverOptions))
                {
                    var partitionProperties = await receiver.GetPartitionPropertiesAsync(cancellationSource.Token);

                    Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");
                    Assert.That(partitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
                    Assert.That(partitionProperties.Id, Is.EqualTo(partitionId), "The partition identifier should match.");
                    Assert.That(partitionProperties.EventHubName, Is.EqualTo(scope.EventHubName).Using((IEqualityComparer <string>)StringComparer.InvariantCultureIgnoreCase), "The Event Hub path should match.");
                    Assert.That(partitionProperties.BeginningSequenceNumber, Is.Not.EqualTo(default(long)), "The beginning sequence number should have been populated.");
                    Assert.That(partitionProperties.LastEnqueuedSequenceNumber, Is.Not.EqualTo(default(long)), "The last sequence number should have been populated.");
                    Assert.That(partitionProperties.LastEnqueuedOffset, Is.Not.EqualTo(default(long)), "The last offset should have been populated.");
                }
            }
        }
示例#8
0
        public async Task ProducerSequencesEvents()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);
                var options          = new EventHubProducerClientOptions {
                    EnableIdempotentPartitions = true
                };

                await using var producer = new EventHubProducerClient(connectionString, options);

                var cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

                var partition   = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).Last();
                var sendOptions = new SendEventOptions {
                    PartitionId = partition
                };

                var partitionProperties = await producer.GetPartitionPublishingPropertiesAsync(partition);

                var eventSequenceNumber = partitionProperties.LastPublishedSequenceNumber;

                var events = EventGenerator.CreateEvents(10).ToArray();
                Assert.That(events.Any(item => item.PublishedSequenceNumber.HasValue), Is.False, "Events should start out as unpublished with no sequence number.");

                await producer.SendAsync(events, sendOptions, cancellationSource.Token);

                Assert.That(events.All(item => item.PublishedSequenceNumber.HasValue), Is.True, "Events should be sequenced after publishing.");

                foreach (var item in events)
                {
                    Assert.That(item.PublishedSequenceNumber, Is.EqualTo(++eventSequenceNumber), $"The sequence numbers should be contiguous.  Event { eventSequenceNumber } was out of order.");
                }
            }
        }
        public async Task ConnectionTransportCanRetrieveMetadataWhenCustomValidationAcceptsTheCertificate()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);
                var retryOptions     = new EventHubsRetryOptions {
                    TryTimeout = TimeSpan.FromMinutes(2)
                };

                var clientOptions = new EventHubConnectionOptions
                {
                    CertificateValidationCallback = (object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) => true
                };

                await using (var connection = new TestConnectionWithRetryPolicy(connectionString, clientOptions))
                {
                    connection.RetryPolicy = new BasicRetryPolicy(retryOptions);

                    var partition = (await connection.GetPartitionIdsAsync()).First();
                    Assert.That(async() => await connection.GetPropertiesAsync(), Throws.Nothing);
                    Assert.That(async() => await connection.GetPartitionPropertiesAsync(partition), Throws.Nothing);
                }
            }
        }
        public async Task ProducerCanSendMultipleBatchesOfEventsUsingAPartitionHashHey()
        {
            await using (var scope = await EventHubScope.CreateAsync(4))
            {
                var batchOptions = new SendOptions {
                    PartitionKey = "some123key-!d"
                };

                for (var index = 0; index < 5; ++index)
                {
                    var events = Enumerable
                                 .Range(0, 25)
                                 .Select(index => new EventData(Encoding.UTF8.GetBytes(new String((char)(65 + index), index + 5))));

                    var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                    await using (var client = new EventHubClient(connectionString))
                        await using (var producer = client.CreateProducer())
                        {
                            Assert.That(async() => await producer.SendAsync(events, batchOptions), Throws.Nothing, $"Batch { index } should not have thrown an exception.");
                        }
                }
            }
        }
示例#11
0
        public async Task ProducerInitializesPropertiesWhenRequested()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);
                var options          = new EventHubProducerClientOptions {
                    EnableIdempotentPartitions = true
                };

                await using var producer = new EventHubProducerClient(connectionString, options);

                var cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

                var partition           = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).Last();
                var partitionProperties = await producer.GetPartitionPublishingPropertiesAsync(partition);

                Assert.That(partitionProperties, Is.Not.Null, "The properties should have been created.");
                Assert.That(partitionProperties.IsIdempotentPublishingEnabled, Is.True, "Idempotent publishing should be enabled.");
                Assert.That(partitionProperties.ProducerGroupId.HasValue, Is.True, "The producer group identifier should have a value.");
                Assert.That(partitionProperties.OwnerLevel.HasValue, Is.True, "The owner level should have a value.");
                Assert.That(partitionProperties.LastPublishedSequenceNumber.HasValue, Is.True, "The last published sequence number should have a value.");
            }
        }
        public async Task ProcessorClientCanStartFromAnInitialPosition()
        {
            // Setup the environment.

            await using EventHubScope scope = await EventHubScope.CreateAsync(1);

            var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

            // Send a set of events.

            var sourceEvents    = EventGenerator.CreateEvents(25).ToList();
            var lastSourceEvent = sourceEvents.Last();
            var sentCount       = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Read the initial set back, marking the offset and sequence number of the last event in the initial set.

            var startingOffset = 0L;

            await using (var consumer = new EventHubConsumerClient(scope.ConsumerGroups.First(), connectionString))
            {
                await foreach (var partitionEvent in consumer.ReadEventsAsync(new ReadEventOptions {
                    MaximumWaitTime = null
                }, cancellationSource.Token))
                {
                    if (partitionEvent.Data.IsEquivalentTo(lastSourceEvent))
                    {
                        startingOffset = partitionEvent.Data.Offset;

                        break;
                    }
                }
            }

            // Send the second set of events to be read by the processor.

            sourceEvents = EventGenerator.CreateEvents(20).ToList();
            sentCount    = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the second set of events.

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var options          = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250)
            };
            var processor = CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options);

            processor.PartitionInitializingAsync += args =>
            {
                args.DefaultStartingPosition = EventPosition.FromOffset(startingOffset, false);
                return(Task.CompletedTask);
            };

            processor.ProcessErrorAsync += CreateAssertingErrorHandler();
            processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token);

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            cancellationSource.Cancel();

            // Validate the events that were processed.

            foreach (var sourceEvent in sourceEvents)
            {
                var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString();
                Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed.");
                Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event.");
            }
        }
示例#13
0
        public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
        {
            await using EventHubScope scope = await EventHubScope.CreateAsync(2);

            var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

            await using var connection = new EventHubConnection(connectionString);
            int receivedEventsCount = 0;

            // Send some events.

            var            expectedEventsCount  = 20;
            var            firstBatchEventCount = 30;
            DateTimeOffset enqueuedTime         = DateTimeOffset.MinValue;

            await using var producer = new EventHubProducerClient(connection);
            // Send a few dummy events.  We are not expecting to receive these.

            using (var dummyBatch = await producer.CreateBatchAsync())
            {
                for (int i = 0; i < firstBatchEventCount; i++)
                {
                    dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(firstBatchBody)));
                }

                await producer.SendAsync(dummyBatch);
            }

            // Wait a reasonable amount of time so the there is a time gap between the first and second batch.

            await Task.Delay(2000);

            // Send the events we expect to receive.

            using (var dummyBatch = await producer.CreateBatchAsync())
            {
                for (int i = 0; i < expectedEventsCount; i++)
                {
                    dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(eventBody)));
                }

                await producer.SendAsync(dummyBatch);
            }

            // Create the event processor manager to manage the event processor which will receive all events and set the enqueuedTime of the latest event from the first batch.

            using var firstBatchCancellationSource = new CancellationTokenSource();
            firstBatchCancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
            var receivedFromFirstBatch = 0;

            var eventProcessorManager = new EventProcessorManager
                                        (
                EventHubConsumerClient.DefaultConsumerGroupName,
                connectionString,
                onInitialize: eventArgs =>
                eventArgs.DefaultStartingPosition = EventPosition.Earliest,
                onProcessEvent: eventArgs =>
            {
                if (eventArgs.Data != null)
                {
                    var dataAsString = Encoding.UTF8.GetString(eventArgs.Data.Body.Span.ToArray());
                    if (dataAsString == firstBatchBody)
                    {
                        enqueuedTime = enqueuedTime > eventArgs.Data.EnqueuedTime ? enqueuedTime : eventArgs.Data.EnqueuedTime;
                        receivedFromFirstBatch++;
                        if (receivedFromFirstBatch == firstBatchEventCount)
                        {
                            firstBatchCancellationSource.Cancel();
                        }
                    }
                }
            }
                                        );

            eventProcessorManager.AddEventProcessors(1);

            // Start the event processors.

            await eventProcessorManager.StartAllAsync();

            // Wait for the event processors to receive events.

            try
            {
                await Task.Delay(Timeout.Infinite, firstBatchCancellationSource.Token);
            }
            catch (TaskCanceledException) { /*expected*/ }


            // Stop the event processors.

            await eventProcessorManager.StopAllAsync();

            // Validate that we set at least one enqueuedTime

            Assert.That(enqueuedTime, Is.GreaterThan(DateTimeOffset.MinValue));

            // Create the event processor manager to manage the event processor which will receive all events FromEnqueuedTime of enqueuedTime.

            using var secondBatchCancellationSource = new CancellationTokenSource();
            secondBatchCancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

            var eventProcessorManager2 = new EventProcessorManager
                                         (
                EventHubConsumerClient.DefaultConsumerGroupName,
                connectionString,
                onInitialize: eventArgs =>
                eventArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(enqueuedTime),
                onProcessEvent: eventArgs =>
            {
                if (eventArgs.Data != null)
                {
                    Interlocked.Increment(ref receivedEventsCount);
                    if (receivedEventsCount >= expectedEventsCount)
                    {
                        secondBatchCancellationSource.Cancel();
                    }
                }
            }
                                         );

            eventProcessorManager2.AddEventProcessors(1);

            // Start the event processors.

            await eventProcessorManager2.StartAllAsync();

            // Wait for the event processors to receive events.

            try
            {
                await Task.Delay(Timeout.Infinite, secondBatchCancellationSource.Token);
            }
            catch (TaskCanceledException) { /*expected*/ }

            // Stop the event processors.

            await eventProcessorManager2.StopAllAsync();

            // Validate results.

            Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
        }
示例#14
0
        public async Task EventProcessorCannotReceiveMoreThanMaximumMessageCountMessagesAtATime(int maximumMessageCount)
        {
            var partitions = 2;

            await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    var unexpectedMessageCount = -1;

                    // Send some events.

                    await using (EventHubProducer producer = client.CreateProducer())
                    {
                        var eventSet = Enumerable
                                       .Range(0, 20 * maximumMessageCount)
                                       .Select(index => new EventData(new byte[10]))
                                       .ToList();

                        // Send one set per partition.

                        for (int i = 0; i < partitions; i++)
                        {
                            await producer.SendAsync(eventSet);
                        }
                    }

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        options: new EventProcessorOptions {
                        MaximumMessageCount = maximumMessageCount
                    },
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        // In case we find a message count greater than the allowed amount, we only store the first
                        // occurrence and ignore the subsequent ones.

                        if (eventsList.Count > maximumMessageCount)
                        {
                            Interlocked.CompareExchange(ref unexpectedMessageCount, eventsList.Count, -1);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    Assert.That(unexpectedMessageCount, Is.EqualTo(-1), $"A set of { unexpectedMessageCount } events was received.");
                }
            }
        }
示例#15
0
        public async Task EventProcessorWaitsMaximumReceiveWaitTimeForEvents(int maximumWaitTimeInSecs)
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    var timestamps = new ConcurrentDictionary <string, List <DateTimeOffset> >();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        options: new EventProcessorOptions {
                        MaximumReceiveWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs)
                    },
                        onInitialize: partitionContext =>
                        timestamps.TryAdd(partitionContext.PartitionId, new List <DateTimeOffset> {
                        DateTimeOffset.UtcNow
                    }),
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                        timestamps.AddOrUpdate
                        (
                            // The key already exists, so the 'addValue' factory will never be called.

                            partitionContext.PartitionId,
                            partitionId => null,
                            (partitionId, list) =>
                    {
                        list.Add(DateTimeOffset.UtcNow);
                        return(list);
                    }
                        )
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    foreach (KeyValuePair <string, List <DateTimeOffset> > kvp in timestamps)
                    {
                        var partitionId = kvp.Key;
                        List <DateTimeOffset> partitionTimestamps = kvp.Value;

                        Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more timestamp samples were expected.");

                        for (int index = 1; index < partitionTimestamps.Count; index++)
                        {
                            var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds;

                            Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short.");
                            Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long.");

                            ++index;
                        }
                    }
                }
            }
        }
示例#16
0
        public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    int receivedEventsCount = 0;

                    // Send some events.

                    var            expectedEventsCount = 20;
                    var            dummyEvent          = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
                    DateTimeOffset enqueuedTime;

                    await using (EventHubProducer producer = client.CreateProducer())
                    {
                        // Send a few dummy events.  We are not expecting to receive these.

                        for (int i = 0; i < 30; i++)
                        {
                            await producer.SendAsync(dummyEvent);
                        }

                        // Wait a reasonable amount of time so the events are able to reach the service.

                        await Task.Delay(1000);

                        // Send the events we expect to receive.

                        enqueuedTime = DateTimeOffset.UtcNow;

                        for (int i = 0; i < expectedEventsCount; i++)
                        {
                            await producer.SendAsync(dummyEvent);
                        }
                    }

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        options: new EventProcessorOptions {
                        InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime)
                    },
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            Interlocked.Add(ref receivedEventsCount, eventsList.Count);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
                }
            }
        }
示例#17
0
        public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    // Send some events.

                    EventData lastEvent;
                    var       dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few events.  We are only interested in the last one of them.

                            var dummyEventsCount = 10;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            lastEvent = receivedEvents.Last();
                        }

                    // Create a partition manager so we can retrieve the created checkpoint from it.

                    var partitionManager = new InMemoryPartitionManager();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Any())
                        {
                            partitionContext.UpdateCheckpointAsync(eventsList.Last());
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);

                    Assert.That(ownershipEnumerable, Is.Not.Null);
                    Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));

                    PartitionOwnership ownership = ownershipEnumerable.Single();

                    Assert.That(ownership.Offset.HasValue, Is.True);
                    Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));

                    Assert.That(ownership.SequenceNumber.HasValue, Is.True);
                    Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
                }
            }
        }
        public async Task ProducerSendsEventsInTheSameBatchToTheSamePartition()
        {
            var partitions = 10;

            await using (var scope = await EventHubScope.CreateAsync(partitions))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                    await using (var producer = client.CreateProducer())
                    {
                        var eventBatch = Enumerable
                                         .Range(0, 30)
                                         .Select(index => new EventData(Encoding.UTF8.GetBytes("I'm getting used to this amount of messages")))
                                         .ToList();

                        var partitionIds = await client.GetPartitionIdsAsync();

                        var partitionsCount     = 0;
                        var receivedEventsCount = 0;
                        var consumers           = new List <EventHubConsumer>();

                        try
                        {
                            for (var index = 0; index < partitions; index++)
                            {
                                consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));

                                // Initiate an operation to force the consumer to connect and set its position at the
                                // end of the event stream.

                                await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
                            }

                            // Send the batch of events.

                            await producer.SendAsync(eventBatch);

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            foreach (var consumer in consumers)
                            {
                                var receivedEvents = new List <EventData>();
                                var index          = 0;

                                while (++index < ReceiveRetryLimit)
                                {
                                    receivedEvents.AddRange(await consumer.ReceiveAsync(eventBatch.Count + 10, TimeSpan.FromMilliseconds(25)));
                                }

                                if (receivedEvents.Count > 0)
                                {
                                    partitionsCount++;
                                    receivedEventsCount += receivedEvents.Count;
                                }
                            }
                        }
                        finally
                        {
                            foreach (var consumer in consumers)
                            {
                                consumer.Close();
                            }
                        }

                        Assert.That(partitionsCount, Is.EqualTo(1));
                        Assert.That(receivedEventsCount, Is.EqualTo(eventBatch.Count));
                    }
            }
        }
示例#19
0
        public async Task EventProcessorWaitsMaximumWaitTimeForEvents(int maximumWaitTimeInSecs)
        {
            await using EventHubScope scope = await EventHubScope.CreateAsync(2);

            var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

            await using var connection = new EventHubConnection(connectionString);
            var timestamps = new ConcurrentDictionary <string, ConcurrentBag <DateTimeOffset> >();

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
            var receivedCount = 0;

            // Create the event processor manager to manage our event processors.

            var eventProcessorManager = new EventProcessorManager
                                        (
                EventHubConsumerClient.DefaultConsumerGroupName,
                connectionString,
                clientOptions: new EventProcessorClientOptions {
                MaximumWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs)
            },
                onInitialize: eventArgs =>
                timestamps.TryAdd(eventArgs.PartitionId, new ConcurrentBag <DateTimeOffset> {
                DateTimeOffset.UtcNow
            }),
                onProcessEvent: eventArgs =>
            {
                timestamps[eventArgs.Partition.PartitionId].Add(DateTimeOffset.UtcNow);
                receivedCount++;
                if (receivedCount >= 5)
                {
                    cancellationSource.Cancel();
                }
            }
                                        );

            eventProcessorManager.AddEventProcessors(1);

            // Start the event processors.

            await eventProcessorManager.StartAllAsync();

            // Make sure the event processors have enough time to receive some events.

            try
            {
                await Task.Delay(Timeout.Infinite, cancellationSource.Token);
            }
            catch (TaskCanceledException) { /*expected*/ }


            // Stop the event processors.

            await eventProcessorManager.StopAllAsync();

            // Validate results.

            foreach (KeyValuePair <string, ConcurrentBag <DateTimeOffset> > kvp in timestamps)
            {
                var partitionId         = kvp.Key;
                var partitionTimestamps = kvp.Value.ToList();
                partitionTimestamps.Sort();

                Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more time stamp samples were expected.");

                for (int index = 1; index < partitionTimestamps.Count; index++)
                {
                    var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds;

                    Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short.");
                    Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long.");

                    ++index;
                }
            }
        }
        public async Task EventsCanBeReadByMultipleProcessorClients()
        {
            // Setup the environment.

            await using EventHubScope scope = await EventHubScope.CreateAsync(4);

            var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

            // Send a set of events.

            var sourceEvents = EventGenerator.CreateEvents(500).ToList();
            var sentCount    = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the events.

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);

            // Create multiple processors using the same handlers; events may not appear in the same order, but each event
            // should be read once.

            var options = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(500)
            };

            var processors = new[]
            {
                CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options),
                CreateProcessor(scope.ConsumerGroups.First(), connectionString, options: options)
            };

            foreach (var processor in processors)
            {
                processor.ProcessErrorAsync += CreateAssertingErrorHandler();
                processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token);
            }

            await Task.WhenAll(processors.Select(processor => processor.StartProcessingAsync(cancellationSource.Token)));

            // Allow the processors to complete, while respecting the test timeout.

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled.  { processedEvents.Count } events were processed.");

            await Task.WhenAll(processors.Select(processor => processor.StopProcessingAsync(cancellationSource.Token)));

            cancellationSource.Cancel();

            // Validate the events that were processed.

            foreach (var sourceEvent in sourceEvents)
            {
                var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString();
                Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed.");
                Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event.");
            }
        }
示例#21
0
        public async Task PartitionProcessorCanCreateACheckpoint()
        {
            await using EventHubScope scope = await EventHubScope.CreateAsync(1);

            var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

            await using var connection = new EventHubConnection(connectionString);
            // Send some events.

            EventData lastEvent;

            await using var producer = new EventHubProducerClient(connection);
            await using var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString);

            // Send a few events.  We are only interested in the last one of them.

            var dummyEventsCount = 10;

            using var dummyBatch = await producer.CreateBatchAsync();

            for (int i = 0; i < dummyEventsCount; i++)
            {
                dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(eventBody)));
            }

            await producer.SendAsync(dummyBatch);

            var receivedEvents = new List <EventData>();

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

            await foreach (var evt in consumer.ReadEventsAsync(new ReadEventOptions {
                MaximumWaitTime = TimeSpan.FromSeconds(30)
            }, cancellationSource.Token))
            {
                receivedEvents.Add(evt.Data);
                if (receivedEvents.Count == dummyEventsCount)
                {
                    break;
                }
            }

            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

            lastEvent = receivedEvents.Last();

            // Create a storage manager so we can retrieve the created checkpoint from it.

            var storageManager = new MockCheckPointStorage();

            // Create the event processor manager to manage our event processors.

            var eventProcessorManager = new EventProcessorManager
                                        (
                EventHubConsumerClient.DefaultConsumerGroupName,
                connectionString,
                storageManager,
                onProcessEvent: eventArgs =>
            {
                if (eventArgs.Data != null)
                {
                    eventArgs.UpdateCheckpointAsync();
                }
            }
                                        );

            eventProcessorManager.AddEventProcessors(1);

            // Start the event processors.

            await eventProcessorManager.StartAllAsync();

            // Make sure the event processors have enough time to stabilize and receive events.

            await eventProcessorManager.WaitStabilization();

            // Stop the event processors.

            await eventProcessorManager.StopAllAsync();

            // Validate results.

            IEnumerable <EventProcessorPartitionOwnership> ownershipEnumerable = await storageManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName);

            Assert.That(ownershipEnumerable, Is.Not.Null);
            Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
        }
示例#22
0
        public async Task ProcessorDoesNotProcessCheckpointedEventsAgain()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                var firstEventBatch = Enumerable
                                      .Range(0, 20)
                                      .Select(index => new EventData(Encoding.UTF8.GetBytes($"First event batch: { index }")))
                                      .ToList();

                var secondEventBatch = Enumerable
                                       .Range(0, 10)
                                       .Select(index => new EventData(Encoding.UTF8.GetBytes($"Second event batch: { index }")))
                                       .ToList();

                var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
                var firstBatchReceivedEventsCount = 0;

                // Send the first batch of events and checkpoint after the last one.

                var checkpointStorage = new MockCheckPointStorage();
                var firstProcessor    = new EventProcessorClient(checkpointStorage, EventHubConsumerClient.DefaultConsumerGroupName,
                                                                 TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, () => new EventHubConnection(connectionString), default);

                firstProcessor.ProcessEventAsync += async eventArgs =>
                {
                    if (++firstBatchReceivedEventsCount == firstEventBatch.Count)
                    {
                        await eventArgs.UpdateCheckpointAsync();

                        completionSource.SetResult(true);
                    }
                };

                firstProcessor.ProcessErrorAsync += eventArgs => Task.CompletedTask;

                await using (var producer = new EventHubProducerClient(connectionString))
                {
                    using var batch = await producer.CreateBatchAsync();

                    foreach (var eventData in firstEventBatch)
                    {
                        batch.TryAdd(eventData);
                    }

                    await producer.SendAsync(batch);
                }

                // Establish timed cancellation to ensure that the test doesn't hang.

                using var cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromMinutes(5));

                await firstProcessor.StartProcessingAsync(cancellationSource.Token);

                while (!completionSource.Task.IsCompleted &&
                       !cancellationSource.IsCancellationRequested)
                {
                    await Task.Delay(25);
                }

                await firstProcessor.StopProcessingAsync(cancellationSource.Token);

                // Send the second batch of events. Only the new events should be read by the second processor.

                await using (var producer = new EventHubProducerClient(connectionString))
                {
                    using var batch = await producer.CreateBatchAsync();

                    foreach (var eventData in secondEventBatch)
                    {
                        batch.TryAdd(eventData);
                    }

                    await producer.SendAsync(batch);
                }

                completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
                var secondBatchReceivedEvents = new List <EventData>();

                var secondProcessor = new EventProcessorClient(checkpointStorage, EventHubConsumerClient.DefaultConsumerGroupName,
                                                               TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, () => new EventHubConnection(connectionString), default);

                secondProcessor.ProcessEventAsync += eventArgs =>
                {
                    secondBatchReceivedEvents.Add(eventArgs.Data);

                    if (secondBatchReceivedEvents.Count == firstEventBatch.Count)
                    {
                        completionSource.SetResult(true);
                    }

                    return(Task.CompletedTask);
                };

                var wasErrorHandlerCalled = false;

                secondProcessor.ProcessErrorAsync += eventArgs =>
                {
                    wasErrorHandlerCalled = true;
                    return(Task.CompletedTask);
                };

                await secondProcessor.StartProcessingAsync(cancellationSource.Token);

                while (!completionSource.Task.IsCompleted &&
                       !cancellationSource.IsCancellationRequested)
                {
                    await Task.Delay(25);
                }

                await secondProcessor.StopProcessingAsync(cancellationSource.Token);

                Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The processors should have stopped without cancellation.");
                Assert.That(wasErrorHandlerCalled, Is.False, "No errors should have happened while resuming from checkpoint.");

                var index = 0;

                foreach (var eventData in secondBatchReceivedEvents)
                {
                    Assert.That(eventData.IsEquivalentTo(secondEventBatch[index]), Is.True, "The received and sent event datas do not match.");
                    index++;
                }

                Assert.That(index, Is.EqualTo(secondEventBatch.Count), $"The second processor did not receive the expected amount of events.");
            }
        }
        public async Task ProcessorClientCreatesOwnership()
        {
            // Setup the environment.

            var partitionCount = 2;
            var partitionIds   = new HashSet <string>();

            await using EventHubScope scope = await EventHubScope.CreateAsync(partitionCount);

            var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

            // Discover the partitions.

            await using (var producer = new EventHubProducerClient(connectionString))
            {
                foreach (var partitionId in (await producer.GetPartitionIdsAsync()))
                {
                    partitionIds.Add(partitionId);
                }
            }

            // Send a set of events.

            var sourceEvents = EventGenerator.CreateEvents(200).ToList();
            var sentCount    = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the events.

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var storageManager   = new InMemoryStorageManager(_ => { });
            var options          = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250)
            };
            var processor = CreateProcessorWithIdentity(scope.ConsumerGroups.First(), scope.EventHubName, storageManager, options);

            processor.ProcessErrorAsync += CreateAssertingErrorHandler();
            processor.ProcessEventAsync += CreateEventTrackingHandler(sentCount, processedEvents, completionSource, cancellationSource.Token);

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, $"The cancellation token should not have been signaled.  { processedEvents.Count } events were processed.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            cancellationSource.Cancel();

            // Validate that events that were processed.

            var ownership = (await storageManager.ListOwnershipAsync(EventHubsTestEnvironment.Instance.FullyQualifiedNamespace, scope.EventHubName, scope.ConsumerGroups.First(), cancellationSource.Token))?.ToList();

            Assert.That(ownership, Is.Not.Null, "The ownership list should have been returned.");
            Assert.That(ownership.Count, Is.AtLeast(1), "At least one partition should have been owned.");

            foreach (var partitionOwnership in ownership)
            {
                Assert.That(partitionIds.Contains(partitionOwnership.PartitionId), Is.True, $"The partition `{ partitionOwnership.PartitionId }` is not valid for the Event Hub.");
                Assert.That(partitionOwnership.OwnerIdentifier, Is.Empty, "Ownership should have bee relinquished when the processor was stopped.");
            }
        }
示例#24
0
        public async Task LoadBalancingIsEnforcedWhenDistributionIsUneven()
        {
            var partitions = 10;

            await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    ConcurrentDictionary <string, int> ownedPartitionsCount = new ConcurrentDictionary <string, int>();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        onInitialize: partitionContext =>
                        ownedPartitionsCount.AddOrUpdate(partitionContext.OwnerIdentifier, 1, (ownerId, value) => value + 1),
                        onClose: (partitionContext, reason) =>
                        ownedPartitionsCount.AddOrUpdate(partitionContext.OwnerIdentifier, 0, (ownerId, value) => value - 1)
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize.

                    await eventProcessorManager.WaitStabilization();

                    // Assert all partitions have been claimed.

                    Assert.That(ownedPartitionsCount.ToArray().Single().Value, Is.EqualTo(partitions));

                    // Insert a new event processor into the manager so it can start stealing partitions.

                    eventProcessorManager.AddEventProcessors(1);

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize.

                    await eventProcessorManager.WaitStabilization();

                    // Take a snapshot of the current partition balancing status.

                    IEnumerable <int> ownedPartitionsCountSnapshot = ownedPartitionsCount.ToArray().Select(kvp => kvp.Value);

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    var minimumOwnedPartitionsCount = partitions / 2;
                    var maximumOwnedPartitionsCount = minimumOwnedPartitionsCount + 1;

                    foreach (var count in ownedPartitionsCountSnapshot)
                    {
                        Assert.That(count, Is.InRange(minimumOwnedPartitionsCount, maximumOwnedPartitionsCount));
                    }

                    Assert.That(ownedPartitionsCountSnapshot.Sum(), Is.EqualTo(partitions));
                }
            }
        }
        public async Task ProcessorClientBeginsWithTheNextEventAfterCheckpointing()
        {
            // Setup the environment.

            await using EventHubScope scope = await EventHubScope.CreateAsync(1);

            var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);

            using var cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);

            // Send a set of events.

            var segmentEventCount      = 25;
            var beforeCheckpointEvents = EventGenerator.CreateEvents(segmentEventCount).ToList();
            var afterCheckpointEvents  = EventGenerator.CreateEvents(segmentEventCount).ToList();
            var sourceEvents           = Enumerable.Concat(beforeCheckpointEvents, afterCheckpointEvents).ToList();
            var checkpointEvent        = beforeCheckpointEvents.Last();
            var sentCount = await SendEvents(connectionString, sourceEvents, cancellationSource.Token);

            Assert.That(sentCount, Is.EqualTo(sourceEvents.Count), "Not all of the source events were sent.");

            // Attempt to read back the first half of the events and checkpoint.

            Func <ProcessEventArgs, Task> processedEventCallback = async args =>
            {
                if (args.Data.IsEquivalentTo(checkpointEvent))
                {
                    await args.UpdateCheckpointAsync(cancellationSource.Token);
                }
            };

            var processedEvents  = new ConcurrentDictionary <string, EventData>();
            var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var beforeCheckpointProcessHandler = CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token, processedEventCallback);
            var options = new EventProcessorOptions {
                LoadBalancingUpdateInterval = TimeSpan.FromMilliseconds(250)
            };
            var storageManager = new InMemoryStorageManager(_ => { });
            var processor      = CreateProcessor(scope.ConsumerGroups.First(), connectionString, storageManager, options);

            processor.ProcessErrorAsync += CreateAssertingErrorHandler();
            processor.ProcessEventAsync += beforeCheckpointProcessHandler;

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            // Validate a checkpoint was created and that events were processed.

            var checkpoints = (await storageManager.ListCheckpointsAsync(processor.FullyQualifiedNamespace, processor.EventHubName, processor.ConsumerGroup, cancellationSource.Token))?.ToList();

            Assert.That(checkpoints, Is.Not.Null, "A checkpoint should have been created.");
            Assert.That(checkpoints.Count, Is.EqualTo(1), "A single checkpoint should exist.");
            Assert.That(processedEvents.Count, Is.AtLeast(beforeCheckpointEvents.Count), "All events before the checkpoint should have been processed.");

            // Reset state and start the processor again; it should resume from the event following the checkpoint.

            processedEvents.Clear();
            completionSource             = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            processor.ProcessEventAsync -= beforeCheckpointProcessHandler;
            processor.ProcessEventAsync += CreateEventTrackingHandler(segmentEventCount, processedEvents, completionSource, cancellationSource.Token);

            await processor.StartProcessingAsync(cancellationSource.Token);

            await Task.WhenAny(completionSource.Task, Task.Delay(Timeout.Infinite, cancellationSource.Token));

            Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");

            await processor.StopProcessingAsync(cancellationSource.Token);

            cancellationSource.Cancel();

            foreach (var sourceEvent in afterCheckpointEvents)
            {
                var sourceId = sourceEvent.Properties[EventGenerator.IdPropertyName].ToString();
                Assert.That(processedEvents.TryGetValue(sourceId, out var processedEvent), Is.True, $"The event with custom identifier [{ sourceId }] was not processed.");
                Assert.That(sourceEvent.IsEquivalentTo(processedEvent), $"The event with custom identifier [{ sourceId }] did not match the corresponding processed event.");
            }
        }
示例#26
0
        public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            allReceivedEvents.AddOrUpdate
                            (
                                partitionContext.PartitionId,
                                partitionId => eventsList,
                                (partitionId, list) =>
                            {
                                list.AddRange(eventsList);
                                return(list);
                            }
                            );
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Send some events.

                    var partitionIds = await client.GetPartitionIdsAsync();

                    var expectedEvents = new Dictionary <string, List <EventData> >();

                    foreach (var partitionId in partitionIds)
                    {
                        // Send a similar set of events for every partition.

                        expectedEvents[partitionId] = new List <EventData>
                        {
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
                            new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
                        };

                        await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
                            PartitionId = partitionId
                        }))
                        {
                            await producer.SendAsync(expectedEvents[partitionId]);
                        }
                    }

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.  Make sure we received every event in the correct partition processor,
                    // in the order they were sent.

                    foreach (var partitionId in partitionIds)
                    {
                        Assert.That(allReceivedEvents.TryGetValue(partitionId, out List <EventData> partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
                        Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");

                        var index = 0;

                        foreach (EventData receivedEvent in partitionReceivedEvents)
                        {
                            Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
                            ++index;
                        }
                    }

                    Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
                }
            }
        }
        public async Task ProducerDoesNotSendToSpecificPartitionWhenPartitionIdIsNotSpecified(bool nullPartition)
        {
            var partitions = 10;

            await using (var scope = await EventHubScope.CreateAsync(partitions))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    var producerOptions = new EventHubProducerOptions {
                    };

                    if (nullPartition)
                    {
                        producerOptions.PartitionId = null;
                    }

                    await using (var producer = client.CreateProducer(producerOptions))
                    {
                        var batches      = 30;
                        var partitionIds = await client.GetPartitionIdsAsync();

                        var partitionsCount = 0;
                        var consumers       = new List <EventHubConsumer>();

                        try
                        {
                            for (var index = 0; index < partitions; index++)
                            {
                                consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));

                                // Initiate an operation to force the consumer to connect and set its position at the
                                // end of the event stream.

                                await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
                            }

                            // Send the batches of events.

                            for (var index = 0; index < batches; index++)
                            {
                                await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("It's not healthy to send so many messages")));
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            foreach (var consumer in consumers)
                            {
                                var receivedEvents = new List <EventData>();
                                var index          = 0;

                                while (++index < ReceiveRetryLimit)
                                {
                                    receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
                                }

                                if (receivedEvents.Count > 0)
                                {
                                    partitionsCount++;
                                }
                            }
                        }
                        finally
                        {
                            await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync()));
                        }

                        Assert.That(partitionsCount, Is.GreaterThan(1));
                    }
                }
            }
        }
示例#28
0
        public async Task EventProcessorCanStartAgainAfterStopping()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    int receivedEventsCount = 0;

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            Interlocked.Add(ref receivedEventsCount, eventsList.Count);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Send some events.

                    var expectedEventsCount = 20;

                    await using (EventHubProducer producer = client.CreateProducer())
                    {
                        var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));

                        for (int i = 0; i < expectedEventsCount; i++)
                        {
                            await producer.SendAsync(dummyEvent);
                        }
                    }

                    // We'll start and stop the event processors twice.  This way, we can assert they will behave
                    // the same way both times, reprocessing all events in the second run.

                    for (int i = 0; i < 2; i++)
                    {
                        receivedEventsCount = 0;

                        // Start the event processors.

                        await eventProcessorManager.StartAllAsync();

                        // Make sure the event processors have enough time to stabilize and receive events.

                        await eventProcessorManager.WaitStabilization();

                        // Stop the event processors.

                        await eventProcessorManager.StopAllAsync();

                        // Validate results.

                        Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }.");
                    }
                }
            }
        }
        public async Task ProducerSendsEventsWithTheSamePartitionHashKeyToTheSamePartition()
        {
            var partitions   = 10;
            var partitionKey = "some123key-!d";

            await using (var scope = await EventHubScope.CreateAsync(partitions))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                    await using (var producer = client.CreateProducer())
                    {
                        var batches      = 5;
                        var partitionIds = await client.GetPartitionIdsAsync();

                        var partitionsCount     = 0;
                        var receivedEventsCount = 0;
                        var consumers           = new List <EventHubConsumer>();

                        try
                        {
                            for (var index = 0; index < partitions; index++)
                            {
                                consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));

                                // Initiate an operation to force the consumer to connect and set its position at the
                                // end of the event stream.

                                await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
                            }

                            // Send the batches of events.

                            var batchOptions = new SendOptions {
                                PartitionKey = partitionKey
                            };

                            for (var index = 0; index < batches; index++)
                            {
                                await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes($"Just a few messages ({ index })")), batchOptions);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            foreach (var consumer in consumers)
                            {
                                var receivedEvents = new List <EventData>();
                                var index          = 0;

                                while (++index < ReceiveRetryLimit)
                                {
                                    receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
                                }

                                if (receivedEvents.Count > 0)
                                {
                                    partitionsCount++;
                                    receivedEventsCount += receivedEvents.Count;

                                    foreach (var receivedEvent in receivedEvents)
                                    {
                                        Assert.That(receivedEvent.PartitionKey, Is.EqualTo(partitionKey));
                                    }
                                }
                            }
                        }
                        finally
                        {
                            foreach (var consumer in consumers)
                            {
                                consumer.Close();
                            }
                        }

                        Assert.That(partitionsCount, Is.EqualTo(1));
                        Assert.That(receivedEventsCount, Is.EqualTo(batches));
                    }
            }
        }
示例#30
0
        public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    int receivedEventsCount = 0;

                    // Send some events.

                    var  expectedEventsCount        = 20;
                    var  dummyEvent                 = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
                    long?checkpointedSequenceNumber = default;

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few dummy events.  We are not expecting to receive these.

                            var dummyEventsCount = 30;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;

                            // Send the events we expect to receive.

                            for (int i = 0; i < expectedEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }
                        }

                    // Create a partition manager and add an ownership with a checkpoint in it.

                    var partitionManager = new InMemoryPartitionManager();

                    await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
                    {
                        new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
                                               EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
                                               sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
                    });

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            Interlocked.Add(ref receivedEventsCount, eventsList.Count);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
                }
            }
        }