public void SettingTheRetryPolicyUpdatesState()
        {
            var retryOptions = new RetryOptions
            {
                Delay          = TimeSpan.FromSeconds(1),
                MaximumDelay   = TimeSpan.FromSeconds(2),
                TryTimeout     = TimeSpan.FromSeconds(3),
                MaximumRetries = 4,
                Mode           = RetryMode.Fixed
            };

            var customRetry     = Mock.Of <EventHubRetryPolicy>();
            var consumerOptions = new EventHubConsumerOptions {
                RetryOptions = retryOptions
            };
            var consumer = new EventHubConsumer(new ObservableTransportConsumerMock(), "dummy", "consumerGroup", "0", EventPosition.Latest, consumerOptions, new BasicRetryPolicy(retryOptions));

            Assert.That(consumer.RetryPolicy, Is.InstanceOf <BasicRetryPolicy>(), "The retry policy should have been created from options");

            consumer.RetryPolicy = customRetry;
            Assert.That(consumer.RetryPolicy, Is.SameAs(customRetry), "The custom retry policy should have been set.");

            var activeOptions = (EventHubConsumerOptions)
                                typeof(EventHubConsumer)
                                .GetProperty("Options", BindingFlags.Instance | BindingFlags.NonPublic)
                                .GetValue(consumer);

            Assert.That(activeOptions.RetryOptions, Is.Null, "Setting a custom policy should clear the retry options.");
        }
Example #2
0
        private static void InitEventConsumer()
        {
            var config = EventHubConfigSection.GetConfig();

            _eventHubConsumer = new EventHubConsumer(config, "weather");
            _eventHubConsumer.RegisterEventProcessor <SimpleEventProcessor>();
        }
        public void ConstructorSetsTheConsumerGroup()
        {
            var consumerGroup     = "SomeGroup";
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", consumerGroup, "0", EventPosition.Latest, new EventHubConsumerOptions(), Mock.Of <EventHubRetryPolicy>());

            Assert.That(consumer.ConsumerGroup, Is.EqualTo(consumerGroup));
        }
        public void ConstructorSetsTheStartingPosition()
        {
            var expectedPosition  = EventPosition.FromSequenceNumber(5641);
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, "0", expectedPosition, new EventHubConsumerOptions(), Mock.Of <EventHubRetryPolicy>());

            Assert.That(consumer.StartingPosition, Is.EqualTo(expectedPosition));
        }
        public void ConstructorSetsThePartition()
        {
            var partition         = "aPartition";
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, partition, EventPosition.FromSequenceNumber(1), new EventHubConsumerOptions(), Mock.Of <EventHubRetryPolicy>());

            Assert.That(consumer.PartitionId, Is.EqualTo(partition));
        }
        public void ConstructorSetsTheRetryPolicy()
        {
            var retryPolicy       = Mock.Of <EventHubRetryPolicy>();
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", "consumerGroup", "0", EventPosition.Latest, new EventHubConsumerOptions(), retryPolicy);

            Assert.That(consumer.RetryPolicy, Is.SameAs(retryPolicy));
        }
        public async Task CloseAsyncClosesTheTransportConsumer()
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroup, "0", EventPosition.Latest, new EventHubConsumerOptions());

            await consumer.CloseAsync();

            Assert.That(transportConsumer.WasCloseCalled, Is.True);
        }
        public void ReceiveAsyncValidatesTheMaximumCount(int maximumMessageCount)
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroup, "0", EventPosition.Latest, new EventHubConsumerOptions());
            var cancellation      = new CancellationTokenSource();
            var expectedWaitTime  = TimeSpan.FromDays(1);

            Assert.That(async() => await consumer.ReceiveAsync(maximumMessageCount, expectedWaitTime, cancellation.Token), Throws.InstanceOf <ArgumentException>());
        }
        public void ReceiveAsyncValidatesTheMaximumWaitTime(int timeSpanDelta)
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, "0", EventPosition.Latest, new EventHubConsumerOptions(), Mock.Of <EventHubRetryPolicy>());
            var cancellation      = new CancellationTokenSource();
            var expectedWaitTime  = TimeSpan.FromMilliseconds(timeSpanDelta);

            Assert.That(async() => await consumer.ReceiveAsync(32, expectedWaitTime, cancellation.Token), Throws.InstanceOf <ArgumentException>());
        }
        public void CloseClosesTheTransportConsumer()
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, "0", EventPosition.Latest, new EventHubConsumerOptions(), Mock.Of <EventHubRetryPolicy>());

            consumer.Close();

            Assert.That(transportConsumer.WasCloseCalled, Is.True);
        }
        public void CloseClosesTheTransportConsumer()
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", "0", EventPosition.Latest, new EventHubConsumerOptions());

            consumer.Close();

            Assert.That(transportConsumer.WasCloseCalled, Is.True);
        }
        public void SettingTheRetryPolicyUpdatesTheTransportConsumer()
        {
            var customRetry       = Mock.Of <EventHubRetryPolicy>();
            var transportConsumer = new ObservableTransportConsumerMock();
            var consumerOptions   = new EventHubConsumerOptions();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", "consumerGroup", "0", EventPosition.Latest, consumerOptions, Mock.Of <EventHubRetryPolicy>());

            consumer.RetryPolicy = customRetry;
            Assert.That(transportConsumer.UpdateRetryPolicyCalledWith, Is.SameAs(customRetry), "The custom retry policy should have been set.");
        }
        public void ConstructorSetsTheConsumerGroup()
        {
            var options = new EventHubConsumerOptions
            {
                ConsumerGroup = "SomeGroup"
            };

            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", "0", EventPosition.Latest, options);

            Assert.That(consumer.ConsumerGroup, Is.EqualTo(options.ConsumerGroup));
        }
        public void ConstructorSetsThePriority(long?priority)
        {
            var options = new EventHubConsumerOptions
            {
                OwnerLevel = priority
            };

            var transportConsumer = new ObservableTransportConsumerMock();
            var consumer          = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, "0", EventPosition.FromOffset(65), options, Mock.Of <EventHubRetryPolicy>());

            Assert.That(consumer.OwnerLevel, Is.EqualTo(priority));
        }
Example #15
0
        private static async Task CreateSenderAndReceiver()
        {
            Console.Write("Creating the Sender and Receivers... ");
            var partition       = (await client.GetPartitionIdsAsync()).First();
            var producerOptions = new EventHubProducerOptions
            {
                PartitionId = partition
            };

            sender   = client.CreateProducer(producerOptions);
            receiver = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partition, EventPosition.Latest);
            Console.WriteLine("\tdone");
        }
        public async Task ReceiveAsyncInvokesTheTransportConsumer()
        {
            var options = new EventHubConsumerOptions {
                DefaultMaximumReceiveWaitTime = TimeSpan.FromMilliseconds(8)
            };
            var transportConsumer    = new ObservableTransportConsumerMock();
            var consumer             = new EventHubConsumer(transportConsumer, "dummy", EventHubConsumer.DefaultConsumerGroupName, "0", EventPosition.Latest, options, Mock.Of <EventHubRetryPolicy>());
            var cancellation         = new CancellationTokenSource();
            var expectedMessageCount = 45;

            await consumer.ReceiveAsync(expectedMessageCount, null, cancellation.Token);

            (var actualMessageCount, var actualWaitTime) = transportConsumer.ReceiveCalledWith;

            Assert.That(actualMessageCount, Is.EqualTo(expectedMessageCount), "The message counts should match.");
            Assert.That(actualWaitTime, Is.EqualTo(options.DefaultMaximumReceiveWaitTime), "The wait time should match.");
        }
Example #17
0
        static async Task ReceiveMessages(string connectionString, int numPartitions, int numClients, bool verbose)
        {
            Console.WriteLine($"Receiving messages from {numPartitions} partitions using {numClients} client instances");

            var clients = new EventHubClient[numClients];

            for (var i = 0; i < numClients; i++)
            {
                clients[i] = new EventHubClient(connectionString, _eventHubName);
            }

            try
            {
                var client       = clients.First();
                var partitionIds = (await client.GetPartitionIdsAsync()).Take(numPartitions);
                var partitions   = await Task.WhenAll(partitionIds.Select(id => client.GetPartitionPropertiesAsync(id)));

                var totalCount = (long)0;
                foreach (var partition in partitions)
                {
                    var begin = partition.BeginningSequenceNumber;
                    var end   = partition.LastEnqueuedSequenceNumber;
                    var count = end - begin + 1;
                    totalCount += count;

                    if (verbose)
                    {
                        Console.WriteLine($"Partition: {partition.Id}, Begin: {begin}, End: {end}, Count: {count}");
                    }
                }
                if (verbose)
                {
                    Console.WriteLine($"Total Count: {totalCount}");
                }

                var consumers = new EventHubConsumer[numPartitions];
                for (var i = 0; i < numPartitions; i++)
                {
                    consumers[i] = clients[i % numClients].CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitions[i].Id, EventPosition.Earliest);
                }

                try
                {
                    var receiveTasks = new Task <(int messagesReceived, long lastSequenceNumber)> [numPartitions];
                    var sw           = Stopwatch.StartNew();
                    for (var i = 0; i < numPartitions; i++)
                    {
                        receiveTasks[i] = ReceiveAllMessages(consumers[i], partitions[i]);
                    }
                    var results = await Task.WhenAll(receiveTasks);

                    sw.Stop();

                    var elapsed            = sw.Elapsed.TotalSeconds;
                    var messagesReceived   = results.Select(r => r.messagesReceived).Sum();
                    var messagesPerSecond  = messagesReceived / elapsed;
                    var megabytesPerSecond = (messagesPerSecond * _bytesPerMessage) / (1024 * 1024);

                    Console.WriteLine($"Received {messagesReceived} messages of size {_bytesPerMessage} in {elapsed:N2}s " +
                                      $"({messagesPerSecond:N2} msg/s, {megabytesPerSecond:N2} MB/s)");
                }
Example #18
0
        public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    // Send some events.

                    EventData lastEvent;
                    var       dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few events.  We are only interested in the last one of them.

                            var dummyEventsCount = 10;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            lastEvent = receivedEvents.Last();
                        }

                    // Create a partition manager so we can retrieve the created checkpoint from it.

                    var partitionManager = new InMemoryPartitionManager();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Any())
                        {
                            partitionContext.UpdateCheckpointAsync(eventsList.Last());
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);

                    Assert.That(ownershipEnumerable, Is.Not.Null);
                    Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));

                    PartitionOwnership ownership = ownershipEnumerable.Single();

                    Assert.That(ownership.Offset.HasValue, Is.True);
                    Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));

                    Assert.That(ownership.SequenceNumber.HasValue, Is.True);
                    Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
                }
            }
        }
Example #19
0
        public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    int receivedEventsCount = 0;

                    // Send some events.

                    var  expectedEventsCount        = 20;
                    var  dummyEvent                 = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
                    long?checkpointedSequenceNumber = default;

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few dummy events.  We are not expecting to receive these.

                            var dummyEventsCount = 30;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;

                            // Send the events we expect to receive.

                            for (int i = 0; i < expectedEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }
                        }

                    // Create a partition manager and add an ownership with a checkpoint in it.

                    var partitionManager = new InMemoryPartitionManager();

                    await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
                    {
                        new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
                                               EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
                                               sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
                    });

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            Interlocked.Add(ref receivedEventsCount, eventsList.Count);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
                }
            }
        }
Example #20
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // An Event Hub consumer is associated with a specific Event Hub partition and a consumer group.  The consumer group is
            // a label that identifies one or more consumers as a set.  Often, consumer groups are named after the responsibility
            // of the consumer in an application, such as "Telemetry" or "OrderProcessing".  When an Event Hub is created, a default
            // consumer group is created with it, called "$Default."
            //
            // Each consumer has a unique view of the events in the partition, meaning that events are available to all consumers
            // and are not removed from the partition when a consumer reads them.  This allows for different consumers to read and
            // process events from the partition at different speeds and beginning with different events without interfering with
            // one another.
            //
            // When events are published, they will continue to exist in the partition and be available for consuming until they
            // reach an age where they are older than the retention period.
            // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
            //
            // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
            // would like to begin reading events.  For example, this may be starting from the very beginning of the stream, at an
            // offset from the beginning, the next event available after a specific point in time, or at a specific event.

            // We will start by creating a client using its default set of options.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // With our client, we can now inspect the partitions and find the identifier
                // of the first.

                string firstPartition = (await client.GetPartitionIdsAsync()).First();

                // In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
                // that is created with an Event Hub.  Our consumer will begin watching the partition at the very end, reading only new events
                // that we will publish for it.

                await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
                    await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
                        PartitionId = firstPartition
                    }))
                    {
                        // Because our consumer is reading from the latest position, it won't see events that have previously
                        // been published.  Before we can publish the events, we will need to ask the consumer to perform an operation,
                        // because it opens its connection only when it needs to.  The first receive that we ask of it will not see
                        // any events, but will allow the consumer to start watching the partition.
                        //
                        // Because the maximum wait time is specivied as zero, this call will return immediately and will not
                        // have consumed any events.

                        await consumer.ReceiveAsync(1, TimeSpan.Zero);

                        // Now that the consumer is watching the partition, let's publish the event that we would like to
                        // receive.

                        await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));

                        Console.WriteLine("The event batch has been published.");

                        // Because publishing and receving events is asynchronous, the events that we published may not
                        // be immediately available for our consumer to see.
                        //
                        // Each receive specifies the maximum amount of events that we would like in the batch and the maximum
                        // amount of time that we would like to wait for them.  If there are enough events available to meet the
                        // requested amount, they'll be returned immediately.  If not, the consumer will wait and collect events
                        // as they become available in an attempt to reach the requested amount.  If the maximum time that we've
                        // allowed it to wait passes, the consumer will return the events that it has collected so far.
                        //
                        // Each Receive call may return between zero and the number of events that we requested, depending on the
                        // state of events in the partition.  Likewise, it may return immediately or take up to the maximum wait
                        // time that we've allowed.
                        //
                        // We will ask for just our event, but allow a fairly long wait period to ensure that we're able to receive it.
                        // If you observe the time that the call takes, it is extremely likely that the request to recieve will complete
                        // long before the maximum wait time.

                        Stopwatch watch = Stopwatch.StartNew();
                        IEnumerable <EventData> receivedBatch = await consumer.ReceiveAsync(1, TimeSpan.FromSeconds(2.5));

                        watch.Stop();

                        // Print out the events that we received.

                        Console.WriteLine();
                        Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");

                        foreach (EventData eventData in receivedBatch)
                        {
                            // The body of our event was an encoded string; we'll recover the
                            // message by reversing the encoding process.

                            string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                            Console.WriteLine($"\tMessage: \"{ message }\"");
                        }
                    }
            }

            // At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
Example #21
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // We will start by creating a client using its default set of options.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // With our client, we can now inspect the partitions and find the identifier of the first.

                string firstPartition = (await client.GetPartitionIdsAsync()).First();

                // In this example, we will make use of multiple consumers for the first partition in the Event Hub, using the default consumer group
                // that is created with an Event Hub.  Our initial consumer will begin watching the partition at the very end, reading only new events
                // that we will publish for it.

                await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
                    PartitionId = firstPartition
                }))
                {
                    // Each event that the initial consumer reads will have attributes set that describe the event's place in the
                    // partition, such as it's offset, sequence number, and the date/time that it was enqueued.  These attributes can be
                    // used to create a new consumer that begins consuming at a known position.
                    //
                    // With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed,
                    // and to manage where in the partition the consumer begins reading events.  This is done by using the position information to track
                    // state, commonly known as "creating a checkpoint."
                    //
                    // The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the
                    // consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that
                    // begins reading at the position where it left off.
                    //
                    // It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint.  A well-designed
                    // consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues.
                    // Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart,
                    // there is a small possibility that the service will return an event multiple times.
                    //
                    // In this example, we will publish a batch of events to be received with an initial consumer.  The third event that is consumed will be captured
                    // and another consumer will use it's attributes to start reading the event that follows, consuming the same set of events that our initial consumer
                    // read, skipping over the first three.

                    int eventBatchSize = 50;

                    EventData thirdEvent;

                    await using (EventHubConsumer initialConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
                    {
                        // The first receive that we ask of it will not see any events, but allows the consumer to start watching the partition.  Because
                        // the maximum wait time is specified as zero, this call will return immediately and will not have consumed any events.

                        await initialConsumer.ReceiveAsync(1, TimeSpan.Zero);

                        // Now that the consumer is watching the partition, let's publish a batch of events.

                        EventData[] eventBatch = new EventData[eventBatchSize];

                        for (int index = 0; index < eventBatchSize; ++index)
                        {
                            eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"));
                        }

                        await producer.SendAsync(eventBatch);

                        Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");

                        // We will consume the events until all of the published events have been received.

                        var receivedEvents = new List <EventData>();

                        CancellationTokenSource cancellationSource = new CancellationTokenSource();
                        cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                        await foreach (EventData currentEvent in initialConsumer.SubscribeToEvents(cancellationSource.Token))
                        {
                            receivedEvents.Add(currentEvent);

                            if (receivedEvents.Count >= eventBatchSize)
                            {
                                break;
                            }
                        }

                        // Print out the events that we received.

                        Console.WriteLine();
                        Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published.  { eventBatchSize } were expected.");

                        foreach (EventData eventData in receivedEvents)
                        {
                            // The body of our event was an encoded string; we'll recover the
                            // message by reversing the encoding process.

                            string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                            Console.WriteLine($"\tMessage: \"{ message }\"");
                        }

                        // Remember the third event that was consumed.

                        thirdEvent = receivedEvents[2];
                    }

                    // At this point, our initial consumer has passed its "using" scope and has been safely disposed of.
                    //
                    // Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available
                    // sequence number, allowing it to read the set of published events beginning with the fourth one.

                    await using (EventHubConsumer newConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value)))
                    {
                        // We will consume the events using the new consumer until all of the published events have been received.

                        CancellationTokenSource cancellationSource = new CancellationTokenSource();
                        cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                        int expectedCount  = (eventBatchSize - 3);
                        var receivedEvents = new List <EventData>();

                        await foreach (EventData currentEvent in newConsumer.SubscribeToEvents(cancellationSource.Token))
                        {
                            receivedEvents.Add(currentEvent);

                            if (receivedEvents.Count >= expectedCount)
                            {
                                break;
                            }
                        }

                        // Print out the events that we received.

                        Console.WriteLine();
                        Console.WriteLine();
                        Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published.  { expectedCount } were expected.");

                        foreach (EventData eventData in receivedEvents)
                        {
                            // The body of our event was an encoded string; we'll recover the
                            // message by reversing the encoding process.

                            string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                            Console.WriteLine($"\tMessage: \"{ message }\"");
                        }
                    }
                }
            }

            // At this point, our client, all consumers, and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // An Event Hub consumer is associated with a specific Event Hub partition and a consumer group.  The consumer group is
            // a label that identifies one or more consumers as a set.  Often, consumer groups are named after the responsibility
            // of the consumer in an application, such as "Telemetry" or "OrderProcessing".  When an Event Hub is created, a default
            // consumer group is created with it, called "$Default."
            //
            // Each consumer has a unique view of the events in the partition, meaning that events are available to all consumers
            // and are not removed from the partition when a consumer reads them.  This allows for different consumers to read and
            // process events from the partition at different speeds and beginning with different events without interfering with
            // one another.
            //
            // When events are published, they will continue to exist in the partition and be available for consuming until they
            // reach an age where they are older than the retention period.
            // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
            //
            // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
            // would like to begin reading events.  For example, this may be starting from the very beginning of the stream, at an
            // offset from the beginning, the next event available after a specific point in time, or at a specific event.

            // We will start by creating a client using its default set of options.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // With our client, we can now inspect the partitions and find the identifier
                // of the first.

                string firstPartition = (await client.GetPartitionIdsAsync()).First();

                // In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
                // that is created with an Event Hub.  Our consumer will begin watching the partition at the very end, reading only new events
                // that we will publish for it.

                await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
                    await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
                        PartitionId = firstPartition
                    }))
                    {
                        // Because our consumer is reading from the latest position, it won't see events that have previously
                        // been published.  Before we can publish the events, we will need to ask the consumer to perform an operation,
                        // because it opens its connection only when it needs to.  The first receive that we ask of it will not see
                        // any events, but will allow the consumer to start watching the partition.
                        //
                        // Because the maximum wait time is specified as zero, this call will return immediately and will not
                        // have consumed any events.

                        await consumer.ReceiveAsync(1, TimeSpan.Zero);

                        // Now that the consumer is watching the partition, let's publish the events that we would like to
                        // receive.

                        EventData[] eventsToPublish = new EventData[]
                        {
                            new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")),
                            new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!"))
                        };

                        await producer.SendAsync(eventsToPublish);

                        Console.WriteLine("The event batch has been published.");

                        // Because publishing and receiving events is asynchronous, the events that we published may not be immediately
                        // available for our consumer to see.  We will iterate over the available events in the partition, which should be
                        // just the events that we published.
                        //
                        // When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed,
                        // whether or not an event is available in the partition.  If no event was available a null value will be emitted instead.
                        // This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event
                        // processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue
                        // if events are not arriving.
                        //
                        // For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more
                        // event than we published, which is expected to be a null value triggered by exceeding the wait time.
                        //
                        // To be sure that we do not end up in an infinite loop, we will specify a fairly long time to allow processing to complete
                        // then cancel.

                        CancellationTokenSource cancellationSource = new CancellationTokenSource();
                        cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                        TimeSpan         maximumWaitTime = TimeSpan.FromMilliseconds(250);
                        List <EventData> receivedEvents  = new List <EventData>();
                        Stopwatch        watch           = Stopwatch.StartNew();

                        await foreach (EventData currentEvent in consumer.SubscribeToEvents(maximumWaitTime, cancellationSource.Token))
                        {
                            receivedEvents.Add(currentEvent);

                            if (receivedEvents.Count > eventsToPublish.Length)
                            {
                                watch.Stop();
                                break;
                            }
                        }

                        // Print out the events that we received.

                        Console.WriteLine();
                        Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");

                        foreach (EventData eventData in receivedEvents)
                        {
                            // The body of our event was an encoded string; we'll recover the
                            // message by reversing the encoding process.

                            string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray());
                            Console.WriteLine($"\tMessage: \"{ message }\"");
                        }
                    }
            }

            // At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
Example #23
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // We will start by creating a client using its default set of options.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // With our client, we can now inspect the partitions and find the identifier
                // of the first.

                string firstPartition = (await client.GetPartitionIdsAsync()).First();

                // In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
                // that is created with an Event Hub.  Our consumer will begin watching the partition at the very end, reading only new events
                // that we will publish for it.

                await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
                    await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
                        PartitionId = firstPartition
                    }))
                    {
                        // Because our consumer is reading from the latest position, it won't see events that have previously
                        // been published. Before we can publish the events, we will need to ask the consumer to perform an operation,
                        // because it opens its connection only when it needs to.  The first receive that we ask of it will not see
                        // any events, but will allow the consumer to start watching the partition.
                        //
                        // Because the maximum wait time is specivied as zero, this call will return immediately and will not
                        // have consumed any events.

                        await consumer.ReceiveAsync(1, TimeSpan.Zero);

                        // Now that the consumer is watching the partition, let's publish a fair sized batch of events
                        // we would like for it to consume.

                        int         eventBatchSize = 100;
                        EventData[] eventBatch     = new EventData[eventBatchSize];

                        for (int index = 0; index < eventBatchSize; ++index)
                        {
                            eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"));
                        }

                        await producer.SendAsync(eventBatch);

                        Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");

                        // To allow for throughput in our application, we will process the published events by consuming them in
                        // small batches with a small wait time.  This will allow us to receive and process more quickly than blocking
                        // to wait on a larger batch size.
                        //
                        // The values that are used in this example are intended just for illustration and not as guidance for a recommended
                        // set of defaults.  In a real-world application, determining the right balance of batch size and wait time to achieve
                        // the desired performance will often vary depending on the application and event data being consumed.  It is an area
                        // where some experimentation in the application context may prove helpful.
                        //
                        // Our example will attempt to read about 1/5 of the batch at a time, which should result in the need for 5 batches to
                        // be consumed.  Because publishing and receving events is asynchronous, the events that we published may not be
                        // immediately available for our consumer to see. To compensate, we will allow for a small number of extra attempts beyond
                        // the expected 5 to to be sure that we don't stop reading before we receive all of our events.

                        var receivedEvents   = new List <EventData>();
                        int consumeBatchSize = (int)Math.Floor(eventBatchSize / 5.0f);
                        int maximumAttempts  = 15;
                        int attempts         = 0;

                        while ((receivedEvents.Count < eventBatchSize) && (++attempts < maximumAttempts))
                        {
                            // Each receive, we ask for the maximum amount of events that we would like in the batch and
                            // specify the maximum amount of time that we would like to wait for them.
                            //
                            // The batch of events will be returned to us when either we have read our maximum number of events
                            // or when the time that we asked to wait has elapsed.  This means that a batch we receive may have
                            // between zero and the maximum we asked for, depending on what was available in the partition.
                            //
                            // For this attempt, we will ask to receive our computed batch size (1/5 of published events) and wait, at most,
                            // 25 milliseconds to receive them.  We'll then process them and if we haven't gotten all that we published, we'll
                            // make another attempt until either we have them or we have tried for our maximum number of attempts.

                            IEnumerable <EventData> receivedBatch = await consumer.ReceiveAsync(consumeBatchSize, TimeSpan.FromMilliseconds(25));

                            receivedEvents.AddRange(receivedBatch);
                        }

                        // Print out the events that we received.

                        Console.WriteLine();
                        Console.WriteLine($"Events Consumed: { receivedEvents.Count }");

                        foreach (EventData eventData in receivedEvents)
                        {
                            // The body of our event was an encoded string; we'll recover the
                            // message by reversing the encoding process.

                            string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                            Console.WriteLine($"\tMessage: \"{ message }\"");
                        }
                    }
            }

            // At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }