コード例 #1
0
        private static TimeSpan maxWaitTimeForFirstEvent = new TimeSpan(0, 0, 10); // 10 seconds

        /// <summary>
        /// Saves all data from all partitions of an eventhub consumer group
        /// </summary>
        /// <param name="ehNsConnectionString">Connection string for the EventHub Namespace</param>
        /// <param name="ehName">Name of the EventHub</param>
        /// <param name="ehConsumerGroup">The consumer group of the EventHub</param>
        /// <param name="outputFilename">The x dimension size to crop the picture. The default is 0 indicating no cropping is required.</param>

        static async Task Main(string ehNsConnectionString, string ehName, string ehConsumerGroup, FileInfo outputFilename)
        {
            Console.WriteLine($"EH-Name {ehName}");
            Console.WriteLine($"EH-Consumer Group {ehConsumerGroup}");

            EventHubConsumerClient ehClient = new EventHubConsumerClient(ehConsumerGroup, ehNsConnectionString, ehName);

            var ehPartitions = await ehClient.GetPartitionIdsAsync();

            Console.WriteLine($"Reading data from {ehPartitions.Length} partitions");

            // configure data retrieval
            var readOptions = new ReadEventOptions();

            readOptions.MaximumWaitTime = maxWaitTimeForFirstEvent;
            var ehEvents = ehClient.ReadEventsAsync(readOptions);

            // delete target file if it exists
            if (outputFilename.Exists)
            {
                outputFilename.Delete();
            }

            // grab data
            using (var outStream = new FileStream(outputFilename.FullName, FileMode.Append)) {
                await foreach (PartitionEvent ehEvent in ehEvents)
                {
                    byte[] data = ehEvent.Data.Body.ToArray();
                    Console.WriteLine($"Retrieved {data.Length} bytes");
                    outStream.Write(data, 0, data.Length);
                }
            }
        }
コード例 #2
0
        public async Task ReadPartitionTrackLastEnqueued()
        {
            await using var scope = await EventHubScope.CreateAsync(1);

            #region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued

            var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
            var eventHubName     = "<< NAME OF THE EVENT HUB >>";
            var consumerGroup    = EventHubConsumerClient.DefaultConsumerGroupName;
            /*@@*/
            /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
            /*@@*/ eventHubName     = scope.EventHubName;

            var consumer = new EventHubConsumerClient(
                consumerGroup,
                connectionString,
                eventHubName);

            try
            {
                using CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                string        firstPartition   = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
                EventPosition startingPosition = EventPosition.Earliest;

                var options = new ReadEventOptions
                {
                    TrackLastEnqueuedEventProperties = true
                };

                await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
                                   firstPartition,
                                   startingPosition,
                                   options,
                                   cancellationSource.Token))
                {
                    LastEnqueuedEventProperties properties =
                        partitionEvent.Partition.ReadLastEnqueuedEventProperties();

                    Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }");
                    Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }");
                    Debug.WriteLine($"\tThe last offset is: { properties.Offset }");
                    Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC.");
                    Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC.");
                }
            }
            catch (TaskCanceledException)
            {
                // This is expected if the cancellation token is
                // signaled.
            }
            finally
            {
                await consumer.CloseAsync();
            }

            #endregion
        }
コード例 #3
0
        public async Task ReadAllPartitionsWaitTime()
        {
            await using var scope = await EventHubScope.CreateAsync(1);

            #region Snippet:EventHubs_Sample05_ReadAllPartitionsWaitTime

#if SNIPPET
            var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
            var eventHubName     = "<< NAME OF THE EVENT HUB >>";
#else
            var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
            var eventHubName     = scope.EventHubName;
#endif
            var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;

            var consumer = new EventHubConsumerClient(
                consumerGroup,
                connectionString,
                eventHubName);

            try
            {
                int loopTicks    = 0;
                int maximumTicks = 10;

                var options = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromSeconds(1)
                };

                await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(options))
                {
                    if (partitionEvent.Data != null)
                    {
                        string readFromPartition = partitionEvent.Partition.PartitionId;
                        byte[] eventBodyBytes    = partitionEvent.Data.EventBody.ToArray();

                        Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
                    }
                    else
                    {
                        Debug.WriteLine("Wait time elapsed; no event was available.");
                    }

                    loopTicks++;

                    if (loopTicks >= maximumTicks)
                    {
                        break;
                    }
                }
            }
            finally
            {
                await consumer.CloseAsync();
            }

            #endregion
        }
コード例 #4
0
        public void CloneProducesACopy()
        {
            var options = new ReadEventOptions
            {
                OwnerLevel = 99,
                TrackLastEnqueuedEventProperties = false,
                MaximumWaitTime = TimeSpan.FromMinutes(65)
            };

            ReadEventOptions clone = options.Clone();

            Assert.That(clone, Is.Not.Null, "The clone should not be null.");

            Assert.That(clone.OwnerLevel, Is.EqualTo(options.OwnerLevel), "The owner level of the clone should match.");
            Assert.That(clone.TrackLastEnqueuedEventProperties, Is.EqualTo(options.TrackLastEnqueuedEventProperties), "The tracking of last event information of the clone should match.");
            Assert.That(clone.MaximumWaitTime, Is.EqualTo(options.MaximumWaitTime), "The default maximum wait time of the clone should match.");
        }
コード例 #5
0
        public async Task StartListeningAsync(int maxBatchSize, TimeSpan waitTimeout, CancellationToken cancellationToken)
        {
            var readOptions = new ReadEventOptions
            {
                MaximumWaitTime = waitTimeout,
            };

            await using var client = new EventHubConsumerClient(consumerGroup: "$default", connectionString: connectionString);
            await foreach (var receivedEvent in client.ReadEventsAsync(startReadingAtEarliestEvent:false, readOptions, cancellationToken))
            {
                if (receivedEvent.Data == null)
                {
                    // timeout. Yield and try again.
                    await Task.Yield();

                    continue;
                }

                await ProcessAsync(Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray()), cancellationToken);
            }
        }
コード例 #6
0
        public void CloneProducesACopy()
        {
            var options = new ReadEventOptions
            {
                OwnerLevel = 99,
                TrackLastEnqueuedEventProperties = false,
                MaximumWaitTime     = TimeSpan.FromMinutes(65),
                CacheEventCount     = 1,
                PrefetchCount       = 0,
                PrefetchSizeInBytes = 0
            };

            ReadEventOptions clone = options.Clone();

            Assert.That(clone, Is.Not.Null, "The clone should not be null.");

            Assert.That(clone.OwnerLevel, Is.EqualTo(options.OwnerLevel), "The owner level of the clone should match.");
            Assert.That(clone.TrackLastEnqueuedEventProperties, Is.EqualTo(options.TrackLastEnqueuedEventProperties), "The tracking of last event information of the clone should match.");
            Assert.That(clone.MaximumWaitTime, Is.EqualTo(options.MaximumWaitTime), "The maximum wait time of the clone should match.");
            Assert.That(clone.CacheEventCount, Is.EqualTo(options.CacheEventCount), "The event cache count of the clone should match.");
            Assert.That(clone.PrefetchCount, Is.EqualTo(options.PrefetchCount), "The prefetch count of the clone should match.");
            Assert.That(clone.PrefetchSizeInBytes, Is.EqualTo(options.PrefetchSizeInBytes), "The prefetch size of the clone should match.");
        }
コード例 #7
0
        //Read all events based on partitionId
        public async Task ConsumerReadEventPartitionEvent(string consumerGroup, string partitionId)
        {
            try
            {
                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
                EventHubConsumerClient eventConsumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName);

                ReadEventOptions readEventOptions = new ReadEventOptions()
                {
                    MaximumWaitTime = TimeSpan.FromSeconds(30)
                };

                await foreach (PartitionEvent partitionEvent in eventConsumer.ReadEventsFromPartitionAsync(partitionId, EventPosition.Latest, readEventOptions, cancellationSource.Token))
                {
                    Console.WriteLine("---Execution from ConsumerReadEventPartitionEvent method---");
                    Console.WriteLine("------");
                    if (partitionEvent.Data != null)
                    {
                        Console.WriteLine("Event Data recieved {0} ", Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray()));
                        if (partitionEvent.Data.Properties != null)
                        {
                            foreach (var keyValue in partitionEvent.Data.Properties)
                            {
                                Console.WriteLine("Event data key = {0}, Event data value = {1}", keyValue.Key, keyValue.Value);
                            }
                        }
                    }
                }

                await Task.CompletedTask;
            }
            catch (Exception exp)
            {
                Console.WriteLine("Error occruied {0}. Try again later", exp.Message);
            }
        }
コード例 #8
0
        public void MaximumWaitTimeAllowsNull()
        {
            var options = new ReadEventOptions();

            Assert.That(() => options.MaximumWaitTime = null, Throws.Nothing);
        }
コード例 #9
0
        public async Task ReadPartitionWaitTime()
        {
            await using var scope = await EventHubScope.CreateAsync(1);

            #region Snippet:EventHubs_Sample05_ReadPartitionWaitTime

            var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
            var eventHubName     = "<< NAME OF THE EVENT HUB >>";
            var consumerGroup    = EventHubConsumerClient.DefaultConsumerGroupName;
            /*@@*/
            /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
            /*@@*/ eventHubName     = scope.EventHubName;

            var consumer = new EventHubConsumerClient(
                consumerGroup,
                connectionString,
                eventHubName);

            try
            {
                using CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                string        firstPartition   = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
                EventPosition startingPosition = EventPosition.Earliest;

                int loopTicks    = 0;
                int maximumTicks = 10;

                var options = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromSeconds(1)
                };

                await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
                                   firstPartition,
                                   startingPosition,
                                   options))
                {
                    if (partitionEvent.Data != null)
                    {
                        string readFromPartition = partitionEvent.Partition.PartitionId;
                        byte[] eventBodyBytes    = partitionEvent.Data.EventBody.ToArray();

                        Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
                    }
                    else
                    {
                        Debug.WriteLine("Wait time elapsed; no event was available.");
                    }

                    loopTicks++;

                    if (loopTicks >= maximumTicks)
                    {
                        break;
                    }
                }
            }
            catch (TaskCanceledException)
            {
                // This is expected if the cancellation token is
                // signaled.
            }
            finally
            {
                await consumer.CloseAsync();
            }

            #endregion
        }
コード例 #10
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // In this example, our consumer will read from the latest position instead of the earliest.  As a result, it won't see events that
            // have previously been published.  Before we can publish the events and have them observed, we will need to ask the consumer
            // to perform a read operation in order for it to begin observing the Event Hub partitions.
            //
            // Each partition of an Event Hub represents potentially infinite stream of events.  When a consumer is reading, there is no definitive
            // point where it can assess that all events have been read and no more will be available.  As a result, when the consumer reaches the end of
            // the available events for a partition, it will continue to wait for new events to arrive so that it can surface them to be processed. During this
            // time, the iterator will block.
            //
            // In order to prevent the consumer from waiting forever for events, and blocking other code, there are two methods available for developers to
            // control this behavior.  First, signaling the cancellation token passed when reading will cause the consumer to stop waiting and end iteration
            // immediately.  This is desirable when you have decided that you are done reading and do not wish to continue.  It is not ideal, however, when
            // you would like control returned to your code momentarily to perform some action and then to continue reading.
            //
            // In that scenario, you may specify a maximum wait time which is applied to each iteration of the enumerator.  If that interval passes without an
            // event being available to read, the enumerator will emit an empty event in order to return control to the loop body.  This allows you to take action,
            // such as sending a heartbeat, emitting telemetry, or simply exiting the loop.
            //
            // For our loop, we'll specify a small wait time when we begin reading, which will allow control to return to our code so that we may publish
            // the events after we ensure the consumer is observing the partition.

            await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
            {
                bool             wereEventsPublished = false;
                int              eventBatchCount     = 0;
                List <EventData> receivedEvents      = new List <EventData>();

                // Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
                // an empty event, so that our code has the chance to run without indefinite blocking.

                ReadEventOptions readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromMilliseconds(150)
                };

                // As a preventative measure, we'll also specify that cancellation should occur after 2 minutes, so that we don't iterate indefinitely
                // in the event of a service error where the events we've published cannot be read.

                using CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromMinutes(2));

                // The reading of all events will default to the earliest events available in each partition; in order to begin reading at the
                // latest event, we'll need to specify that reading should not start at earliest.

                await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsAsync(startReadingAtEarliestEvent: false, readOptions, cancellationSource.Token))
                {
                    if (!wereEventsPublished)
                    {
                        await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
                        {
                            using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();

                            eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
                            eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")));

                            await producerClient.SendAsync(eventBatch);

                            wereEventsPublished = true;
                            eventBatchCount     = eventBatch.Count;

                            await Task.Delay(250);

                            Console.WriteLine("The event batch has been published.");
                        }

                        // Since we know that there was no event to observe for this iteration,
                        // we'll just skip to the next one.

                        continue;
                    }

                    // Because publishing and receiving events is asynchronous, the events that we published may not
                    // be immediately available for our consumer to see, so we'll have to guard against an empty event being sent
                    // if our wait time interval has elapsed before the consumer observed the events that we published.

                    if (currentEvent.Data != null)
                    {
                        receivedEvents.Add(currentEvent.Data);

                        if (receivedEvents.Count >= eventBatchCount)
                        {
                            break;
                        }
                    }
                }

                // Print out the events that we received; the body is an encoded string; we'll recover the message by reversing the encoding process.

                Console.WriteLine();

                foreach (EventData currentEvent in receivedEvents)
                {
                    string message = Encoding.UTF8.GetString(currentEvent.Body.ToArray());
                    Console.WriteLine($"\tEvent Message: \"{ message }\"");
                }
            }

            // At this point, our clients have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
コード例 #11
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace.  This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against</param>
        /// <param name="tenantId">The Azure Active Directory tenant that holds the service principal</param>
        /// <param name="clientId">The Azure Active Directory client identifier of the service principal</param>
        /// <param name="secret">The Azure Active Directory secret of the service principal</param>
        ///
        public async Task RunAsync(string fullyQualifiedNamespace,
                                   string eventHubName,
                                   string tenantId,
                                   string clientId,
                                   string secret)
        {
            // Service principal authentication is a means for applications to authenticate
            // against Azure Active Directory and consume Azure services. This is advantageous compared
            // to signing in using fully privileged users as it allows to enforce role-based authorization
            // from the portal.
            //
            // Service principal authentication differs from managed identity authentication also because the principal to be used
            // will be able to authenticate with the portal without the need for the code to run within the portal.
            // The authentication between the Event Hubs client and the portal is performed through OAuth 2.0.
            // (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals)

            // ClientSecretCredential allows performing service principal authentication passing
            // tenantId, clientId and clientSecret directly from the constructor.
            // (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow)

            var credentials = new ClientSecretCredential(tenantId, clientId, secret);

            // EventHubProducerClient takes ClientSecretCredential from its constructor and tries to issue a token from Azure Active Directory.

            await using (EventHubProducerClient client = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
            {
                // It will then use that token to authenticate on the portal and enquiry the Hub properties.

                Console.WriteLine($"Contacting the hub using the token issued from client credentials.");

                var properties = await client.GetEventHubPropertiesAsync();

                Console.WriteLine($"Event Hub \"{ properties.Name }\" reached successfully.");
            }

            // The instances of "EventHubProducerClient" and "EventHubProducerClient" will be created
            // passing in "ClientSecretCredential" instead of the connection string. In this way, two things will happen:
            //
            // 1. An OAuth 2.0 token will be created by authenticating against Azure Active Directory using the tenant, client and the secret passed in
            // 2. Role-based authorization will be performed and the "Azure Event Hubs Data Owner" role will be needed to produce and consume events
            //
            // (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-definitions)
            // (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-event-hubs-data-owner)

            await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credentials))
            {
                string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();

                PartitionEvent receivedEvent;

                ReadEventOptions readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromMilliseconds(150)
                };

                Stopwatch watch = Stopwatch.StartNew();
                bool      wereEventsPublished = false;

                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
                {
                    if (!wereEventsPublished)
                    {
                        await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
                        {
                            using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });

                            eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));

                            await producerClient.SendAsync(eventBatch);

                            wereEventsPublished = true;

                            Console.WriteLine("The event batch has been published.");
                        }
                    }

                    if (currentEvent.Data != null)
                    {
                        receivedEvent = currentEvent;
                        watch.Stop();
                        break;
                    }
                }

                Console.WriteLine();
                Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:");

                string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
                Console.WriteLine($"\tMessage: \"{ message }\"");
            }

            Console.WriteLine();
        }
コード例 #12
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // In this example, we'll make use of multiple clients in order to publish an event that we will then read back and use as the starting point
            // for reading events in the partition.  Our initial consumer will begin watching for new events published to the first partition in our Event
            // Hub.  Before we can publish events and have them observed, we will need to ask the consumer to perform an operation for it to begin observing
            // the partition.
            //
            // Each event that a consumer reads will have attributes set that describe the event's location in the partition, such as its offset, sequence
            // number, and the date/time that it was enqueued.  These attributes can be used to create a new consumer that begins consuming at a known position.
            //
            // With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed,
            // and to manage where in the partition the consumer begins reading events.  This is done by using the position information to track
            // state, commonly known as "creating a checkpoint."
            //
            // The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the
            // consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that
            // begins reading at the position where it left off.
            //
            // It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint.  A well-designed
            // consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues.
            // Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart,
            // there is a small possibility that the service will return an event multiple times.
            //
            // To demonstrate, we will publish a batch of events to be read by an initial consumer.  The third event that is read will be captured
            // and another consumer will use its attributes to start reading the event that follows, reading the set of events that we published skipping over
            // the first three.

            string    firstPartition;
            EventData thirdEvent;

            int eventBatchSize = 50;

            await using (var initialConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
            {
                // We will start by using the consumer client inspect the Event Hub and select the first partition to operate against.

                firstPartition = (await initialConsumerClient.GetPartitionIdsAsync()).First();

                // Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
                // an empty event, so that our code has the chance to run without indefinite blocking.

                ReadEventOptions readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromMilliseconds(150)
                };

                // As a preventative measure, we'll also specify that cancellation should occur after 30 seconds, so that we don't iterate indefinitely
                // in the event of a service error where the events we've published cannot be read.

                using CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                List <EventData> receivedEvents      = new List <EventData>();
                bool             wereEventsPublished = false;

                await foreach (PartitionEvent currentEvent in initialConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
                {
                    if (!wereEventsPublished)
                    {
                        await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
                        {
                            // When we publish the event batch, we'll specify the partition to ensure that our consumer and producer clients are
                            // operating against the same partition.

                            using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });

                            for (int index = 0; index < eventBatchSize; ++index)
                            {
                                eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")));
                            }

                            await producerClient.SendAsync(eventBatch);

                            wereEventsPublished = true;

                            await Task.Delay(250);

                            Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
                        }

                        // Since we know that there was no event to observe for this iteration,
                        // we'll just skip to the next one.

                        continue;
                    }

                    // Because publishing and reading events is asynchronous, the events that we published may not
                    // be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as
                    // punctuation if our actual event is not available within the waiting time period.

                    if (currentEvent.Data != null)
                    {
                        receivedEvents.Add(currentEvent.Data);

                        if (receivedEvents.Count >= eventBatchSize)
                        {
                            break;
                        }
                    }
                }

                // Print out the events that we read, which will be the entire set that
                // we had published.

                Console.WriteLine();
                Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published.  { eventBatchSize } were expected.");

                foreach (EventData eventData in receivedEvents)
                {
                    // The body of our event was an encoded string; we'll recover the
                    // message by reversing the encoding process.

                    string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                    Console.WriteLine($"\tEvent Message: \"{ message }\"");
                }

                // Remember the third event that was consumed.

                thirdEvent = receivedEvents[2];
            }

            // At this point, our initial consumer client has passed its "using" scope and has been safely disposed of.
            //
            // Create a new consumer for the partition, specifying the sequence number of the third event as the location to begin reading. Because sequence numbers are non-inclusive
            // by default, the consumer will read the next available event following that sequence number, allowing it to read the set of published events beginning with the fourth one.
            //
            // Because our second consumer will begin watching the partition at a specific event, there is no need to ask for an initial operation to set our place; when
            // we begin iterating, the consumer will locate the proper place in the partition to read from.

            await using (var newConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
            {
                // We will consume the events using the new consumer until all of the published events have been received.

                using CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                List <EventData> receivedEvents = new List <EventData>();
                int           expectedCount     = (eventBatchSize - 3);
                EventPosition startingPosition  = EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value);

                await foreach (PartitionEvent currentEvent in newConsumerClient.ReadEventsFromPartitionAsync(firstPartition, startingPosition, cancellationSource.Token))
                {
                    receivedEvents.Add(currentEvent.Data);

                    if (receivedEvents.Count >= expectedCount)
                    {
                        break;
                    }
                }

                // Print out the events that we received.

                Console.WriteLine();
                Console.WriteLine();
                Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published.  { expectedCount } were expected.");

                foreach (EventData eventData in receivedEvents)
                {
                    // The body of our event was an encoded string; we'll recover the
                    // message by reversing the encoding process.

                    string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
                    Console.WriteLine($"\tEvent Message: \"{ message }\"");
                }
            }

            // At this point, our clients and connection have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
コード例 #13
0
        /// <summary>
        ///   Starts running a task responsible for receiving and processing events in the context of a specified partition.
        /// </summary>
        ///
        /// <param name="partitionId">The identifier of the Event Hub partition the task is associated with.  Events will be read only from this partition.</param>
        /// <param name="startingPosition">The position within the partition where the task should begin reading events.</param>
        /// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param>
        /// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param>
        /// <param name="trackLastEnqueuedEventProperties">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param>
        /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
        ///
        /// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns>
        ///
        protected virtual Task RunPartitionProcessingAsync(string partitionId,
                                                           EventPosition startingPosition,
                                                           TimeSpan?maximumReceiveWaitTime,
                                                           EventHubsRetryOptions retryOptions,
                                                           bool trackLastEnqueuedEventProperties,
                                                           CancellationToken cancellationToken = default)
        {
            // TODO: should the retry options used here be the same for the abstract RetryPolicy property?

            Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId));
            Argument.AssertNotNull(retryOptions, nameof(retryOptions));

            return(Task.Run(async() =>
            {
                // TODO: should we double check if a previous run already exists and close it?  We have a race condition.  Maybe we should throw in case another task exists.

                var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                var taskCancellationToken = cancellationSource.Token;

                ActivePartitionProcessorTokenSources[partitionId] = cancellationSource;

                // Context is set to default if operation fails.  This shouldn't fail unless the user tries processing
                // a partition they don't own.

                PartitionContexts.TryGetValue(partitionId, out var context);

                var clientOptions = new EventHubConsumerClientOptions
                {
                    RetryOptions = retryOptions
                };

                var readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = maximumReceiveWaitTime,
                    TrackLastEnqueuedEventProperties = trackLastEnqueuedEventProperties
                };

                await using var connection = CreateConnection();

                await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, clientOptions))
                {
                    await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, readOptions, taskCancellationToken))
                    {
                        using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
                        diagnosticScope.AddAttribute("kind", "server");

                        if (diagnosticScope.IsEnabled &&
                            partitionEvent.Data != null &&
                            EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId))
                        {
                            diagnosticScope.AddLink(diagnosticId);
                        }

                        diagnosticScope.Start();

                        try
                        {
                            await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false);
                        }
                        catch (Exception eventProcessingException)
                        {
                            diagnosticScope.Failed(eventProcessingException);
                            throw;
                        }
                    }
                }
            }));
        }
コード例 #14
0
        public async Task Receive(string connectionString, CancellationToken cancellationToken)
        {
            // In here, our consumer will read from the latest position instead of the earliest.  As a result, it won't see events that
            // have previously been published.
            //
            // Each partition of an Event Hub represents a potentially infinite stream of events.  When a consumer is reading, there is no definitive
            // point where it can assess that all events have been read and no more will be available.  As a result, when the consumer reaches the end of
            // the available events for a partition, it will continue to wait for new events to arrive so that it can surface them to be processed. During this
            // time, the iterator will block.
            //
            // In order to prevent the consumer from waiting forever for events, and blocking other code, there is a method available for developers to
            // control this behavior. It's signaling the cancellation token passed when reading will cause the consumer to stop waiting and end iteration
            // immediately.  This is desirable when you have decided that you are done reading and do not wish to continue.

            await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString))
            {
                Console.WriteLine("The application will now start to listen for incoming message.");

                // Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
                // an empty event, so that our code has the chance to run without indefinite blocking.

                ReadEventOptions readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromMilliseconds(500)
                };

                using (var csvOutput = File.CreateText("./result.csv"))
                {
                    csvOutput.WriteLine("EventCount,BatchCount,BatchFrom,BatchTo,MinLatency,MaxLatency,AvgLatency");
                    try
                    {
                        // The client is safe and intended to be long-lived.

                        await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsAsync(readOptions, cancellationToken))
                        {
                            var eventData = currentEvent.Data;

                            // Because publishing and receiving events is asynchronous, the events that published may not
                            // be immediately available for our consumer to see, so we'll have to guard against an empty event being sent
                            if (eventData == null)
                            {
                                continue;
                            }

                            int eventCount   = 0;
                            var listTimeSpan = new List <TimeSpan>();
                            var listDateTime = new List <DateTimeOffset>();

                            var eventBody = Encoding.UTF8.GetString(eventData.Body.ToArray()).Split('\n');

                            foreach (var bodyInfo in eventBody)
                            {
                                try
                                {
                                    var message = JsonSerializer.Deserialize <Dictionary <string, object> >(bodyInfo);
                                    eventCount++;

                                    var timeCreated      = DateTime.Parse(message["createdAt"].ToString());
                                    var timeIn           = DateTime.Parse(message["EventEnqueuedUtcTime"].ToString());
                                    var timeProcessed    = DateTime.Parse(message["EventProcessedUtcTime"].ToString());
                                    var timeAsaProcessed = DateTime.Parse(message["ASAProcessedUtcTime"].ToString());
                                    var timeOut          = eventData.EnqueuedTime.UtcDateTime.ToLocalTime();

                                    listDateTime.Add(timeIn);
                                    var elapsed = timeOut - timeIn;
                                    listTimeSpan.Add(elapsed);
                                }
                                catch (Exception ex)
                                {
                                    Console.WriteLine("Error while parsing event body.");
                                    Console.WriteLine("Error:" + ex.Message);
                                    Console.WriteLine("Message:" + bodyInfo);
                                }
                            }

                            var batchFrom  = listDateTime.Min();
                            var batchTo    = listDateTime.Min();
                            var minLatency = listTimeSpan.Min().TotalMilliseconds;
                            var maxLatency = listTimeSpan.Max().TotalMilliseconds;
                            var avgLatency = Math.Round(listTimeSpan.Average(ts => ts.TotalMilliseconds), 0);

                            Console.Write($"Received {eventCount} events.");
                            Console.Write($"\tBatch (From/To): {batchFrom.ToString("HH:mm:ss.ffffff")}/{batchTo.ToString("HH:mm:ss.ffffff")}");
                            Console.Write($"\tElapsed msec (Min/Max/Avg): {minLatency}/{maxLatency}/{avgLatency}");
                            Console.WriteLine();

                            csvOutput.WriteLine($"{eventCount},{eventCount},{batchFrom.ToString("o")},{batchTo.ToString("o")},{minLatency},{maxLatency},{avgLatency}");
                        }
                    }
                    catch (TaskCanceledException)
                    {
                        // This is okay because the task was cancelled.
                    }
                }
            }
            // At this point, our clients have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.
        }
コード例 #15
0
        static async Task <int> Main(string[] args)
        {
            var app = new CommandLineApplication();

            app.FullName    = "Event Hubs Consumer";
            app.Description = "Event Hubs Consumer Sample";
            app.HelpOption("-h|-?|--help");
            var hubName          = String.Empty;
            var connectionString = app.Option("-cs", "Specify ConnectionString", CommandOptionType.SingleValue, true);
            var group            = app.Option("-cg", "Specify Consumer Group", CommandOptionType.SingleValue);
            var hub       = app.Option("-hub", "Specify Event Hub", CommandOptionType.SingleValue);
            var allEvents = app.Option("-all", "Read all events", CommandOptionType.NoValue);

            app.Execute(args);

            if (!connectionString.HasValue() || string.IsNullOrEmpty(connectionString.Value()))
            {
                ShowHelp();
                return(0);
            }

            var csValue      = connectionString.Value().Replace("\"", "").Trim();
            var csValueItems = csValue.Split(";");

            foreach (var csValueItem in csValueItems)
            {
                var valuePair = csValueItem.Split("=");

                if (valuePair[0].ToLower() == "entitypath")
                {
                    hubName = valuePair[1];
                    break;
                }
            }

            if (hubName == String.Empty)
            {
                if (!hub.HasValue() || string.IsNullOrEmpty(hub.Value()))
                {
                    ShowHelp();
                    return(0);
                }
                else
                {
                    hubName = hub.Value().Replace("\"", "").Trim();
                }
            }

            //
            // https://docs.microsoft.com/en-us/dotnet/api/azure.messaging.eventhubs.consumer.eventhubconsumerclient?view=azure-dotnet
            // Select Consumer Group
            //
            string consumerGroup;

            if (group.HasValue() && !string.IsNullOrEmpty(group.Value()))
            {
                consumerGroup = group.Value().Replace("\"", "").Trim();
            }
            else
            {
                consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
            }

            Console.WriteLine("Connection String : {0}....", connectionString.Value().Replace("\"", "").Trim().Substring(0, 50));
            Console.WriteLine($"Event Hub         : {hubName}");
            Console.WriteLine($"Consumer Group    : {consumerGroup}");
            Console.WriteLine($"Read all events   : {allEvents.HasValue()}");
            Console.WriteLine($"Timeout           : {readTimetoutSeconds} seconds");

            EventHubConsumerClient consumer = new EventHubConsumerClient(consumerGroup, connectionString.Value().Replace("\"", "").Trim(), hubName);

            var part = await consumer.GetPartitionIdsAsync();

            var part_prop = await consumer.GetPartitionPropertiesAsync(part[0]);

            var hub_prop = await consumer.GetEventHubPropertiesAsync();

            using CancellationTokenSource cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromSeconds(readTimetoutSeconds));

            //
            // https://docs.microsoft.com/en-us/dotnet/api/azure.messaging.eventhubs.consumer.eventhubconsumerclient.readeventsasync?view=azure-dotnet
            //
            // Timeout 10 Min
            //
            ReadEventOptions readOptions = new ReadEventOptions
            {
                MaximumWaitTime = TimeSpan.FromSeconds(600)
            };

            try
            {
                Console.WriteLine($"Read Start        : {DateTime.Now.ToString()}");
                Console.WriteLine("Waiting for events.  CTRL+C to exit");
                //
                // If this is IoT Hub Eventhub Compatible Endpoint, device id is in ev.Data.SystemProperties["iothub-connection-device-id"]
                //
                await foreach (PartitionEvent ev in consumer.ReadEventsAsync(startReadingAtEarliestEvent: allEvents.HasValue(), readOptions, cancellationSource.Token))
                {
                    Console.WriteLine("Enqueue at {0:yyyy/MM/dd H:mm:ss:fff} | Seq # {1:D4} | Partition {2} | Offset : {3:D6} | {4}",
                                      ev.Data.EnqueuedTime,
                                      ev.Data.SequenceNumber,
                                      ev.Partition.PartitionId,
                                      ev.Data.Offset,
                                      Encoding.UTF8.GetString(ev.Data.Body.ToArray())
                                      );
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Exception         : {ex.Message}");
            }
            Console.WriteLine($"Read Exit         : {DateTime.Now.ToString()}");

            return(1);
        }
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // An Event Hub consumer is associated with a specific Event Hub and consumer group.  The consumer group is
            // a label that identifies one or more consumers as a set.  Often, consumer groups are named after the responsibility
            // of the consumer in an application, such as "Telemetry" or "OrderProcessing".  When an Event Hub is created, a default
            // consumer group is created with it, called "$Default."
            //
            // Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
            // consumers and are not removed from the partition when a consumer reads them.  This allows for one or more consumers to read and
            // process events from the partition at different speeds and beginning with different events without interfering with
            // one another.
            //
            // When events are published, they will continue to exist in the partition and be available for consuming until they
            // reach an age where they are older than the retention period.
            // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
            //
            // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
            // would like to begin reading events.  For example, this may be starting from the very beginning of the stream, at an
            // offset from the beginning, the next event available after a specific point in time, or at a specific event.
            //
            // In this example, we will create our consumer client using the default consumer group that is created with an Event Hub.
            // Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it.

            await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
            {
                // We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being
                // published and read from the same partition.

                string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();

                // Because our consumer is reading from the latest position, it won't see events that have previously
                // been published.  Before we can publish the events and have them observed, we will need to ask the consumer
                // to perform an operation, because it opens its connection only when it needs to.
                //
                // When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed,
                // whether or not an event is available in the partition.  If no event was available a null value will be emitted instead.
                // This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event
                // processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue
                // if events are not arriving.
                //
                // We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when
                // no event is available.  For the first call, we'll publish so that we can receive them.
                //
                // For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more
                // event than we published, which is expected to be a null value triggered by exceeding the wait time.
                //
                // To be sure that we do not block forever waiting on an event that is not published, we will request cancellation after a
                // fairly long interval.

                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

                ReadEventOptions readOptions = new ReadEventOptions
                {
                    MaximumWaitTime = TimeSpan.FromMilliseconds(250)
                };

                bool             wereEventsPublished = false;
                int              eventBatchCount     = 0;
                List <EventData> receivedEvents      = new List <EventData>();

                Stopwatch watch = Stopwatch.StartNew();

                await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
                {
                    if (!wereEventsPublished)
                    {
                        await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
                        {
                            using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });

                            eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
                            eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")));

                            await producerClient.SendAsync(eventBatch);

                            wereEventsPublished = true;
                            eventBatchCount     = eventBatch.Count;

                            await Task.Delay(250);

                            Console.WriteLine("The event batch has been published.");
                        }
                    }
                    else
                    {
                        receivedEvents.Add(currentEvent.Data);

                        if (receivedEvents.Count > eventBatchCount)
                        {
                            watch.Stop();
                            break;
                        }
                    }
                }

                // Print out the events that we received.

                Console.WriteLine();
                Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");

                foreach (EventData eventData in receivedEvents)
                {
                    // The body of our event was an encoded string; we'll recover the
                    // message by reversing the encoding process.

                    string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray());
                    Console.WriteLine($"\tMessage: \"{ message }\"");
                }
            }

            // At this point, our clients have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }