/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // An Event Hub consumer is associated with a specific Event Hub partition and a consumer group. The consumer group is // a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility // of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default // consumer group is created with it, called "$Default." // // Each consumer has a unique view of the events in the partition, meaning that events are available to all consumers // and are not removed from the partition when a consumer reads them. This allows for different consumers to read and // process events from the partition at different speeds and beginning with different events without interfering with // one another. // // When events are published, they will continue to exist in the partition and be available for consuming until they // reach an age where they are older than the retention period. // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events) // // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it // would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an // offset from the beginning, the next event available after a specific point in time, or at a specific event. // We will start by creating a client using its default set of options. await using (var client = new EventHubClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier // of the first. string firstPartition = (await client.GetPartitionIdsAsync()).First(); // In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group // that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest)) await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions { PartitionId = firstPartition })) { // Because our consumer is reading from the latest position, it won't see events that have previously // been published. Before we can publish the events, we will need to ask the consumer to perform an operation, // because it opens its connection only when it needs to. The first receive that we ask of it will not see // any events, but will allow the consumer to start watching the partition. // // Because the maximum wait time is specified as zero, this call will return immediately and will not // have consumed any events. await consumer.ReceiveAsync(1, TimeSpan.Zero); // Now that the consumer is watching the partition, let's publish the events that we would like to // receive. EventData[] eventsToPublish = new EventData[] { new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")), new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")) }; await producer.SendAsync(eventsToPublish); Console.WriteLine("The event batch has been published."); // Because publishing and receiving events is asynchronous, the events that we published may not be immediately // available for our consumer to see. We will iterate over the available events in the partition, which should be // just the events that we published. // // When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed, // whether or not an event is available in the partition. If no event was available a null value will be emitted instead. // This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event // processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue // if events are not arriving. // // For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more // event than we published, which is expected to be a null value triggered by exceeding the wait time. // // To be sure that we do not end up in an infinite loop, we will specify a fairly long time to allow processing to complete // then cancel. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(250); List <EventData> receivedEvents = new List <EventData>(); Stopwatch watch = Stopwatch.StartNew(); await foreach (EventData currentEvent in consumer.SubscribeToEvents(maximumWaitTime, cancellationSource.Token)) { receivedEvents.Add(currentEvent); if (receivedEvents.Count > eventsToPublish.Length) { watch.Stop(); break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:"); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } } // At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // We will start by creating a client using its default set of options. await using (var client = new EventHubClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier of the first. string firstPartition = (await client.GetPartitionIdsAsync()).First(); // In this example, we will make use of multiple consumers for the first partition in the Event Hub, using the default consumer group // that is created with an Event Hub. Our initial consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions { PartitionId = firstPartition })) { // Each event that the initial consumer reads will have attributes set that describe the event's place in the // partition, such as it's offset, sequence number, and the date/time that it was enqueued. These attributes can be // used to create a new consumer that begins consuming at a known position. // // With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed, // and to manage where in the partition the consumer begins reading events. This is done by using the position information to track // state, commonly known as "creating a checkpoint." // // The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the // consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that // begins reading at the position where it left off. // // It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed // consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues. // Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart, // there is a small possibility that the service will return an event multiple times. // // In this example, we will publish a batch of events to be received with an initial consumer. The third event that is consumed will be captured // and another consumer will use it's attributes to start reading the event that follows, consuming the same set of events that our initial consumer // read, skipping over the first three. int eventBatchSize = 50; EventData thirdEvent; await using (EventHubConsumer initialConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest)) { // The first receive that we ask of it will not see any events, but allows the consumer to start watching the partition. Because // the maximum wait time is specified as zero, this call will return immediately and will not have consumed any events. await initialConsumer.ReceiveAsync(1, TimeSpan.Zero); // Now that the consumer is watching the partition, let's publish a batch of events. EventData[] eventBatch = new EventData[eventBatchSize]; for (int index = 0; index < eventBatchSize; ++index) { eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")); } await producer.SendAsync(eventBatch); Console.WriteLine($"The event batch with { eventBatchSize } events has been published."); // We will consume the events until all of the published events have been received. var receivedEvents = new List <EventData>(); CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); await foreach (EventData currentEvent in initialConsumer.SubscribeToEvents(cancellationSource.Token)) { receivedEvents.Add(currentEvent); if (receivedEvents.Count >= eventBatchSize) { break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } // Remember the third event that was consumed. thirdEvent = receivedEvents[2]; } // At this point, our initial consumer has passed its "using" scope and has been safely disposed of. // // Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available // sequence number, allowing it to read the set of published events beginning with the fourth one. await using (EventHubConsumer newConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value))) { // We will consume the events using the new consumer until all of the published events have been received. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); int expectedCount = (eventBatchSize - 3); var receivedEvents = new List <EventData>(); await foreach (EventData currentEvent in newConsumer.SubscribeToEvents(cancellationSource.Token)) { receivedEvents.Add(currentEvent); if (receivedEvents.Count >= expectedCount) { break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine(); Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } } } // At this point, our client, all consumers, and producer have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }