private static async Task SendAndReceiveEvents() { var eventBatch = new[] { new EventData(Encoding.UTF8.GetBytes("First event data")), new EventData(Encoding.UTF8.GetBytes("Second event data")), new EventData(Encoding.UTF8.GetBytes("Third event data")) }; var index = 0; var receivedEvents = new List <EventData>(); //Before sending any event, start the receiver await receiver.ReceiveAsync(1, TimeSpan.Zero); Console.Write("Ready to send a batch of " + eventBatch.Count().ToString() + " events... "); await sender.SendAsync(eventBatch); Console.Write("Sent\n"); Console.Write("Receiving events... "); while ((receivedEvents.Count < eventBatch.Length) && (++index < 3)) { receivedEvents.AddRange(await receiver.ReceiveAsync(eventBatch.Length + 10, TimeSpan.FromMilliseconds(25))); } if (receivedEvents.Count == 0) { throw new Exception(String.Format("Error, No events received.")); } Console.Write(receivedEvents.Count() + " events received.\n"); if (receivedEvents.Count() < eventBatch.Count()) { throw new Exception(String.Format($"Error, expecting {eventBatch.Count()} events, but only got {receivedEvents.Count().ToString()}.")); } Console.WriteLine("done"); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is // a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility // of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default // consumer group is created with it, called "$Default." // // Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all // consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and // process events from the partition at different speeds and beginning with different events without interfering with // one another. // // When events are published, they will continue to exist in the partition and be available for consuming until they // reach an age where they are older than the retention period. // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events) // // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it // would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an // offset from the beginning, the next event available after a specific point in time, or at a specific event. // We will start by creating a client to inspect the Event Hub and select a partition to operate against to ensure that // events are being published and read from the same partition. string firstPartition; await using (var inspectionClient = new EventHubProducerClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier // of the first. firstPartition = (await inspectionClient.GetPartitionIdsAsync()).First(); } // In this example, we will create our consumer client for the first partition in the Event Hub, using the default consumer group // that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.Latest, connectionString, eventHubName)) await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName, new EventHubProducerClientOptions { PartitionId = firstPartition })) { // Because our consumer is reading from the latest position, it won't see events that have previously // been published. Before we can publish the events, we will need to ask the consumer to perform an operation, // because it opens its connection only when it needs to. The first receive that we ask of it will not see // any events, but will allow the consumer to start watching the partition. // // Because the maximum wait time is specified as zero, this call will return immediately and will not // have consumed any events. await consumerClient.ReceiveAsync(1, TimeSpan.Zero); // Now that the consumer is watching the partition, let's publish the event that we would like to // receive. await producerClient.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!"))); Console.WriteLine("The event batch has been published."); // Because publishing and receiving events is asynchronous, the events that we published may not // be immediately available for our consumer to see. // // We will iterate over the available events in the partition, which should be just the event that we published. Because // we're expecting only the one event, we will exit the loop when we receive it. To be sure that we do not block forever // waiting on an event that is not published, we will specify a fairly long time to wait and then cancel waiting. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); EventData receivedEvent = null; Stopwatch watch = Stopwatch.StartNew(); await foreach (EventData currentEvent in consumerClient.SubscribeToEvents(cancellationSource.Token)) { receivedEvent = currentEvent; watch.Stop(); break; } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:"); // The body of our event was an encoded string; we'll recover the message by reversing the encoding process. string message = (receivedEvent == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } // At this point, our clients have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { // Send some events. EventData lastEvent; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); var partitionId = (await connection.GetPartitionIdsAsync(DefaultRetryPolicy)).First(); await using (var producer = new EventHubProducerClient(connection)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connectionString)) { // Send a few events. We are only interested in the last one of them. var dummyEventsCount = 10; for (int i = 0; i < dummyEventsCount; i++) { await producer.SendAsync(dummyEvent); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); lastEvent = receivedEvents.Last(); } // Create a partition manager so we can retrieve the created checkpoint from it. var partitionManager = new InMemoryPartitionManager(); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connection, partitionManager, onProcessEvents: (partitionContext, events) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Any()) { partitionContext.UpdateCheckpointAsync(eventsList.Last()); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName); Assert.That(ownershipEnumerable, Is.Not.Null); Assert.That(ownershipEnumerable.Count, Is.EqualTo(1)); PartitionOwnership ownership = ownershipEnumerable.Single(); Assert.That(ownership.Offset.HasValue, Is.True); Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset)); Assert.That(ownership.SequenceNumber.HasValue, Is.True); Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber)); } } }
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var connection = new EventHubConnection(connectionString)) { int receivedEventsCount = 0; // Send some events. var expectedEventsCount = 20; var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy.")); long?checkpointedSequenceNumber = default; var partitionId = (await connection.GetPartitionIdsAsync(DefaultRetryPolicy)).First(); await using (var producer = new EventHubProducerClient(connectionString)) await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connection)) { // Send a few dummy events. We are not expecting to receive these. var dummyEventsCount = 30; for (int i = 0; i < dummyEventsCount; i++) { await producer.SendAsync(dummyEvent); } // Receive the events; because there is some non-determinism in the messaging flow, the // sent events may not be immediately available. Allow for a small number of attempts to receive, in order // to account for availability delays. var receivedEvents = new List <EventData>(); var index = 0; while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit)) { receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25))); } Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount)); checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber; // Send the events we expect to receive. for (int i = 0; i < expectedEventsCount; i++) { await producer.SendAsync(dummyEvent); } } // Create a partition manager and add an ownership with a checkpoint in it. var partitionManager = new InMemoryPartitionManager(); await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>() { new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId, sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow) }); // Create the event processor manager to manage our event processors. var eventProcessorManager = new EventProcessorManager ( EventHubConsumerClient.DefaultConsumerGroupName, connection, partitionManager, onProcessEvents: (partitionContext, events) => { // Make it a list so we can safely enumerate it. var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>()); if (eventsList.Count > 0) { Interlocked.Add(ref receivedEventsCount, eventsList.Count); } } ); eventProcessorManager.AddEventProcessors(1); // Start the event processors. await eventProcessorManager.StartAllAsync(); // Make sure the event processors have enough time to stabilize and receive events. await eventProcessorManager.WaitStabilization(); // Stop the event processors. await eventProcessorManager.StopAllAsync(); // Validate results. Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount)); } } }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // We will start by creating a client to inspect the Event Hub and select a partition to operate against to ensure that // events are being published and read from the same partition. string firstPartition; await using (var inspectionClient = new EventHubProducerClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier // of the first. firstPartition = (await inspectionClient.GetPartitionIdsAsync()).First(); } // In this example, we will create our consumer client for the first partition in the Event Hub, using the default consumer group // that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.Latest, connectionString, eventHubName)) await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName, new EventHubProducerClientOptions { PartitionId = firstPartition })) { // Because our consumer client is reading from the latest position, it won't see events that have previously // been published. Before we can publish the events, we will need to ask the consumer to perform an operation, // because it opens its connection only when it needs to. The first receive that we ask of it will not see // any events, but will allow the consumer to start watching the partition. // // Because the maximum wait time is specified as zero, this call will return immediately and will not // have consumed any events. await consumerClient.ReceiveAsync(1, TimeSpan.Zero); // Now that the consumer is watching the partition, let's publish a fair sized batch of events // we would like for it to consume. int eventBatchSize = 100; EventData[] eventBatch = new EventData[eventBatchSize]; for (int index = 0; index < eventBatchSize; ++index) { eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")); } await producerClient.SendAsync(eventBatch); Console.WriteLine($"The event batch with { eventBatchSize } events has been published."); // To allow for throughput in our application, we will process the published events by consuming them in // small batches with a small wait time. This will allow us to receive and process more quickly than blocking // to wait on a larger batch size. // // The values that are used in this example are intended just for illustration and not intended as guidance for a recommended // set of defaults. In a real-world application, determining the right balance of batch size and wait time to achieve // the desired performance will often vary depending on the application and event data being consumed. It is an area // where some experimentation in the application context may prove helpful. // // Our example will attempt to read about 1/5 of the batch at a time, which should result in the need for 5 batches to // be consumed. Because publishing and receiving events is asynchronous, the events that we published may not be // immediately available for our consumer to see. To compensate, we will allow for a small number of extra attempts beyond // the expected 5 to be sure that we don't stop reading before we receive all of our events. var receivedEvents = new List <EventData>(); int consumeBatchSize = (int)Math.Floor(eventBatchSize / 5.0f); int maximumAttempts = 15; int attempts = 0; while ((receivedEvents.Count < eventBatchSize) && (++attempts < maximumAttempts)) { // Each receive, we ask for the maximum amount of events that we would like in the batch and // specify the maximum amount of time that we would like to wait for them. // // The batch of events will be returned to us when either we have read our maximum number of events // or when the time that we asked to wait has elapsed. This means that a batch we receive may have // between zero and the maximum we asked for, depending on what was available in the partition. // // For this attempt, we will ask to receive our computed batch size (1/5 of published events) and wait, at most, // 25 milliseconds to receive them. We'll then process them and if we haven't gotten all that we published, we'll // make another attempt until either we have them or we have tried for our maximum number of attempts. IEnumerable <EventData> receivedBatch = await consumerClient.ReceiveAsync(consumeBatchSize, TimeSpan.FromMilliseconds(25)); receivedEvents.AddRange(receivedBatch); } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"Events Consumed: { receivedEvents.Count }"); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } // At this point, our clients have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // We will start by creating a client to inspect the Event Hub and select a partition to operate against to ensure that // events are being published and read from the same partition. string firstPartition; await using (var inspectionClient = new EventHubProducerClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier // of the first. firstPartition = (await inspectionClient.GetPartitionIdsAsync()).First(); } // In this example, we will make use of multiple clients. Because clients are typically responsible for managing their own connection to the // Event Hubs service, each will implicitly create their own connection. In this example, we will create a connection that may be shared amongst // clients in order to illustrate connection sharing. Because we are explicitly creating the connection, we assume responsibility for managing its // lifespan and ensuring that it is properly closed or disposed when we are done using it. await using (var eventHubConnection = new EventHubConnection(connectionString, eventHubName)) await using (var producerClient = new EventHubProducerClient(eventHubConnection, new EventHubProducerClientOptions { PartitionId = firstPartition })) { // Our initial consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. // // Each event that the initial consumer reads will have attributes set that describe the event's place in the // partition, such as its offset, sequence number, and the date/time that it was enqueued. These attributes can be // used to create a new consumer that begins consuming at a known position. // // With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed, // and to manage where in the partition the consumer begins reading events. This is done by using the position information to track // state, commonly known as "creating a checkpoint." // // The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the // consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that // begins reading at the position where it left off. // // It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed // consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues. // Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart, // there is a small possibility that the service will return an event multiple times. // // In this example, we will publish a batch of events to be received with an initial consumer. The third event that is consumed will be captured // and another consumer will use its attributes to start reading the event that follows, consuming the same set of events that our initial consumer // read, skipping over the first three. int eventBatchSize = 50; EventData thirdEvent; await using (var initialConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.Latest, eventHubConnection)) { // The first receive that we ask of it will not see any events, but allows the consumer to start watching the partition. Because // the maximum wait time is specified as zero, this call will return immediately and will not have consumed any events. await initialConsumerClient.ReceiveAsync(1, TimeSpan.Zero); // Now that the consumer is watching the partition, let's publish a batch of events. EventData[] eventBatch = new EventData[eventBatchSize]; for (int index = 0; index < eventBatchSize; ++index) { eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")); } await producerClient.SendAsync(eventBatch); Console.WriteLine($"The event batch with { eventBatchSize } events has been published."); // We will consume the events until all of the published events have been received. var receivedEvents = new List <EventData>(); CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); await foreach (EventData currentEvent in initialConsumerClient.SubscribeToEvents(cancellationSource.Token)) { receivedEvents.Add(currentEvent); if (receivedEvents.Count >= eventBatchSize) { break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } // Remember the third event that was consumed. thirdEvent = receivedEvents[2]; } // At this point, our initial consumer client has passed its "using" scope and has been safely disposed of. // // Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available // sequence number, allowing it to read the set of published events beginning with the fourth one. await using (var newConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value), eventHubConnection)) { // We will consume the events using the new consumer until all of the published events have been received. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); int expectedCount = (eventBatchSize - 3); var receivedEvents = new List <EventData>(); await foreach (EventData currentEvent in newConsumerClient.SubscribeToEvents(cancellationSource.Token)) { receivedEvents.Add(currentEvent); if (receivedEvents.Count >= expectedCount) { break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine(); Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } } // At this point, our clients and connection have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }