// Reads events from the requested partition as an asynchronous // enumerable, allowing events to be iterated as they become available // on the partition, waiting as necessary should there be no events available. // As you can see, this method is supplied with an argument that defines // the target partition. Recall that for the default configuration where // 4 partitions are specified, this method is called 4 times, each // running asynchronously and in parallel, one for each partition. private static async Task ReceiveMessagesFromDeviceAsync(string partition) { EventPosition startingPosition = EventPosition.Earliest; // Reads events from the requested partition as an asynchronous // enumerable, allowing events to be iterated as they become available // on the partition, waiting as necessary should there be no events // available. await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync( partition, startingPosition)) { string readFromPartition = partitionEvent.Partition.PartitionId; // Each event data body is converted from BinaryData to a byte // array, and from there, to a string and written to the // console for logging purposes. ReadOnlyMemory <byte> eventBodyBytes = partitionEvent.Data.EventBody.ToMemory(); string data = Encoding.UTF8.GetString(eventBodyBytes.ToArray()); ConsoleHelper.WriteGreenMessage("Telemetry received: " + data); // The event data properties are then iterated and, in this // case, checked to see if a value is true - in the current // scenario, this represents an alert. Should an alert be // found, it is written to the console. foreach (var prop in partitionEvent.Data.Properties) { if (prop.Value.ToString() == "true") { ConsoleHelper.WriteRedMessage(prop.Key); } } Console.WriteLine(); } }
/// <summary> /// Performs the tasks needed to initialize and set up the environment for an instance /// of the test scenario. When multiple instances are run in parallel, setup will be /// run once for each prior to its execution. /// </summary> /// public async override Task SetupAsync() { await base.SetupAsync(); // Attempt to take a consumer group from the available set; to ensure that the // test scenario can support the requested level of parallelism without violating // the concurrent reader limits of a consumer group, the default consumer group // should not be used. if (!ConsumerGroups.TryDequeue(out var consumerGroup)) { throw new InvalidOperationException("Unable to reserve a consumer group to read from."); } _consumer = new EventHubConsumerClient(consumerGroup, TestEnvironment.EventHubsConnectionString, Scope.EventHubName); // In order to allow reading across multiple iterations, capture an enumerator. Without using a consistent // enumerator, a new AMQP link would be created and the position reset each time RunAsync was invoked. _readEnumerator = _consumer .ReadEventsFromPartitionAsync(PartitionId, EventPosition.Earliest) .ConfigureAwait(false) .GetAsyncEnumerator(); // Force the connection and link creation by reading a single event. if (!(await _readEnumerator.MoveNextAsync())) { throw new InvalidOperationException("Unable to read from the partition."); } }
public async IAsyncEnumerable <ReadOnlyMemory <byte> > ReadAsync([EnumeratorCancellation] CancellationToken cancellationToken) { var parititions = await _consumer.GetPartitionIdsAsync(cancellationToken); var channel = Channel.CreateBounded <ReadOnlyMemory <byte> >(100); var readertasks = new List <Task>(); foreach (var paritition in parititions) { var reader = Task.Run(async() => { await foreach (var eventData in _consumer.ReadEventsFromPartitionAsync( paritition, EventPosition.FromEnqueuedTime(DateTime.Now.AddMinutes(-5)), cancellationToken)) { await channel.Writer.WriteAsync(eventData.Data.EventBody.ToMemory()); } }, cancellationToken); readertasks.Add(reader); } await foreach (var item in channel.Reader.ReadAllAsync(cancellationToken)) { yield return(item); } }
public async Task ReadPartitionTrackLastEnqueued() { await using var scope = await EventHubScope.CreateAsync(1); #region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>"; var eventHubName = "<< NAME OF THE EVENT HUB >>"; var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; /*@@*/ /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString; /*@@*/ eventHubName = scope.EventHubName; var consumer = new EventHubConsumerClient( consumerGroup, connectionString, eventHubName); try { using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First(); EventPosition startingPosition = EventPosition.Earliest; var options = new ReadEventOptions { TrackLastEnqueuedEventProperties = true }; await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync( firstPartition, startingPosition, options, cancellationSource.Token)) { LastEnqueuedEventProperties properties = partitionEvent.Partition.ReadLastEnqueuedEventProperties(); Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }"); Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }"); Debug.WriteLine($"\tThe last offset is: { properties.Offset }"); Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC."); Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC."); } } catch (TaskCanceledException) { // This is expected if the cancellation token is // signaled. } finally { await consumer.CloseAsync(); } #endregion }
private async Task BackgroundReceive(string connectionString, string eventHubName, string partitionId, CancellationToken cancellationToken) { var reportTasks = new List <Task>(); EventPosition eventPosition; if (LastReceivedSequenceNumber.TryGetValue(partitionId, out long sequenceNumber)) { eventPosition = EventPosition.FromSequenceNumber(sequenceNumber, false); } else { eventPosition = EventPosition.Latest; } await using (var consumerClient = new EventHubConsumerClient("$Default", connectionString, eventHubName)) { Interlocked.Decrement(ref consumersToConnect); await foreach (var receivedEvent in consumerClient.ReadEventsFromPartitionAsync(partitionId, eventPosition, new ReadEventOptions { MaximumWaitTime = TimeSpan.FromSeconds(5) })) { if (receivedEvent.Data != null) { var key = Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray()); if (MissingEvents.TryRemove(key, out var expectedEvent)) { if (HaveSameProperties(expectedEvent, receivedEvent.Data)) { Interlocked.Increment(ref successfullyReceivedEventsCount); } else { reportTasks.Add(ReportCorruptedPropertiesEvent(partitionId, expectedEvent, receivedEvent.Data)); } } else { reportTasks.Add(ReportCorruptedBodyEvent(partitionId, receivedEvent.Data)); } LastReceivedSequenceNumber[partitionId] = receivedEvent.Data.SequenceNumber; } if (cancellationToken.IsCancellationRequested) { break; } } } await Task.WhenAll(reportTasks); }
public async Task WatchMessagesWithLimit(int limit, int messageTimeout, string consumerGroup, string connectionString, string eventHubName) { string consumerGroupWithDefault = EventHubConsumerClient.DefaultConsumerGroupName; if (consumerGroup != null) { consumerGroupWithDefault = consumerGroup; } await using (var consumer = new EventHubConsumerClient(consumerGroupWithDefault, connectionString, eventHubName)) { EventPosition startingPosition = EventPosition.Latest; using var cancellationSource = new System.Threading.CancellationTokenSource(); int maxWaitTime = messageTimeout == 0 ? 30 : messageTimeout; cancellationSource.CancelAfter(TimeSpan.FromSeconds(maxWaitTime)); string[] partitionIds = await consumer.GetPartitionIdsAsync(); var partitions = new IAsyncEnumerable <PartitionEvent> [partitionIds.Length]; for (int i = 0; i < partitionIds.Length; i++) { partitions[i] = consumer.ReadEventsFromPartitionAsync(partitionIds[i], startingPosition, cancellationSource.Token); } var mergedPartitions = AsyncEnumerable.Merge <PartitionEvent>(partitions); var maxMessages = Math.Max(1, limit); try { Console.WriteLine("Waiting for messages.."); int messageCount = 0; await foreach (var pe in mergedPartitions.Take <PartitionEvent>(maxMessages)) { //Console.WriteLine($"Event received on partition {pe.Partition.PartitionId} with body {Encoding.UTF8.GetString(pe.Data.Body.ToArray())}"); DisplayMessage(pe); messageCount = messageCount + 1; if (messageCount >= maxMessages) { Console.WriteLine($"Total messages received: {messageCount}"); break; } } } catch (Exception ex) { Console.WriteLine($"{ex}"); } } }
public async Task ReadPartitionFromSequence() { await using var scope = await EventHubScope.CreateAsync(1); #region Snippet:EventHubs_Sample05_ReadPartitionFromSequence #if SNIPPET var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>"; var eventHubName = "<< NAME OF THE EVENT HUB >>"; #else var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString; var eventHubName = scope.EventHubName; #endif var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; var consumer = new EventHubConsumerClient( consumerGroup, connectionString, eventHubName); try { using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First(); PartitionProperties properties = await consumer.GetPartitionPropertiesAsync(firstPartition, cancellationSource.Token); EventPosition startingPosition = EventPosition.FromSequenceNumber(properties.LastEnqueuedSequenceNumber); await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync( firstPartition, startingPosition, cancellationSource.Token)) { string readFromPartition = partitionEvent.Partition.PartitionId; byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray(); Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }"); } } catch (TaskCanceledException) { // This is expected if the cancellation token is // signaled. } finally { await consumer.CloseAsync(); } #endregion }
public async Task ReadPartitionFromDate() { #region Snippet:EventHubs_Sample05_ReadPartitionFromDate var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>"; var eventHubName = "<< NAME OF THE EVENT HUB >>"; var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; /*@@*/ /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString; /*@@*/ eventHubName = _scope.EventHubName; /*@@*/ consumerGroup = _availableConsumerGroups.Dequeue(); var consumer = new EventHubConsumerClient( consumerGroup, connectionString, eventHubName); try { using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); DateTimeOffset oneHourAgo = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1)); EventPosition startingPosition = EventPosition.FromEnqueuedTime(oneHourAgo); string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First(); await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync( firstPartition, startingPosition, cancellationSource.Token)) { string readFromPartition = partitionEvent.Partition.PartitionId; byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray(); Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }"); } } catch (TaskCanceledException) { // This is expected if the cancellation token is // signaled. } finally { await consumer.CloseAsync(); } #endregion }
static private async Task GetEvents() { EventHubConsumerClient client = new EventHubConsumerClient("$Default", connstring, hubname); string _partition = (await client.GetPartitionIdsAsync()).First(); var cancellation = new CancellationToken(); EventPosition _position = EventPosition.FromSequenceNumber(5); Console.WriteLine("Getting events from a certain position from a particular partition"); await foreach (PartitionEvent _recent_event in client.ReadEventsFromPartitionAsync(_partition, _position, cancellation)) { EventData event_data = _recent_event.Data; Console.WriteLine(Encoding.UTF8.GetString(event_data.Body.ToArray())); Console.WriteLine($"Sequence Number : {event_data.SequenceNumber}"); } }
private static async Task ReadFromPartition(string partitionNumber) { var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); await using var consumerClient = new EventHubConsumerClient(ConsumerGroup, ConnectionString, EventHubName); try { var props = await consumerClient.GetPartitionPropertiesAsync(partitionNumber); var startingPosition = EventPosition.FromSequenceNumber( //props.LastEnqueuedSequenceNumber props.BeginningSequenceNumber); await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsFromPartitionAsync(partitionNumber, startingPosition, cancellationTokenSource.Token)) { Console.WriteLine("***** NEW COFFEE *****"); var partitionId = partitionEvent.Partition.PartitionId; var sequenceNumber = partitionEvent.Data.SequenceNumber; var key = partitionEvent.Data.PartitionKey; Console.WriteLine($"Partition Id: {partitionId}{Environment.NewLine}" + $"SenquenceNumber: {sequenceNumber}{Environment.NewLine}" + $"Partition key: {key}"); var coffee = JsonSerializer.Deserialize <CoffeeData>(partitionEvent.Data.EventBody.ToArray()); Console.WriteLine($"Temperature: {coffee.WaterTemperature}, time: {coffee.BeadingTime}, type: {coffee.CoffeeType}"); } } catch (Exception ex) { Console.WriteLine(ex); } finally { await consumerClient.CloseAsync(); } }
public async Task ReadPartition() { await using var scope = await EventHubScope.CreateAsync(1); try { #region Snippet:EventHubs_ReadMe_ReadPartition #if SNIPPET var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>"; var eventHubName = "<< NAME OF THE EVENT HUB >>"; #else var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString; var eventHubName = scope.EventHubName; #endif string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; await using (var consumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName)) { EventPosition startingPosition = EventPosition.Earliest; string partitionId = (await consumer.GetPartitionIdsAsync()).First(); using var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(45)); await foreach (PartitionEvent receivedEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, cancellationSource.Token)) { // At this point, the loop will wait for events to be available in the partition. When an event // is available, the loop will iterate with the event that was received. Because we did not // specify a maximum wait time, the loop will wait forever unless cancellation is requested using // the cancellation token. } } #endregion } catch (TaskCanceledException) { // Expected } }
internal async Task startReadMessage(StartReadMessageCallback startReadMessageCallback) { _logger.LogInformation("Successfully created the EventHub Client from IoT Hub connection string."); var partitionIds = await _resultsClient.GetPartitionIdsAsync(); _logger.LogInformation("The partition ids are: " + String.Join(", ", partitionIds)); foreach (var id in partitionIds) { var task = new Task(async() => { await foreach (var message in _resultsClient.ReadEventsFromPartitionAsync(id, EventPosition.FromEnqueuedTime(DateTime.Now), readEventsCanseler.Token)) { var deviceId = (string)message.Data.SystemProperties["iothub-connection-device-id"]; startReadMessageCallback(message.Data.Body, message.Data.EnqueuedTime, deviceId); } }); task.Start(); receiveHandlers.Add(new Tuple <Task, CancellationTokenSource>(task, readEventsCanseler)); } }
//Read all events based on partitionId public async Task ConsumerReadEventPartitionEvent(string consumerGroup, string partitionId) { try { CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); EventHubConsumerClient eventConsumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName); ReadEventOptions readEventOptions = new ReadEventOptions() { MaximumWaitTime = TimeSpan.FromSeconds(30) }; await foreach (PartitionEvent partitionEvent in eventConsumer.ReadEventsFromPartitionAsync(partitionId, EventPosition.Latest, readEventOptions, cancellationSource.Token)) { Console.WriteLine("---Execution from ConsumerReadEventPartitionEvent method---"); Console.WriteLine("------"); if (partitionEvent.Data != null) { Console.WriteLine("Event Data recieved {0} ", Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray())); if (partitionEvent.Data.Properties != null) { foreach (var keyValue in partitionEvent.Data.Properties) { Console.WriteLine("Event data key = {0}, Event data value = {1}", keyValue.Key, keyValue.Value); } } } } await Task.CompletedTask; } catch (Exception exp) { Console.WriteLine("Error occruied {0}. Try again later", exp.Message); } }
private static async Task <CancellationTokenSource> GetEvents(EventHubConsumerClient eventHubClient, EventPosition startingPosition, DateTimeOffset endEnqueueTime) { var cancellationSource = new CancellationTokenSource(); if (int.TryParse(configuration["TerminateAfterSeconds"], out int TerminateAfterSeconds) == false) { throw new ArgumentException("Invalid TerminateAfterSeconds"); } cancellationSource.CancelAfter(TimeSpan.FromSeconds(TerminateAfterSeconds)); string path = Path.Combine(Directory.GetCurrentDirectory(), $"output-{Path.GetRandomFileName()}.json"); int count = 0; byte[] encodedText; using FileStream sourceStream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Write, bufferSize: 4096, useAsync: true); { encodedText = Encoding.Unicode.GetBytes("{\r\n\"events\": [" + Environment.NewLine); await sourceStream.WriteAsync(encodedText, 0, encodedText.Length); encodedText = Encoding.Unicode.GetBytes(""); await foreach (PartitionEvent receivedEvent in eventHubClient.ReadEventsFromPartitionAsync(partitionId, startingPosition, cancellationSource.Token)) { if (encodedText.Length > 0) { await sourceStream.WriteAsync(encodedText, 0, encodedText.Length); } count++; using var sr = new StreamReader(receivedEvent.Data.BodyAsStream); var data = sr.ReadToEnd(); var partition = receivedEvent.Data.PartitionKey; var offset = receivedEvent.Data.Offset; var sequence = receivedEvent.Data.SequenceNumber; try { dynamic message = AddMetaData(count, receivedEvent, data, partition, offset, sequence); var textWithMetaData = JsonConvert.SerializeObject(message); encodedText = Encoding.Unicode.GetBytes(textWithMetaData + "," + Environment.NewLine); } catch (Exception ex) { Console.WriteLine($"Serialization issue Partition: { partition}, Offset: {offset}, Sequence Number: { sequence }"); Console.WriteLine(ex.Message); } if (receivedEvent.Data.EnqueuedTime > endEnqueueTime) { Console.WriteLine($"Last Message EnqueueTime: {receivedEvent.Data.EnqueuedTime:o}, Offset: {receivedEvent.Data.Offset}, Sequence: {receivedEvent.Data.SequenceNumber}"); Console.WriteLine($"Total Events Streamed: {count}"); Console.WriteLine($"-----------"); break; } } encodedText = await FinaliseFile(encodedText, sourceStream); } Console.WriteLine($"\r\n Output located at: {path}"); return(cancellationSource); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { string firstPartition; // In this example, we will make use of multiple clients. Because clients are typically responsible for managing their own connection to the // Event Hubs service, each will implicitly create their own connection. In this example, we will create a connection that may be shared amongst // clients in order to illustrate connection sharing. Because we are explicitly creating the connection, we assume responsibility for managing its // lifespan and ensuring that it is properly closed or disposed when we are done using it. await using (var eventHubConnection = new EventHubConnection(connectionString, eventHubName)) { // Our initial consumer will begin watching the partition at the very end, reading only new events that we will publish for it. Before we can publish // the events and have them observed, we will need to ask the consumer to perform an operation, // because it opens its connection only when it needs to. // // We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when // no event is available. For the first call, we'll publish so that we can receive them. // // Each event that the initial consumer reads will have attributes set that describe the event's place in the // partition, such as its offset, sequence number, and the date/time that it was enqueued. These attributes can be // used to create a new consumer that begins consuming at a known position. // // With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed, // and to manage where in the partition the consumer begins reading events. This is done by using the position information to track // state, commonly known as "creating a checkpoint." // // The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the // consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that // begins reading at the position where it left off. // // It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed // consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues. // Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart, // there is a small possibility that the service will return an event multiple times. // // In this example, we will publish a batch of events to be received with an initial consumer. The third event that is consumed will be captured // and another consumer will use its attributes to start reading the event that follows, consuming the same set of events that our initial consumer // read, skipping over the first three. EventData thirdEvent; int eventBatchSize = 50; await using (var initialConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, eventHubConnection)) { // We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being // published and read from the same partition. firstPartition = (await initialConsumerClient.GetPartitionIdsAsync()).First(); // We will consume the events until all of the published events have been received. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); ReadOptions readOptions = new ReadOptions { MaximumWaitTime = TimeSpan.FromMilliseconds(150) }; List <EventData> receivedEvents = new List <EventData>(); bool wereEventsPublished = false; await foreach (PartitionEvent currentEvent in initialConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token)) { if (!wereEventsPublished) { await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName)) { using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition }); for (int index = 0; index < eventBatchSize; ++index) { eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"))); } await producerClient.SendAsync(eventBatch); wereEventsPublished = true; await Task.Delay(250); Console.WriteLine($"The event batch with { eventBatchSize } events has been published."); } } // Because publishing and receiving events is asynchronous, the events that we published may not // be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as // punctuation if our actual event is not available within the waiting time period. if (currentEvent.Data != null) { receivedEvents.Add(currentEvent.Data); if (receivedEvents.Count >= eventBatchSize) { break; } } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } // Remember the third event that was consumed. thirdEvent = receivedEvents[2]; } // At this point, our initial consumer client has passed its "using" scope and has been safely disposed of. // // Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available // sequence number, allowing it to read the set of published events beginning with the fourth one. // // Because our second consumer will begin watching the partition at a specific event, there is no need to ask for an initial operation to set our place; when // we begin iterating, the consumer will locate the proper place in the partition to read from. await using (var newConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, eventHubConnection)) { // We will consume the events using the new consumer until all of the published events have been received. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); int expectedCount = (eventBatchSize - 3); var receivedEvents = new List <EventData>(); await foreach (PartitionEvent currentEvent in newConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value), cancellationSource.Token)) { receivedEvents.Add(currentEvent.Data); if (receivedEvents.Count >= expectedCount) { break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine(); Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected."); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } } // At this point, our clients and connection have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is // a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility // of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default // consumer group is created with it, called "$Default." // // Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all // consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and // process events from the partition at different speeds and beginning with different events without interfering with // one another. // // When events are published, they will continue to exist in the partition and be available for consuming until they // reach an age where they are older than the retention period. // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events) // // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it // would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an // offset from the beginning, the next event available after a specific point in time, or at a specific event. // We will start by creating a client to inspect the Event Hub and select a partition to operate against to ensure that // events are being published and read from the same partition. string firstPartition; await using (var inspectionClient = new EventHubProducerClient(connectionString, eventHubName)) { // With our client, we can now inspect the partitions and find the identifier // of the first. firstPartition = (await inspectionClient.GetPartitionIdsAsync()).First(); } // In this example, we will create our consumer client for the first partition in the Event Hub, using the default consumer group // that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events // that we will publish for it. await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.Latest, connectionString, eventHubName)) await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName, new EventHubProducerClientOptions { PartitionId = firstPartition })) { PartitionEvent receivedEvent; Stopwatch watch = Stopwatch.StartNew(); bool wereEventsPublished = false; // Because our consumer is reading from the latest position, it won't see events that have previously // been published. Before we can publish the events and have them observed, we will need to ask the consumer // to perform an operation, because it opens its connection only when it needs to. // // We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when // no event is available. For the first call, we'll publish so that we can receive them. // // Because publishing and receiving events is asynchronous, the events that we published may not // be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as // punctuation if our actual event is not available within the waiting time period. // // We will iterate over the available events in the partition, which should be just the event that we published. Because // we're expecting only the one event, we will exit the loop when we receive it. To be sure that we do not block forever // waiting on an event that is not published, we will request cancellation after a fairly long interval. CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, TimeSpan.FromMilliseconds(150), cancellationSource.Token)) { if (!wereEventsPublished) { await producerClient.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!"))); wereEventsPublished = true; Console.WriteLine("The event batch has been published."); } // Because publishing is non-deterministic, the event may not be available // before the next timeout returns control to us. We'll guard against that by // ensuring the event has data associated with it. if (currentEvent.Data != null) { receivedEvent = currentEvent; watch.Stop(); break; } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:"); // The body of our event was an encoded string; we'll recover the message by reversing the encoding process. string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } // At this point, our clients have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
public async Task ReadPartitionWaitTime() { await using var scope = await EventHubScope.CreateAsync(1); #region Snippet:EventHubs_Sample05_ReadPartitionWaitTime var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>"; var eventHubName = "<< NAME OF THE EVENT HUB >>"; var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; /*@@*/ /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString; /*@@*/ eventHubName = scope.EventHubName; var consumer = new EventHubConsumerClient( consumerGroup, connectionString, eventHubName); try { using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First(); EventPosition startingPosition = EventPosition.Earliest; int loopTicks = 0; int maximumTicks = 10; var options = new ReadEventOptions { MaximumWaitTime = TimeSpan.FromSeconds(1) }; await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync( firstPartition, startingPosition, options)) { if (partitionEvent.Data != null) { string readFromPartition = partitionEvent.Partition.PartitionId; byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray(); Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }"); } else { Debug.WriteLine("Wait time elapsed; no event was available."); } loopTicks++; if (loopTicks >= maximumTicks) { break; } } } catch (TaskCanceledException) { // This is expected if the cancellation token is // signaled. } finally { await consumer.CloseAsync(); } #endregion }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is // a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility // of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default // consumer group is created with it, called "$Default." // // Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all // consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and // process events from the partition at different speeds and beginning with different events without interfering with // one another. // // When events are published, they will continue to exist in the partition and be available for consuming until they // reach an age where they are older than the retention period. // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events) // // Because events are not removed from the partition when consuming, a consumer must specify where in the partition it // would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an // offset from the beginning, the next event available after a specific point in time, or at a specific event. // // In this example, we will create our consumer client using the default consumer group that is created with an Event Hub. // Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it. await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName)) { // We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being // published and read from the same partition. string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First(); // Because our consumer is reading from the latest position, it won't see events that have previously // been published. Before we can publish the events and have them observed, we will need to ask the consumer // to perform an operation, because it opens its connection only when it needs to. // // When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed, // whether or not an event is available in the partition. If no event was available a null value will be emitted instead. // This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event // processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue // if events are not arriving. // // We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when // no event is available. For the first call, we'll publish so that we can receive them. // // For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more // event than we published, which is expected to be a null value triggered by exceeding the wait time. // // To be sure that we do not block forever waiting on an event that is not published, we will request cancellation after a // fairly long interval. bool wereEventsPublished = false; int eventBatchCount = 0; List <EventData> receivedEvents = new List <EventData>(); Stopwatch watch = Stopwatch.StartNew(); CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(250); await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, maximumWaitTime, cancellationSource.Token)) { if (!wereEventsPublished) { await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName)) { using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition }); eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!"))); eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!"))); await producerClient.SendAsync(eventBatch); wereEventsPublished = true; eventBatchCount = eventBatch.Count; Console.WriteLine("The event batch has been published."); } } else { receivedEvents.Add(currentEvent.Data); if (receivedEvents.Count > eventBatchCount) { watch.Stop(); break; } } } // Print out the events that we received. Console.WriteLine(); Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:"); foreach (EventData eventData in receivedEvents) { // The body of our event was an encoded string; we'll recover the // message by reversing the encoding process. string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } } // At this point, our clients have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
/// <summary> /// Starts running a task responsible for receiving and processing events in the context of a specified partition. /// </summary> /// /// <param name="partitionId">The identifier of the Event Hub partition the task is associated with. Events will be read only from this partition.</param> /// <param name="startingPosition">The position within the partition where the task should begin reading events.</param> /// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param> /// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param> /// <param name="trackLastEnqueuedEventInformation">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param> /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns> /// protected virtual Task RunPartitionProcessingAsync(string partitionId, EventPosition startingPosition, TimeSpan?maximumReceiveWaitTime, RetryOptions retryOptions, bool trackLastEnqueuedEventInformation, CancellationToken cancellationToken = default) { // TODO: should the retry options used here be the same for the abstract RetryPolicy property? Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId)); Argument.AssertNotNull(retryOptions, nameof(retryOptions)); return(Task.Run(async() => { // TODO: should we double check if a previous run already exists and close it? We have a race condition. Maybe we should throw in case another task exists. var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var taskCancellationToken = cancellationSource.Token; ActivePartitionProcessorTokenSources[partitionId] = cancellationSource; // Context is set to default if operation fails. This shouldn't fail unless the user tries processing // a partition they don't own. PartitionContexts.TryGetValue(partitionId, out var context); var options = new EventHubConsumerClientOptions { RetryOptions = retryOptions, TrackLastEnqueuedEventInformation = trackLastEnqueuedEventInformation }; await using var connection = CreateConnection(); await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, options)) { await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, maximumReceiveWaitTime, taskCancellationToken)) { using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName); diagnosticScope.AddAttribute("kind", "server"); if (diagnosticScope.IsEnabled && partitionEvent.Data != null && EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId)) { diagnosticScope.AddLink(diagnosticId); } diagnosticScope.Start(); try { await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false); } catch (Exception eventProcessingException) { diagnosticScope.Failed(eventProcessingException); throw; } } } })); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against</param> /// <param name="tenantId">The Azure Active Directory tenant that holds the service principal</param> /// <param name="clientId">The Azure Active Directory client identifier of the service principal</param> /// <param name="secret">The Azure Active Directory secret of the service principal</param> /// public async Task RunAsync(string fullyQualifiedNamespace, string eventHubName, string tenantId, string clientId, string secret) { // Service principal authentication is a means for applications to authenticate // against Azure Active Directory and consume Azure services. This is advantageous compared // to signing in using fully privileged users as it allows to enforce role-based authorization // from the portal. // // Service principal authentication differs from managed identity authentication also because the principal to be used // will be able to authenticate with the portal without the need for the code to run within the portal. // The authentication between the Event Hubs client and the portal is performed through OAuth 2.0. // (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals) // ClientSecretCredential allows performing service principal authentication passing // tenantId, clientId and clientSecret directly from the constructor. // (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow) var credentials = new ClientSecretCredential(tenantId, clientId, secret); // EventHubProducerClient takes ClientSecretCredential from its constructor and tries to issue a token from Azure Active Directory. await using (EventHubProducerClient client = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials)) { // It will then use that token to authenticate on the portal and enquiry the Hub properties. Console.WriteLine($"Contacting the hub using the token issued from client credentials."); var properties = await client.GetEventHubPropertiesAsync(); Console.WriteLine($"Event Hub \"{ properties.Name }\" reached successfully."); } // The instances of "EventHubProducerClient" and "EventHubProducerClient" will be created // passing in "ClientSecretCredential" instead of the connection string. In this way, two things will happen: // // 1. An OAuth 2.0 token will be created by authenticating against Azure Active Directory using the tenant, client and the secret passed in // 2. Role-based authorization will be performed and the "Azure Event Hubs Data Owner" role will be needed to produce and consume events // // (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-definitions) // (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-event-hubs-data-owner) await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credentials)) { string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First(); PartitionEvent receivedEvent; ReadEventOptions readOptions = new ReadEventOptions { MaximumWaitTime = TimeSpan.FromMilliseconds(150) }; Stopwatch watch = Stopwatch.StartNew(); bool wereEventsPublished = false; CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(30)); await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token)) { if (!wereEventsPublished) { await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials)) { using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition }); eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!"))); await producerClient.SendAsync(eventBatch); wereEventsPublished = true; Console.WriteLine("The event batch has been published."); } } if (currentEvent.Data != null) { receivedEvent = currentEvent; watch.Stop(); break; } } Console.WriteLine(); Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:"); string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray()); Console.WriteLine($"\tMessage: \"{ message }\""); } Console.WriteLine(); }