// Reads events from the requested partition as an asynchronous
// enumerable, allowing events to be iterated as they become available
// on the partition, waiting as necessary should there be no events available.
// As you can see, this method is supplied with an argument that defines
// the target partition. Recall that for the default configuration where
// 4 partitions are specified, this method is called 4 times, each
// running asynchronously and in parallel, one for each partition.
private static async Task ReceiveMessagesFromDeviceAsync(string partition)
{
EventPosition startingPosition = EventPosition.Earliest;
// Reads events from the requested partition as an asynchronous
// enumerable, allowing events to be iterated as they become available
// on the partition, waiting as necessary should there be no events
// available.
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
partition,
startingPosition))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
// Each event data body is converted from BinaryData to a byte
// array, and from there, to a string and written to the
// console for logging purposes.
ReadOnlyMemory <byte> eventBodyBytes = partitionEvent.Data.EventBody.ToMemory();
string data = Encoding.UTF8.GetString(eventBodyBytes.ToArray());
ConsoleHelper.WriteGreenMessage("Telemetry received: " + data);
// The event data properties are then iterated and, in this
// case, checked to see if a value is true - in the current
// scenario, this represents an alert. Should an alert be
// found, it is written to the console.
foreach (var prop in partitionEvent.Data.Properties)
{
if (prop.Value.ToString() == "true")
{
ConsoleHelper.WriteRedMessage(prop.Key);
}
}
Console.WriteLine();
}
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for an instance
/// of the test scenario. When multiple instances are run in parallel, setup will be
/// run once for each prior to its execution.
/// </summary>
///
public async override Task SetupAsync()
{
await base.SetupAsync();
// Attempt to take a consumer group from the available set; to ensure that the
// test scenario can support the requested level of parallelism without violating
// the concurrent reader limits of a consumer group, the default consumer group
// should not be used.
if (!ConsumerGroups.TryDequeue(out var consumerGroup))
{
throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
}
_consumer = new EventHubConsumerClient(consumerGroup, TestEnvironment.EventHubsConnectionString, Scope.EventHubName);
// In order to allow reading across multiple iterations, capture an enumerator. Without using a consistent
// enumerator, a new AMQP link would be created and the position reset each time RunAsync was invoked.
_readEnumerator = _consumer
.ReadEventsFromPartitionAsync(PartitionId, EventPosition.Earliest)
.ConfigureAwait(false)
.GetAsyncEnumerator();
// Force the connection and link creation by reading a single event.
if (!(await _readEnumerator.MoveNextAsync()))
{
throw new InvalidOperationException("Unable to read from the partition.");
}
}
public async IAsyncEnumerable <ReadOnlyMemory <byte> > ReadAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
var parititions = await _consumer.GetPartitionIdsAsync(cancellationToken);
var channel = Channel.CreateBounded <ReadOnlyMemory <byte> >(100);
var readertasks = new List <Task>();
foreach (var paritition in parititions)
{
var reader = Task.Run(async() =>
{
await foreach (var eventData in _consumer.ReadEventsFromPartitionAsync(
paritition,
EventPosition.FromEnqueuedTime(DateTime.Now.AddMinutes(-5)),
cancellationToken))
{
await channel.Writer.WriteAsync(eventData.Data.EventBody.ToMemory());
}
}, cancellationToken);
readertasks.Add(reader);
}
await foreach (var item in channel.Reader.ReadAllAsync(cancellationToken))
{
yield return(item);
}
}
public async Task ReadPartitionTrackLastEnqueued()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
var options = new ReadEventOptions
{
TrackLastEnqueuedEventProperties = true
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options,
cancellationSource.Token))
{
LastEnqueuedEventProperties properties =
partitionEvent.Partition.ReadLastEnqueuedEventProperties();
Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }");
Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }");
Debug.WriteLine($"\tThe last offset is: { properties.Offset }");
Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC.");
Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC.");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
private async Task BackgroundReceive(string connectionString, string eventHubName, string partitionId, CancellationToken cancellationToken)
{
var reportTasks = new List <Task>();
EventPosition eventPosition;
if (LastReceivedSequenceNumber.TryGetValue(partitionId, out long sequenceNumber))
{
eventPosition = EventPosition.FromSequenceNumber(sequenceNumber, false);
}
else
{
eventPosition = EventPosition.Latest;
}
await using (var consumerClient = new EventHubConsumerClient("$Default", connectionString, eventHubName))
{
Interlocked.Decrement(ref consumersToConnect);
await foreach (var receivedEvent in consumerClient.ReadEventsFromPartitionAsync(partitionId, eventPosition, new ReadEventOptions {
MaximumWaitTime = TimeSpan.FromSeconds(5)
}))
{
if (receivedEvent.Data != null)
{
var key = Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
if (MissingEvents.TryRemove(key, out var expectedEvent))
{
if (HaveSameProperties(expectedEvent, receivedEvent.Data))
{
Interlocked.Increment(ref successfullyReceivedEventsCount);
}
else
{
reportTasks.Add(ReportCorruptedPropertiesEvent(partitionId, expectedEvent, receivedEvent.Data));
}
}
else
{
reportTasks.Add(ReportCorruptedBodyEvent(partitionId, receivedEvent.Data));
}
LastReceivedSequenceNumber[partitionId] = receivedEvent.Data.SequenceNumber;
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
}
}
await Task.WhenAll(reportTasks);
}
public async Task WatchMessagesWithLimit(int limit, int messageTimeout, string consumerGroup, string connectionString, string eventHubName)
{
string consumerGroupWithDefault = EventHubConsumerClient.DefaultConsumerGroupName;
if (consumerGroup != null)
{
consumerGroupWithDefault = consumerGroup;
}
await using (var consumer = new EventHubConsumerClient(consumerGroupWithDefault, connectionString, eventHubName))
{
EventPosition startingPosition = EventPosition.Latest;
using var cancellationSource = new System.Threading.CancellationTokenSource();
int maxWaitTime = messageTimeout == 0 ? 30 : messageTimeout;
cancellationSource.CancelAfter(TimeSpan.FromSeconds(maxWaitTime));
string[] partitionIds = await consumer.GetPartitionIdsAsync();
var partitions = new IAsyncEnumerable <PartitionEvent> [partitionIds.Length];
for (int i = 0; i < partitionIds.Length; i++)
{
partitions[i] = consumer.ReadEventsFromPartitionAsync(partitionIds[i], startingPosition, cancellationSource.Token);
}
var mergedPartitions = AsyncEnumerable.Merge <PartitionEvent>(partitions);
var maxMessages = Math.Max(1, limit);
try
{
Console.WriteLine("Waiting for messages..");
int messageCount = 0;
await foreach (var pe in mergedPartitions.Take <PartitionEvent>(maxMessages))
{
//Console.WriteLine($"Event received on partition {pe.Partition.PartitionId} with body {Encoding.UTF8.GetString(pe.Data.Body.ToArray())}");
DisplayMessage(pe);
messageCount = messageCount + 1;
if (messageCount >= maxMessages)
{
Console.WriteLine($"Total messages received: {messageCount}");
break;
}
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex}");
}
}
}
public async Task ReadPartitionFromSequence()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionFromSequence
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
PartitionProperties properties = await consumer.GetPartitionPropertiesAsync(firstPartition, cancellationSource.Token);
EventPosition startingPosition = EventPosition.FromSequenceNumber(properties.LastEnqueuedSequenceNumber);
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task ReadPartitionFromDate()
{
#region Snippet:EventHubs_Sample05_ReadPartitionFromDate
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
/*@@*/ consumerGroup = _availableConsumerGroups.Dequeue();
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
DateTimeOffset oneHourAgo = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1));
EventPosition startingPosition = EventPosition.FromEnqueuedTime(oneHourAgo);
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
static private async Task GetEvents()
{
EventHubConsumerClient client = new EventHubConsumerClient("$Default", connstring, hubname);
string _partition = (await client.GetPartitionIdsAsync()).First();
var cancellation = new CancellationToken();
EventPosition _position = EventPosition.FromSequenceNumber(5);
Console.WriteLine("Getting events from a certain position from a particular partition");
await foreach (PartitionEvent _recent_event in client.ReadEventsFromPartitionAsync(_partition, _position, cancellation))
{
EventData event_data = _recent_event.Data;
Console.WriteLine(Encoding.UTF8.GetString(event_data.Body.ToArray()));
Console.WriteLine($"Sequence Number : {event_data.SequenceNumber}");
}
}
private static async Task ReadFromPartition(string partitionNumber)
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120));
await using var consumerClient = new EventHubConsumerClient(ConsumerGroup, ConnectionString, EventHubName);
try
{
var props = await consumerClient.GetPartitionPropertiesAsync(partitionNumber);
var startingPosition = EventPosition.FromSequenceNumber(
//props.LastEnqueuedSequenceNumber
props.BeginningSequenceNumber);
await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsFromPartitionAsync(partitionNumber, startingPosition, cancellationTokenSource.Token))
{
Console.WriteLine("***** NEW COFFEE *****");
var partitionId = partitionEvent.Partition.PartitionId;
var sequenceNumber = partitionEvent.Data.SequenceNumber;
var key = partitionEvent.Data.PartitionKey;
Console.WriteLine($"Partition Id: {partitionId}{Environment.NewLine}"
+ $"SenquenceNumber: {sequenceNumber}{Environment.NewLine}"
+ $"Partition key: {key}");
var coffee = JsonSerializer.Deserialize <CoffeeData>(partitionEvent.Data.EventBody.ToArray());
Console.WriteLine($"Temperature: {coffee.WaterTemperature}, time: {coffee.BeadingTime}, type: {coffee.CoffeeType}");
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
await consumerClient.CloseAsync();
}
}
public async Task ReadPartition()
{
await using var scope = await EventHubScope.CreateAsync(1);
try
{
#region Snippet:EventHubs_ReadMe_ReadPartition
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
await using (var consumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName))
{
EventPosition startingPosition = EventPosition.Earliest;
string partitionId = (await consumer.GetPartitionIdsAsync()).First();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(45));
await foreach (PartitionEvent receivedEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, cancellationSource.Token))
{
// At this point, the loop will wait for events to be available in the partition. When an event
// is available, the loop will iterate with the event that was received. Because we did not
// specify a maximum wait time, the loop will wait forever unless cancellation is requested using
// the cancellation token.
}
}
#endregion
}
catch (TaskCanceledException)
{
// Expected
}
}
internal async Task startReadMessage(StartReadMessageCallback startReadMessageCallback)
{
_logger.LogInformation("Successfully created the EventHub Client from IoT Hub connection string.");
var partitionIds = await _resultsClient.GetPartitionIdsAsync();
_logger.LogInformation("The partition ids are: " + String.Join(", ", partitionIds));
foreach (var id in partitionIds)
{
var task = new Task(async() => {
await foreach (var message in _resultsClient.ReadEventsFromPartitionAsync(id, EventPosition.FromEnqueuedTime(DateTime.Now), readEventsCanseler.Token))
{
var deviceId = (string)message.Data.SystemProperties["iothub-connection-device-id"];
startReadMessageCallback(message.Data.Body, message.Data.EnqueuedTime, deviceId);
}
});
task.Start();
receiveHandlers.Add(new Tuple <Task, CancellationTokenSource>(task, readEventsCanseler));
}
}
//Read all events based on partitionId
public async Task ConsumerReadEventPartitionEvent(string consumerGroup, string partitionId)
{
try
{
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
EventHubConsumerClient eventConsumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName);
ReadEventOptions readEventOptions = new ReadEventOptions()
{
MaximumWaitTime = TimeSpan.FromSeconds(30)
};
await foreach (PartitionEvent partitionEvent in eventConsumer.ReadEventsFromPartitionAsync(partitionId, EventPosition.Latest, readEventOptions, cancellationSource.Token))
{
Console.WriteLine("---Execution from ConsumerReadEventPartitionEvent method---");
Console.WriteLine("------");
if (partitionEvent.Data != null)
{
Console.WriteLine("Event Data recieved {0} ", Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray()));
if (partitionEvent.Data.Properties != null)
{
foreach (var keyValue in partitionEvent.Data.Properties)
{
Console.WriteLine("Event data key = {0}, Event data value = {1}", keyValue.Key, keyValue.Value);
}
}
}
}
await Task.CompletedTask;
}
catch (Exception exp)
{
Console.WriteLine("Error occruied {0}. Try again later", exp.Message);
}
}
private static async Task <CancellationTokenSource> GetEvents(EventHubConsumerClient eventHubClient, EventPosition startingPosition, DateTimeOffset endEnqueueTime)
{
var cancellationSource = new CancellationTokenSource();
if (int.TryParse(configuration["TerminateAfterSeconds"], out int TerminateAfterSeconds) == false)
{
throw new ArgumentException("Invalid TerminateAfterSeconds");
}
cancellationSource.CancelAfter(TimeSpan.FromSeconds(TerminateAfterSeconds));
string path = Path.Combine(Directory.GetCurrentDirectory(), $"output-{Path.GetRandomFileName()}.json");
int count = 0;
byte[] encodedText;
using FileStream sourceStream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Write, bufferSize: 4096, useAsync: true);
{
encodedText = Encoding.Unicode.GetBytes("{\r\n\"events\": [" + Environment.NewLine);
await sourceStream.WriteAsync(encodedText, 0, encodedText.Length);
encodedText = Encoding.Unicode.GetBytes("");
await foreach (PartitionEvent receivedEvent in eventHubClient.ReadEventsFromPartitionAsync(partitionId, startingPosition, cancellationSource.Token))
{
if (encodedText.Length > 0)
{
await sourceStream.WriteAsync(encodedText, 0, encodedText.Length);
}
count++;
using var sr = new StreamReader(receivedEvent.Data.BodyAsStream);
var data = sr.ReadToEnd();
var partition = receivedEvent.Data.PartitionKey;
var offset = receivedEvent.Data.Offset;
var sequence = receivedEvent.Data.SequenceNumber;
try
{
dynamic message = AddMetaData(count, receivedEvent, data, partition, offset, sequence);
var textWithMetaData = JsonConvert.SerializeObject(message);
encodedText = Encoding.Unicode.GetBytes(textWithMetaData + "," + Environment.NewLine);
}
catch (Exception ex)
{
Console.WriteLine($"Serialization issue Partition: { partition}, Offset: {offset}, Sequence Number: { sequence }");
Console.WriteLine(ex.Message);
}
if (receivedEvent.Data.EnqueuedTime > endEnqueueTime)
{
Console.WriteLine($"Last Message EnqueueTime: {receivedEvent.Data.EnqueuedTime:o}, Offset: {receivedEvent.Data.Offset}, Sequence: {receivedEvent.Data.SequenceNumber}");
Console.WriteLine($"Total Events Streamed: {count}");
Console.WriteLine($"-----------");
break;
}
}
encodedText = await FinaliseFile(encodedText, sourceStream);
}
Console.WriteLine($"\r\n Output located at: {path}");
return(cancellationSource);
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
string firstPartition;
// In this example, we will make use of multiple clients. Because clients are typically responsible for managing their own connection to the
// Event Hubs service, each will implicitly create their own connection. In this example, we will create a connection that may be shared amongst
// clients in order to illustrate connection sharing. Because we are explicitly creating the connection, we assume responsibility for managing its
// lifespan and ensuring that it is properly closed or disposed when we are done using it.
await using (var eventHubConnection = new EventHubConnection(connectionString, eventHubName))
{
// Our initial consumer will begin watching the partition at the very end, reading only new events that we will publish for it. Before we can publish
// the events and have them observed, we will need to ask the consumer to perform an operation,
// because it opens its connection only when it needs to.
//
// We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when
// no event is available. For the first call, we'll publish so that we can receive them.
//
// Each event that the initial consumer reads will have attributes set that describe the event's place in the
// partition, such as its offset, sequence number, and the date/time that it was enqueued. These attributes can be
// used to create a new consumer that begins consuming at a known position.
//
// With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed,
// and to manage where in the partition the consumer begins reading events. This is done by using the position information to track
// state, commonly known as "creating a checkpoint."
//
// The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the
// consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that
// begins reading at the position where it left off.
//
// It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed
// consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues.
// Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart,
// there is a small possibility that the service will return an event multiple times.
//
// In this example, we will publish a batch of events to be received with an initial consumer. The third event that is consumed will be captured
// and another consumer will use its attributes to start reading the event that follows, consuming the same set of events that our initial consumer
// read, skipping over the first three.
EventData thirdEvent;
int eventBatchSize = 50;
await using (var initialConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, eventHubConnection))
{
// We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being
// published and read from the same partition.
firstPartition = (await initialConsumerClient.GetPartitionIdsAsync()).First();
// We will consume the events until all of the published events have been received.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
ReadOptions readOptions = new ReadOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
List <EventData> receivedEvents = new List <EventData>();
bool wereEventsPublished = false;
await foreach (PartitionEvent currentEvent in initialConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
for (int index = 0; index < eventBatchSize; ++index)
{
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")));
}
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
await Task.Delay(250);
Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
}
}
// Because publishing and receiving events is asynchronous, the events that we published may not
// be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as
// punctuation if our actual event is not available within the waiting time period.
if (currentEvent.Data != null)
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count >= eventBatchSize)
{
break;
}
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
// Remember the third event that was consumed.
thirdEvent = receivedEvents[2];
}
// At this point, our initial consumer client has passed its "using" scope and has been safely disposed of.
//
// Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available
// sequence number, allowing it to read the set of published events beginning with the fourth one.
//
// Because our second consumer will begin watching the partition at a specific event, there is no need to ask for an initial operation to set our place; when
// we begin iterating, the consumer will locate the proper place in the partition to read from.
await using (var newConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, eventHubConnection))
{
// We will consume the events using the new consumer until all of the published events have been received.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
int expectedCount = (eventBatchSize - 3);
var receivedEvents = new List <EventData>();
await foreach (PartitionEvent currentEvent in newConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value), cancellationSource.Token))
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count >= expectedCount)
{
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine();
Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
// At this point, our clients and connection have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
// consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
// would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an
// offset from the beginning, the next event available after a specific point in time, or at a specific event.
// We will start by creating a client to inspect the Event Hub and select a partition to operate against to ensure that
// events are being published and read from the same partition.
string firstPartition;
await using (var inspectionClient = new EventHubProducerClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier
// of the first.
firstPartition = (await inspectionClient.GetPartitionIdsAsync()).First();
}
// In this example, we will create our consumer client for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, firstPartition, EventPosition.Latest, connectionString, eventHubName))
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName, new EventHubProducerClientOptions {
PartitionId = firstPartition
}))
{
PartitionEvent receivedEvent;
Stopwatch watch = Stopwatch.StartNew();
bool wereEventsPublished = false;
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events and have them observed, we will need to ask the consumer
// to perform an operation, because it opens its connection only when it needs to.
//
// We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when
// no event is available. For the first call, we'll publish so that we can receive them.
//
// Because publishing and receiving events is asynchronous, the events that we published may not
// be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as
// punctuation if our actual event is not available within the waiting time period.
//
// We will iterate over the available events in the partition, which should be just the event that we published. Because
// we're expecting only the one event, we will exit the loop when we receive it. To be sure that we do not block forever
// waiting on an event that is not published, we will request cancellation after a fairly long interval.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, TimeSpan.FromMilliseconds(150), cancellationSource.Token))
{
if (!wereEventsPublished)
{
await producerClient.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
wereEventsPublished = true;
Console.WriteLine("The event batch has been published.");
}
// Because publishing is non-deterministic, the event may not be available
// before the next timeout returns control to us. We'll guard against that by
// ensuring the event has data associated with it.
if (currentEvent.Data != null)
{
receivedEvent = currentEvent;
watch.Stop();
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:");
// The body of our event was an encoded string; we'll recover the message by reversing the encoding process.
string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task ReadPartitionWaitTime()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionWaitTime
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
int loopTicks = 0;
int maximumTicks = 10;
var options = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options))
{
if (partitionEvent.Data != null)
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
else
{
Debug.WriteLine("Wait time elapsed; no event was available.");
}
loopTicks++;
if (loopTicks >= maximumTicks)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
// consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
// would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an
// offset from the beginning, the next event available after a specific point in time, or at a specific event.
//
// In this example, we will create our consumer client using the default consumer group that is created with an Event Hub.
// Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
{
// We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being
// published and read from the same partition.
string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events and have them observed, we will need to ask the consumer
// to perform an operation, because it opens its connection only when it needs to.
//
// When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed,
// whether or not an event is available in the partition. If no event was available a null value will be emitted instead.
// This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event
// processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue
// if events are not arriving.
//
// We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when
// no event is available. For the first call, we'll publish so that we can receive them.
//
// For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more
// event than we published, which is expected to be a null value triggered by exceeding the wait time.
//
// To be sure that we do not block forever waiting on an event that is not published, we will request cancellation after a
// fairly long interval.
bool wereEventsPublished = false;
int eventBatchCount = 0;
List <EventData> receivedEvents = new List <EventData>();
Stopwatch watch = Stopwatch.StartNew();
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(250);
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, maximumWaitTime, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")));
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
eventBatchCount = eventBatch.Count;
Console.WriteLine("The event batch has been published.");
}
}
else
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count > eventBatchCount)
{
watch.Stop();
break;
}
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Starts running a task responsible for receiving and processing events in the context of a specified partition.
/// </summary>
///
/// <param name="partitionId">The identifier of the Event Hub partition the task is associated with. Events will be read only from this partition.</param>
/// <param name="startingPosition">The position within the partition where the task should begin reading events.</param>
/// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param>
/// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param>
/// <param name="trackLastEnqueuedEventInformation">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns>
///
protected virtual Task RunPartitionProcessingAsync(string partitionId,
EventPosition startingPosition,
TimeSpan?maximumReceiveWaitTime,
RetryOptions retryOptions,
bool trackLastEnqueuedEventInformation,
CancellationToken cancellationToken = default)
{
// TODO: should the retry options used here be the same for the abstract RetryPolicy property?
Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId));
Argument.AssertNotNull(retryOptions, nameof(retryOptions));
return(Task.Run(async() =>
{
// TODO: should we double check if a previous run already exists and close it? We have a race condition. Maybe we should throw in case another task exists.
var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var taskCancellationToken = cancellationSource.Token;
ActivePartitionProcessorTokenSources[partitionId] = cancellationSource;
// Context is set to default if operation fails. This shouldn't fail unless the user tries processing
// a partition they don't own.
PartitionContexts.TryGetValue(partitionId, out var context);
var options = new EventHubConsumerClientOptions
{
RetryOptions = retryOptions,
TrackLastEnqueuedEventInformation = trackLastEnqueuedEventInformation
};
await using var connection = CreateConnection();
await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, options))
{
await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, maximumReceiveWaitTime, taskCancellationToken))
{
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled &&
partitionEvent.Data != null &&
EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
diagnosticScope.Start();
try
{
await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
throw;
}
}
}
}));
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against</param>
/// <param name="tenantId">The Azure Active Directory tenant that holds the service principal</param>
/// <param name="clientId">The Azure Active Directory client identifier of the service principal</param>
/// <param name="secret">The Azure Active Directory secret of the service principal</param>
///
public async Task RunAsync(string fullyQualifiedNamespace,
string eventHubName,
string tenantId,
string clientId,
string secret)
{
// Service principal authentication is a means for applications to authenticate
// against Azure Active Directory and consume Azure services. This is advantageous compared
// to signing in using fully privileged users as it allows to enforce role-based authorization
// from the portal.
//
// Service principal authentication differs from managed identity authentication also because the principal to be used
// will be able to authenticate with the portal without the need for the code to run within the portal.
// The authentication between the Event Hubs client and the portal is performed through OAuth 2.0.
// (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals)
// ClientSecretCredential allows performing service principal authentication passing
// tenantId, clientId and clientSecret directly from the constructor.
// (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow)
var credentials = new ClientSecretCredential(tenantId, clientId, secret);
// EventHubProducerClient takes ClientSecretCredential from its constructor and tries to issue a token from Azure Active Directory.
await using (EventHubProducerClient client = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
{
// It will then use that token to authenticate on the portal and enquiry the Hub properties.
Console.WriteLine($"Contacting the hub using the token issued from client credentials.");
var properties = await client.GetEventHubPropertiesAsync();
Console.WriteLine($"Event Hub \"{ properties.Name }\" reached successfully.");
}
// The instances of "EventHubProducerClient" and "EventHubProducerClient" will be created
// passing in "ClientSecretCredential" instead of the connection string. In this way, two things will happen:
//
// 1. An OAuth 2.0 token will be created by authenticating against Azure Active Directory using the tenant, client and the secret passed in
// 2. Role-based authorization will be performed and the "Azure Event Hubs Data Owner" role will be needed to produce and consume events
//
// (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-definitions)
// (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-event-hubs-data-owner)
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credentials))
{
string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();
PartitionEvent receivedEvent;
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
Stopwatch watch = Stopwatch.StartNew();
bool wereEventsPublished = false;
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
Console.WriteLine("The event batch has been published.");
}
}
if (currentEvent.Data != null)
{
receivedEvent = currentEvent;
watch.Stop();
break;
}
}
Console.WriteLine();
Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:");
string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
Console.WriteLine();
}
