private static TimeSpan maxWaitTimeForFirstEvent = new TimeSpan(0, 0, 10); // 10 seconds
/// <summary>
/// Saves all data from all partitions of an eventhub consumer group
/// </summary>
/// <param name="ehNsConnectionString">Connection string for the EventHub Namespace</param>
/// <param name="ehName">Name of the EventHub</param>
/// <param name="ehConsumerGroup">The consumer group of the EventHub</param>
/// <param name="outputFilename">The x dimension size to crop the picture. The default is 0 indicating no cropping is required.</param>
static async Task Main(string ehNsConnectionString, string ehName, string ehConsumerGroup, FileInfo outputFilename)
{
Console.WriteLine($"EH-Name {ehName}");
Console.WriteLine($"EH-Consumer Group {ehConsumerGroup}");
EventHubConsumerClient ehClient = new EventHubConsumerClient(ehConsumerGroup, ehNsConnectionString, ehName);
var ehPartitions = await ehClient.GetPartitionIdsAsync();
Console.WriteLine($"Reading data from {ehPartitions.Length} partitions");
// configure data retrieval
var readOptions = new ReadEventOptions();
readOptions.MaximumWaitTime = maxWaitTimeForFirstEvent;
var ehEvents = ehClient.ReadEventsAsync(readOptions);
// delete target file if it exists
if (outputFilename.Exists)
{
outputFilename.Delete();
}
// grab data
using (var outStream = new FileStream(outputFilename.FullName, FileMode.Append)) {
await foreach (PartitionEvent ehEvent in ehEvents)
{
byte[] data = ehEvent.Data.Body.ToArray();
Console.WriteLine($"Retrieved {data.Length} bytes");
outStream.Write(data, 0, data.Length);
}
}
}
public async Task ReadPartitionTrackLastEnqueued()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
var options = new ReadEventOptions
{
TrackLastEnqueuedEventProperties = true
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options,
cancellationSource.Token))
{
LastEnqueuedEventProperties properties =
partitionEvent.Partition.ReadLastEnqueuedEventProperties();
Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }");
Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }");
Debug.WriteLine($"\tThe last offset is: { properties.Offset }");
Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC.");
Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC.");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task ReadAllPartitionsWaitTime()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadAllPartitionsWaitTime
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
int loopTicks = 0;
int maximumTicks = 10;
var options = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(options))
{
if (partitionEvent.Data != null)
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
else
{
Debug.WriteLine("Wait time elapsed; no event was available.");
}
loopTicks++;
if (loopTicks >= maximumTicks)
{
break;
}
}
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public void CloneProducesACopy()
{
var options = new ReadEventOptions
{
OwnerLevel = 99,
TrackLastEnqueuedEventProperties = false,
MaximumWaitTime = TimeSpan.FromMinutes(65)
};
ReadEventOptions clone = options.Clone();
Assert.That(clone, Is.Not.Null, "The clone should not be null.");
Assert.That(clone.OwnerLevel, Is.EqualTo(options.OwnerLevel), "The owner level of the clone should match.");
Assert.That(clone.TrackLastEnqueuedEventProperties, Is.EqualTo(options.TrackLastEnqueuedEventProperties), "The tracking of last event information of the clone should match.");
Assert.That(clone.MaximumWaitTime, Is.EqualTo(options.MaximumWaitTime), "The default maximum wait time of the clone should match.");
}
public async Task StartListeningAsync(int maxBatchSize, TimeSpan waitTimeout, CancellationToken cancellationToken)
{
var readOptions = new ReadEventOptions
{
MaximumWaitTime = waitTimeout,
};
await using var client = new EventHubConsumerClient(consumerGroup: "$default", connectionString: connectionString);
await foreach (var receivedEvent in client.ReadEventsAsync(startReadingAtEarliestEvent:false, readOptions, cancellationToken))
{
if (receivedEvent.Data == null)
{
// timeout. Yield and try again.
await Task.Yield();
continue;
}
await ProcessAsync(Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray()), cancellationToken);
}
}
public void CloneProducesACopy()
{
var options = new ReadEventOptions
{
OwnerLevel = 99,
TrackLastEnqueuedEventProperties = false,
MaximumWaitTime = TimeSpan.FromMinutes(65),
CacheEventCount = 1,
PrefetchCount = 0,
PrefetchSizeInBytes = 0
};
ReadEventOptions clone = options.Clone();
Assert.That(clone, Is.Not.Null, "The clone should not be null.");
Assert.That(clone.OwnerLevel, Is.EqualTo(options.OwnerLevel), "The owner level of the clone should match.");
Assert.That(clone.TrackLastEnqueuedEventProperties, Is.EqualTo(options.TrackLastEnqueuedEventProperties), "The tracking of last event information of the clone should match.");
Assert.That(clone.MaximumWaitTime, Is.EqualTo(options.MaximumWaitTime), "The maximum wait time of the clone should match.");
Assert.That(clone.CacheEventCount, Is.EqualTo(options.CacheEventCount), "The event cache count of the clone should match.");
Assert.That(clone.PrefetchCount, Is.EqualTo(options.PrefetchCount), "The prefetch count of the clone should match.");
Assert.That(clone.PrefetchSizeInBytes, Is.EqualTo(options.PrefetchSizeInBytes), "The prefetch size of the clone should match.");
}
//Read all events based on partitionId
public async Task ConsumerReadEventPartitionEvent(string consumerGroup, string partitionId)
{
try
{
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
EventHubConsumerClient eventConsumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName);
ReadEventOptions readEventOptions = new ReadEventOptions()
{
MaximumWaitTime = TimeSpan.FromSeconds(30)
};
await foreach (PartitionEvent partitionEvent in eventConsumer.ReadEventsFromPartitionAsync(partitionId, EventPosition.Latest, readEventOptions, cancellationSource.Token))
{
Console.WriteLine("---Execution from ConsumerReadEventPartitionEvent method---");
Console.WriteLine("------");
if (partitionEvent.Data != null)
{
Console.WriteLine("Event Data recieved {0} ", Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray()));
if (partitionEvent.Data.Properties != null)
{
foreach (var keyValue in partitionEvent.Data.Properties)
{
Console.WriteLine("Event data key = {0}, Event data value = {1}", keyValue.Key, keyValue.Value);
}
}
}
}
await Task.CompletedTask;
}
catch (Exception exp)
{
Console.WriteLine("Error occruied {0}. Try again later", exp.Message);
}
}
public void MaximumWaitTimeAllowsNull()
{
var options = new ReadEventOptions();
Assert.That(() => options.MaximumWaitTime = null, Throws.Nothing);
}
public async Task ReadPartitionWaitTime()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionWaitTime
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
int loopTicks = 0;
int maximumTicks = 10;
var options = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options))
{
if (partitionEvent.Data != null)
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
else
{
Debug.WriteLine("Wait time elapsed; no event was available.");
}
loopTicks++;
if (loopTicks >= maximumTicks)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// In this example, our consumer will read from the latest position instead of the earliest. As a result, it won't see events that
// have previously been published. Before we can publish the events and have them observed, we will need to ask the consumer
// to perform a read operation in order for it to begin observing the Event Hub partitions.
//
// Each partition of an Event Hub represents potentially infinite stream of events. When a consumer is reading, there is no definitive
// point where it can assess that all events have been read and no more will be available. As a result, when the consumer reaches the end of
// the available events for a partition, it will continue to wait for new events to arrive so that it can surface them to be processed. During this
// time, the iterator will block.
//
// In order to prevent the consumer from waiting forever for events, and blocking other code, there are two methods available for developers to
// control this behavior. First, signaling the cancellation token passed when reading will cause the consumer to stop waiting and end iteration
// immediately. This is desirable when you have decided that you are done reading and do not wish to continue. It is not ideal, however, when
// you would like control returned to your code momentarily to perform some action and then to continue reading.
//
// In that scenario, you may specify a maximum wait time which is applied to each iteration of the enumerator. If that interval passes without an
// event being available to read, the enumerator will emit an empty event in order to return control to the loop body. This allows you to take action,
// such as sending a heartbeat, emitting telemetry, or simply exiting the loop.
//
// For our loop, we'll specify a small wait time when we begin reading, which will allow control to return to our code so that we may publish
// the events after we ensure the consumer is observing the partition.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
{
bool wereEventsPublished = false;
int eventBatchCount = 0;
List <EventData> receivedEvents = new List <EventData>();
// Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
// an empty event, so that our code has the chance to run without indefinite blocking.
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
// As a preventative measure, we'll also specify that cancellation should occur after 2 minutes, so that we don't iterate indefinitely
// in the event of a service error where the events we've published cannot be read.
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromMinutes(2));
// The reading of all events will default to the earliest events available in each partition; in order to begin reading at the
// latest event, we'll need to specify that reading should not start at earliest.
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsAsync(startReadingAtEarliestEvent: false, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")));
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
eventBatchCount = eventBatch.Count;
await Task.Delay(250);
Console.WriteLine("The event batch has been published.");
}
// Since we know that there was no event to observe for this iteration,
// we'll just skip to the next one.
continue;
}
// Because publishing and receiving events is asynchronous, the events that we published may not
// be immediately available for our consumer to see, so we'll have to guard against an empty event being sent
// if our wait time interval has elapsed before the consumer observed the events that we published.
if (currentEvent.Data != null)
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count >= eventBatchCount)
{
break;
}
}
}
// Print out the events that we received; the body is an encoded string; we'll recover the message by reversing the encoding process.
Console.WriteLine();
foreach (EventData currentEvent in receivedEvents)
{
string message = Encoding.UTF8.GetString(currentEvent.Body.ToArray());
Console.WriteLine($"\tEvent Message: \"{ message }\"");
}
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against</param>
/// <param name="tenantId">The Azure Active Directory tenant that holds the service principal</param>
/// <param name="clientId">The Azure Active Directory client identifier of the service principal</param>
/// <param name="secret">The Azure Active Directory secret of the service principal</param>
///
public async Task RunAsync(string fullyQualifiedNamespace,
string eventHubName,
string tenantId,
string clientId,
string secret)
{
// Service principal authentication is a means for applications to authenticate
// against Azure Active Directory and consume Azure services. This is advantageous compared
// to signing in using fully privileged users as it allows to enforce role-based authorization
// from the portal.
//
// Service principal authentication differs from managed identity authentication also because the principal to be used
// will be able to authenticate with the portal without the need for the code to run within the portal.
// The authentication between the Event Hubs client and the portal is performed through OAuth 2.0.
// (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals)
// ClientSecretCredential allows performing service principal authentication passing
// tenantId, clientId and clientSecret directly from the constructor.
// (see: https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow)
var credentials = new ClientSecretCredential(tenantId, clientId, secret);
// EventHubProducerClient takes ClientSecretCredential from its constructor and tries to issue a token from Azure Active Directory.
await using (EventHubProducerClient client = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
{
// It will then use that token to authenticate on the portal and enquiry the Hub properties.
Console.WriteLine($"Contacting the hub using the token issued from client credentials.");
var properties = await client.GetEventHubPropertiesAsync();
Console.WriteLine($"Event Hub \"{ properties.Name }\" reached successfully.");
}
// The instances of "EventHubProducerClient" and "EventHubProducerClient" will be created
// passing in "ClientSecretCredential" instead of the connection string. In this way, two things will happen:
//
// 1. An OAuth 2.0 token will be created by authenticating against Azure Active Directory using the tenant, client and the secret passed in
// 2. Role-based authorization will be performed and the "Azure Event Hubs Data Owner" role will be needed to produce and consume events
//
// (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-definitions)
// (see: https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-event-hubs-data-owner)
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credentials))
{
string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();
PartitionEvent receivedEvent;
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
Stopwatch watch = Stopwatch.StartNew();
bool wereEventsPublished = false;
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credentials))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
Console.WriteLine("The event batch has been published.");
}
}
if (currentEvent.Data != null)
{
receivedEvent = currentEvent;
watch.Stop();
break;
}
}
Console.WriteLine();
Console.WriteLine($"The following event was consumed in { watch.ElapsedMilliseconds } milliseconds:");
string message = (receivedEvent.Data == null) ? "No event was received." : Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// In this example, we'll make use of multiple clients in order to publish an event that we will then read back and use as the starting point
// for reading events in the partition. Our initial consumer will begin watching for new events published to the first partition in our Event
// Hub. Before we can publish events and have them observed, we will need to ask the consumer to perform an operation for it to begin observing
// the partition.
//
// Each event that a consumer reads will have attributes set that describe the event's location in the partition, such as its offset, sequence
// number, and the date/time that it was enqueued. These attributes can be used to create a new consumer that begins consuming at a known position.
//
// With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed,
// and to manage where in the partition the consumer begins reading events. This is done by using the position information to track
// state, commonly known as "creating a checkpoint."
//
// The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the
// consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that
// begins reading at the position where it left off.
//
// It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed
// consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues.
// Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart,
// there is a small possibility that the service will return an event multiple times.
//
// To demonstrate, we will publish a batch of events to be read by an initial consumer. The third event that is read will be captured
// and another consumer will use its attributes to start reading the event that follows, reading the set of events that we published skipping over
// the first three.
string firstPartition;
EventData thirdEvent;
int eventBatchSize = 50;
await using (var initialConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
{
// We will start by using the consumer client inspect the Event Hub and select the first partition to operate against.
firstPartition = (await initialConsumerClient.GetPartitionIdsAsync()).First();
// Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
// an empty event, so that our code has the chance to run without indefinite blocking.
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
// As a preventative measure, we'll also specify that cancellation should occur after 30 seconds, so that we don't iterate indefinitely
// in the event of a service error where the events we've published cannot be read.
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
List <EventData> receivedEvents = new List <EventData>();
bool wereEventsPublished = false;
await foreach (PartitionEvent currentEvent in initialConsumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
// When we publish the event batch, we'll specify the partition to ensure that our consumer and producer clients are
// operating against the same partition.
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
for (int index = 0; index < eventBatchSize; ++index)
{
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }")));
}
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
await Task.Delay(250);
Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
}
// Since we know that there was no event to observe for this iteration,
// we'll just skip to the next one.
continue;
}
// Because publishing and reading events is asynchronous, the events that we published may not
// be immediately available for our consumer to see, so we'll have to guard against an empty event being sent as
// punctuation if our actual event is not available within the waiting time period.
if (currentEvent.Data != null)
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count >= eventBatchSize)
{
break;
}
}
}
// Print out the events that we read, which will be the entire set that
// we had published.
Console.WriteLine();
Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tEvent Message: \"{ message }\"");
}
// Remember the third event that was consumed.
thirdEvent = receivedEvents[2];
}
// At this point, our initial consumer client has passed its "using" scope and has been safely disposed of.
//
// Create a new consumer for the partition, specifying the sequence number of the third event as the location to begin reading. Because sequence numbers are non-inclusive
// by default, the consumer will read the next available event following that sequence number, allowing it to read the set of published events beginning with the fourth one.
//
// Because our second consumer will begin watching the partition at a specific event, there is no need to ask for an initial operation to set our place; when
// we begin iterating, the consumer will locate the proper place in the partition to read from.
await using (var newConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
{
// We will consume the events using the new consumer until all of the published events have been received.
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
List <EventData> receivedEvents = new List <EventData>();
int expectedCount = (eventBatchSize - 3);
EventPosition startingPosition = EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value);
await foreach (PartitionEvent currentEvent in newConsumerClient.ReadEventsFromPartitionAsync(firstPartition, startingPosition, cancellationSource.Token))
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count >= expectedCount)
{
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine();
Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tEvent Message: \"{ message }\"");
}
}
// At this point, our clients and connection have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Starts running a task responsible for receiving and processing events in the context of a specified partition.
/// </summary>
///
/// <param name="partitionId">The identifier of the Event Hub partition the task is associated with. Events will be read only from this partition.</param>
/// <param name="startingPosition">The position within the partition where the task should begin reading events.</param>
/// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param>
/// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param>
/// <param name="trackLastEnqueuedEventProperties">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns>
///
protected virtual Task RunPartitionProcessingAsync(string partitionId,
EventPosition startingPosition,
TimeSpan?maximumReceiveWaitTime,
EventHubsRetryOptions retryOptions,
bool trackLastEnqueuedEventProperties,
CancellationToken cancellationToken = default)
{
// TODO: should the retry options used here be the same for the abstract RetryPolicy property?
Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId));
Argument.AssertNotNull(retryOptions, nameof(retryOptions));
return(Task.Run(async() =>
{
// TODO: should we double check if a previous run already exists and close it? We have a race condition. Maybe we should throw in case another task exists.
var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var taskCancellationToken = cancellationSource.Token;
ActivePartitionProcessorTokenSources[partitionId] = cancellationSource;
// Context is set to default if operation fails. This shouldn't fail unless the user tries processing
// a partition they don't own.
PartitionContexts.TryGetValue(partitionId, out var context);
var clientOptions = new EventHubConsumerClientOptions
{
RetryOptions = retryOptions
};
var readOptions = new ReadEventOptions
{
MaximumWaitTime = maximumReceiveWaitTime,
TrackLastEnqueuedEventProperties = trackLastEnqueuedEventProperties
};
await using var connection = CreateConnection();
await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, clientOptions))
{
await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, readOptions, taskCancellationToken))
{
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled &&
partitionEvent.Data != null &&
EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
diagnosticScope.Start();
try
{
await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
throw;
}
}
}
}));
}
public async Task Receive(string connectionString, CancellationToken cancellationToken)
{
// In here, our consumer will read from the latest position instead of the earliest. As a result, it won't see events that
// have previously been published.
//
// Each partition of an Event Hub represents a potentially infinite stream of events. When a consumer is reading, there is no definitive
// point where it can assess that all events have been read and no more will be available. As a result, when the consumer reaches the end of
// the available events for a partition, it will continue to wait for new events to arrive so that it can surface them to be processed. During this
// time, the iterator will block.
//
// In order to prevent the consumer from waiting forever for events, and blocking other code, there is a method available for developers to
// control this behavior. It's signaling the cancellation token passed when reading will cause the consumer to stop waiting and end iteration
// immediately. This is desirable when you have decided that you are done reading and do not wish to continue.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString))
{
Console.WriteLine("The application will now start to listen for incoming message.");
// Each time the consumer looks to read events, we'll ask that it waits only a short time before emitting
// an empty event, so that our code has the chance to run without indefinite blocking.
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(500)
};
using (var csvOutput = File.CreateText("./result.csv"))
{
csvOutput.WriteLine("EventCount,BatchCount,BatchFrom,BatchTo,MinLatency,MaxLatency,AvgLatency");
try
{
// The client is safe and intended to be long-lived.
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsAsync(readOptions, cancellationToken))
{
var eventData = currentEvent.Data;
// Because publishing and receiving events is asynchronous, the events that published may not
// be immediately available for our consumer to see, so we'll have to guard against an empty event being sent
if (eventData == null)
{
continue;
}
int eventCount = 0;
var listTimeSpan = new List <TimeSpan>();
var listDateTime = new List <DateTimeOffset>();
var eventBody = Encoding.UTF8.GetString(eventData.Body.ToArray()).Split('\n');
foreach (var bodyInfo in eventBody)
{
try
{
var message = JsonSerializer.Deserialize <Dictionary <string, object> >(bodyInfo);
eventCount++;
var timeCreated = DateTime.Parse(message["createdAt"].ToString());
var timeIn = DateTime.Parse(message["EventEnqueuedUtcTime"].ToString());
var timeProcessed = DateTime.Parse(message["EventProcessedUtcTime"].ToString());
var timeAsaProcessed = DateTime.Parse(message["ASAProcessedUtcTime"].ToString());
var timeOut = eventData.EnqueuedTime.UtcDateTime.ToLocalTime();
listDateTime.Add(timeIn);
var elapsed = timeOut - timeIn;
listTimeSpan.Add(elapsed);
}
catch (Exception ex)
{
Console.WriteLine("Error while parsing event body.");
Console.WriteLine("Error:" + ex.Message);
Console.WriteLine("Message:" + bodyInfo);
}
}
var batchFrom = listDateTime.Min();
var batchTo = listDateTime.Min();
var minLatency = listTimeSpan.Min().TotalMilliseconds;
var maxLatency = listTimeSpan.Max().TotalMilliseconds;
var avgLatency = Math.Round(listTimeSpan.Average(ts => ts.TotalMilliseconds), 0);
Console.Write($"Received {eventCount} events.");
Console.Write($"\tBatch (From/To): {batchFrom.ToString("HH:mm:ss.ffffff")}/{batchTo.ToString("HH:mm:ss.ffffff")}");
Console.Write($"\tElapsed msec (Min/Max/Avg): {minLatency}/{maxLatency}/{avgLatency}");
Console.WriteLine();
csvOutput.WriteLine($"{eventCount},{eventCount},{batchFrom.ToString("o")},{batchTo.ToString("o")},{minLatency},{maxLatency},{avgLatency}");
}
}
catch (TaskCanceledException)
{
// This is okay because the task was cancelled.
}
}
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
}
static async Task <int> Main(string[] args)
{
var app = new CommandLineApplication();
app.FullName = "Event Hubs Consumer";
app.Description = "Event Hubs Consumer Sample";
app.HelpOption("-h|-?|--help");
var hubName = String.Empty;
var connectionString = app.Option("-cs", "Specify ConnectionString", CommandOptionType.SingleValue, true);
var group = app.Option("-cg", "Specify Consumer Group", CommandOptionType.SingleValue);
var hub = app.Option("-hub", "Specify Event Hub", CommandOptionType.SingleValue);
var allEvents = app.Option("-all", "Read all events", CommandOptionType.NoValue);
app.Execute(args);
if (!connectionString.HasValue() || string.IsNullOrEmpty(connectionString.Value()))
{
ShowHelp();
return(0);
}
var csValue = connectionString.Value().Replace("\"", "").Trim();
var csValueItems = csValue.Split(";");
foreach (var csValueItem in csValueItems)
{
var valuePair = csValueItem.Split("=");
if (valuePair[0].ToLower() == "entitypath")
{
hubName = valuePair[1];
break;
}
}
if (hubName == String.Empty)
{
if (!hub.HasValue() || string.IsNullOrEmpty(hub.Value()))
{
ShowHelp();
return(0);
}
else
{
hubName = hub.Value().Replace("\"", "").Trim();
}
}
//
// https://docs.microsoft.com/en-us/dotnet/api/azure.messaging.eventhubs.consumer.eventhubconsumerclient?view=azure-dotnet
// Select Consumer Group
//
string consumerGroup;
if (group.HasValue() && !string.IsNullOrEmpty(group.Value()))
{
consumerGroup = group.Value().Replace("\"", "").Trim();
}
else
{
consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
}
Console.WriteLine("Connection String : {0}....", connectionString.Value().Replace("\"", "").Trim().Substring(0, 50));
Console.WriteLine($"Event Hub : {hubName}");
Console.WriteLine($"Consumer Group : {consumerGroup}");
Console.WriteLine($"Read all events : {allEvents.HasValue()}");
Console.WriteLine($"Timeout : {readTimetoutSeconds} seconds");
EventHubConsumerClient consumer = new EventHubConsumerClient(consumerGroup, connectionString.Value().Replace("\"", "").Trim(), hubName);
var part = await consumer.GetPartitionIdsAsync();
var part_prop = await consumer.GetPartitionPropertiesAsync(part[0]);
var hub_prop = await consumer.GetEventHubPropertiesAsync();
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(readTimetoutSeconds));
//
// https://docs.microsoft.com/en-us/dotnet/api/azure.messaging.eventhubs.consumer.eventhubconsumerclient.readeventsasync?view=azure-dotnet
//
// Timeout 10 Min
//
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(600)
};
try
{
Console.WriteLine($"Read Start : {DateTime.Now.ToString()}");
Console.WriteLine("Waiting for events. CTRL+C to exit");
//
// If this is IoT Hub Eventhub Compatible Endpoint, device id is in ev.Data.SystemProperties["iothub-connection-device-id"]
//
await foreach (PartitionEvent ev in consumer.ReadEventsAsync(startReadingAtEarliestEvent: allEvents.HasValue(), readOptions, cancellationSource.Token))
{
Console.WriteLine("Enqueue at {0:yyyy/MM/dd H:mm:ss:fff} | Seq # {1:D4} | Partition {2} | Offset : {3:D6} | {4}",
ev.Data.EnqueuedTime,
ev.Data.SequenceNumber,
ev.Partition.PartitionId,
ev.Data.Offset,
Encoding.UTF8.GetString(ev.Data.Body.ToArray())
);
}
}
catch (Exception ex)
{
Console.WriteLine($"Exception : {ex.Message}");
}
Console.WriteLine($"Read Exit : {DateTime.Now.ToString()}");
return(1);
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
// consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
// would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an
// offset from the beginning, the next event available after a specific point in time, or at a specific event.
//
// In this example, we will create our consumer client using the default consumer group that is created with an Event Hub.
// Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName))
{
// We will start by using the consumer client inspect the Event Hub and select a partition to operate against to ensure that events are being
// published and read from the same partition.
string firstPartition = (await consumerClient.GetPartitionIdsAsync()).First();
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events and have them observed, we will need to ask the consumer
// to perform an operation, because it opens its connection only when it needs to.
//
// When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed,
// whether or not an event is available in the partition. If no event was available a null value will be emitted instead.
// This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event
// processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue
// if events are not arriving.
//
// We'll begin to iterate on the partition using a small wait time, so that control will return to our loop even when
// no event is available. For the first call, we'll publish so that we can receive them.
//
// For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more
// event than we published, which is expected to be a null value triggered by exceeding the wait time.
//
// To be sure that we do not block forever waiting on an event that is not published, we will request cancellation after a
// fairly long interval.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
ReadEventOptions readOptions = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(250)
};
bool wereEventsPublished = false;
int eventBatchCount = 0;
List <EventData> receivedEvents = new List <EventData>();
Stopwatch watch = Stopwatch.StartNew();
await foreach (PartitionEvent currentEvent in consumerClient.ReadEventsFromPartitionAsync(firstPartition, EventPosition.Latest, readOptions, cancellationSource.Token))
{
if (!wereEventsPublished)
{
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(new CreateBatchOptions { PartitionId = firstPartition });
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")));
await producerClient.SendAsync(eventBatch);
wereEventsPublished = true;
eventBatchCount = eventBatch.Count;
await Task.Delay(250);
Console.WriteLine("The event batch has been published.");
}
}
else
{
receivedEvents.Add(currentEvent.Data);
if (receivedEvents.Count > eventBatchCount)
{
watch.Stop();
break;
}
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
