/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// The Event Processor client offers additional options on creation, allowing you to control different aspects of its behavior
// should your scenario have needs that differ from the common defaults. If you choose not to provide these options, the default behaviors
// suitable to most scenarios are used.
//
// The processor allows you to customize how it interacts with the Event Hubs service, such as by influencing how it connects
// to the service by specifying the transport that communication should use and whether a proxy should be used for network communications. Please
// note that a proxy is only supported when using WebSockets as a transport; it isn't compatible with raw TCP or other transport channels.
//
// This sample will customize the transport for the connection, using WebSockets and will adjust some of the retry-related values for
// illustration.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
var processorOptions = new EventProcessorClientOptions
{
ConnectionOptions = new EventHubConnectionOptions
{
TransportType = EventHubsTransportType.AmqpWebSockets,
Proxy = (IWebProxy)null
},
RetryOptions = new EventHubsRetryOptions
{
MaximumRetries = 5,
TryTimeout = TimeSpan.FromMinutes(1)
}
};
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, processorOptions);
// Once the client has been created, it may be used normally. For completeness, we'll
// mirror the minimal skeleton from our first sample, starting and stopping the processor
// without performing any processing.
Task processEventHandler(ProcessEventArgs eventArgs) => Task.CompletedTask;
Task processErrorHandler(ProcessErrorEventArgs eventArgs) => Task.CompletedTask;
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
private static async Task Main(string[] args)
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: false)
.Build();
var logger = LoggerFactory.Create(logger => logger.AddConsole()).CreateLogger <Program>();
var storageClient = new BlobContainerClient(
configuration.GetValue <string>("BlobStorageConnectionString"),
configuration.GetValue <string>("BlobContainerName"));
var processor = new EventProcessorClient(
storageClient,
configuration.GetValue <string>("EventHubConsumerGroup"),
configuration.GetValue <string>("EventHubConnectionString"),
configuration.GetValue <string>("EventHubName"));
processor.ProcessEventAsync += async eventArgs =>
{
try
{
var sensorObservation = eventArgs.Data.EventBody.ToObjectFromJson <SensorObservation>(JsonSerializerOptions);
logger.LogInformation($"Processed a sensor observation:\n" +
$"\tValue:\t\t{sensorObservation.Observation.Value},\n" +
$"\tQuantity Kind:\t{sensorObservation.Observation.QuantityKind},\n" +
$"\tSensor Id:\t{sensorObservation.Observation.SensorId},\n");
logger.LogInformation("Press any key to read next one");
Console.ReadKey(true);
await Task.Yield();
}
catch (Exception e)
{
logger.LogError(e, $"Failed to processed a message with id: {eventArgs.Data.MessageId}");
}
await eventArgs.UpdateCheckpointAsync(eventArgs.CancellationToken);
};
processor.ProcessErrorAsync += async eventArgs =>
{
logger.LogError(eventArgs.Exception, $"Failed to processed a message with on partition: {eventArgs.PartitionId}");
await Task.Yield();
};
await processor.StartProcessingAsync();
logger.LogInformation($"You are listening to EventHub stream. Press Escape to stop listener.");
while (Console.ReadKey(true) is ConsoleKeyInfo key)
{
if (key.Key == ConsoleKey.Escape)
{
break;
}
}
await processor.StopProcessingAsync();
}
private static async Task SendAndReceiveEvents()
{
var partitionId = (await sender.GetPartitionIdsAsync()).First();
var events = new[]
{
new EventData(Encoding.UTF8.GetBytes(".NET event 1")),
new EventData(Encoding.UTF8.GetBytes(".NET event 2")),
new EventData(Encoding.UTF8.GetBytes(".NET event 3"))
};
var eventBatch = await sender.CreateBatchAsync();
foreach (var ev in events)
{
eventBatch.TryAdd(ev);
}
Console.Write("Ready to send a batch of " + eventBatch.Count.ToString() + " events... ");
await sender.SendAsync(eventBatch);
Console.Write("Sent\n");
Console.WriteLine("Receiving events... ");
var receivedEventCount = 0;
var readCancellationSource = new CancellationTokenSource();
readCancellationSource.CancelAfter(TimeSpan.FromSeconds(ReadTimeoutInSeconds));
Task processEvent(ProcessEventArgs eventArgs)
{
// Only display the first few events received to prevent
// filling the console with redundant messages
if (++receivedEventCount <= eventBatch.Count)
{
var bodyReader = new StreamReader(eventArgs.Data.BodyAsStream);
var bodyContents = bodyReader.ReadToEnd();
Console.WriteLine("Received Event: {0}", bodyContents);
}
return(Task.CompletedTask);
}
Task processError(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine("Error has occurred: {0}", eventArgs.Exception.ToString());
readCancellationSource.Cancel();
throw new Exception("Error while processing events", eventArgs.Exception);
}
processor.ProcessEventAsync += processEvent;
processor.ProcessErrorAsync += processError;
try
{
await processor.StartProcessingAsync();
while ((!readCancellationSource.IsCancellationRequested) && (receivedEventCount <= eventBatch.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
await processor.StopProcessingAsync();
}
finally
{
processor.ProcessEventAsync -= processEvent;
processor.ProcessErrorAsync -= processError;
}
}
public async Task SharedAccessSignature()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_SharedAccessSignature
#if SNIPPET
var credential = new AzureSasCredential("<< SHARED ACCESS KEY STRING >>");
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var fullyQualifiedNamespace = "<< NAMESPACE (likely similar to {your-namespace}.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var blobContainerName = storageScope.ContainerName;
var fullyQualifiedNamespace = EventHubsTestEnvironment.Instance.FullyQualifiedNamespace;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
var resource = $"amqps://{ EventHubsTestEnvironment.Instance.FullyQualifiedNamespace }/{ eventHubScope.EventHubName }".ToLowerInvariant();
var signature = new SharedAccessSignature(resource, EventHubsTestEnvironment.Instance.SharedAccessKeyName, EventHubsTestEnvironment.Instance.SharedAccessKey);
var credential = new AzureSasCredential(signature.Value);
#endif
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
fullyQualifiedNamespace,
eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ConnectionStringParse()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_ConnectionStringParse
TokenCredential credential = new DefaultAzureCredential();
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
/*@@*/ credential = EventHubsTestEnvironment.Instance.Credential;
var storageEndpoint = new BlobServiceClient(storageConnectionString).Uri;
var blobUriBuilder = new BlobUriBuilder(storageEndpoint);
blobUriBuilder.BlobContainerName = blobContainerName;
var storageClient = new BlobContainerClient(
blobUriBuilder.ToUri(),
credential);
EventHubsConnectionStringProperties properties =
EventHubsConnectionStringProperties.Parse(eventHubsConnectionString);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
properties.FullyQualifiedNamespace,
properties.EventHubName ?? eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// A checkpoint is the term used to describe a snapshot of the state of processing for a partition which has been persisted
// to durable storage and which allows an Event Processor client to resume processing at a specific location in the partition's
// event stream. When a processor starts, it will seek out an existing checkpoint and, if found, use that as the place where it
// begins reading events. If no checkpoint is found, a default location is used.
//
// Checkpoints are intended as a means to allow an Event Processor client or cluster of processors for a consumer group to coordinate
// on which events were processed, so that processors can dynamically start, stop, join the cluster, and leave the cluster without the
// need to start processing at the beginning of a partition and revisit all of its events.
//
// A checkpoint is based on an event and represents the last event that should be considered as processed for the partition; should a processor
// start with that checkpoint, the next available event would be used as the starting point.
//
// The creation of checkpoints comes at a cost, both in terms of processing performance/throughput and a potential monetary cost associated with
// the underlying storage resource. While it may seem desirable to create checkpoints for each event that is processed, that is typically considered
// an anti-pattern for most scenarios.
//
// When deciding how frequently to checkpoint, you'll need to consider the trade-off between the costs of creating the checkpoint against the costs of
// processing events. For scenarios where processing events is very cheap, it is often a better approach to checkpoint once per some number of events or
// once per time interval. For scenarios where processing events is more expensive, it may be a better approach to checkpoint more frequently.
//
// In either case, it is important to understand that your processing must be tolerant of receiving the same event to be processed more than once; the
// Event Hubs service, like most messaging platforms, guarantees at-least-once delivery. Even were you to create a checkpoint for each event that you
// process, it is entirely possible that you would receive that same event again from the service.
var expectedEvents = new List<EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
};
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// With our events having been published, we'll create an Event Hub Processor to read them.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// When your handler for processing events is invoked, the set of event arguments that it are passed allow the associated event to be
// saved as a checkpoint for the processor. For this example, our handler will create a checkpoint once per batch of events that we
// sent.
int eventIndex = 0;
int eventsSinceLastCheckpoint = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
++eventIndex;
++eventsSinceLastCheckpoint;
if (eventsSinceLastCheckpoint >= eventsPerBatch)
{
// Updating the checkpoint will interact with the Azure Storage. As a service call,
// this is done asynchronously and may be long-running. You may want to influence its behavior,
// such as limiting the time that it may execute in order to ensure throughput for
// processing events.
//
// In our case, we'll limit the checkpoint operation to a second and request cancellation
// if it runs longer.
using CancellationTokenSource cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(1));
try
{
await eventArgs.UpdateCheckpointAsync(cancellationSource.Token);
eventsSinceLastCheckpoint = 0;
Console.WriteLine("Created checkpoint");
}
catch (TaskCanceledException)
{
Console.WriteLine("Checkpoint creation took too long and was canceled.");
}
Console.WriteLine();
}
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.Body.ToArray()) }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return Task.CompletedTask;
}
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return Task.CompletedTask;
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// In order to begin processing, an explicit call must be made to the processor. This will instruct the processor to begin
// processing in the background, invoking your handlers when they are needed.
eventIndex = 0;
await processor.StartProcessingAsync();
// Because processing takes place in the background, we'll continue to wait until all of our events were
// read and handled before stopping. To ensure that we don't wait indefinitely should an unrecoverable
// error be encountered, we'll also add a timed cancellation.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(60));
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints should be persisted.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to a handler delegate for processing using code that you
// provide.
//
// These handler delegates are invoked for each event or error that occurs during operation of the event processor. For
// a given partition, only a single event will be dispatched for processing at a time so that the order of events within a
// partition is preserved. Partitions, however, are processed concurrently. As a result, your handlers are potentially processing
// multiple events or errors at any given time.
// A partition manager may create checkpoints and list/claim partition ownership. A developer may implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of a MockCheckPointStorage to get started with using the `EventProcessor` but in production,
// you should choose an implementation of the PartitionManager interface that will
// store the checkpoints and partition ownership to a persistent store instead.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
var containerClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
// It's also possible to specify custom options upon event processor creation. We don't want to wait
// more than 1 second for every set of events.
var eventProcessorOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessorClient(containerClient, EventHubConsumerClient.DefaultConsumerGroupName, connectionString, eventHubName, eventProcessorOptions);
int totalEventsCount = 0;
int partitionsBeingProcessedCount = 0;
eventProcessor.PartitionInitializingAsync += (eventArgs) =>
{
// This is the last piece of code guaranteed to run before event processing, so all initialization
// must be done by the moment this method returns.
// We want to receive events from the latest available position so older events don't interfere with our sample.
eventArgs.DefaultStartingPosition = EventPosition.Latest;
Interlocked.Increment(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ eventArgs.PartitionId }': partition processing has started.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.PartitionClosingAsync += (eventArgs) =>
{
// The code to be run just before stopping processing events for a partition. This is the right place to dispose
// of objects that will no longer be used.
Interlocked.Decrement(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ eventArgs.PartitionId }': partition processing has stopped. Reason: { eventArgs.Reason }.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessEventAsync += (eventArgs) =>
{
// Here the user can specify what to do with the event received from the event processor. We are counting how
// many events were received across all partitions so we can check whether all sent events were received.
//
// It's important to notice that this method is called even when no event is received after the maximum wait time, which
// can be specified by the user in the event processor options. In this case, the received event is null.
if (eventArgs.Data != null)
{
Interlocked.Increment(ref totalEventsCount);
Console.WriteLine($"\tPartition '{ eventArgs.Partition.PartitionId }': event received.");
}
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessErrorAsync += (eventArgs) =>
{
// Any exception which occurs as a result of the event processor itself will be passed to
// this delegate so it may be handled. The processor will continue to process events if
// it is able to unless this handler explicitly requests that it stop doing so.
//
// It is important to note that this does not include exceptions during event processing; those
// are considered responsibility of the developer implementing the event processing handler. It
// is, therefore, highly encouraged that best practices for exception handling practices are
// followed with that delegate.
//
// This piece of code is not supposed to be reached by this sample. If the following message has been printed
// to the Console, then something unexpected has happened.
Console.WriteLine($"\tPartition '{ eventArgs.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartProcessingAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. This may take some time
// as there's a 10 seconds interval between claims. To be sure that we do not block forever in case the event processor
// fails, we will specify a fairly long time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
var partitionsCount = (await producerClient.GetPartitionIdsAsync()).Length;
while (partitionsBeingProcessedCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// The processor may take some time to connect to the Event Hubs service. Let's wait 1 second before sending
// events so we don't end up missing events.
await Task.Delay(1000);
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
int amountOfSets = 10;
int eventsPerSet = 2;
int expectedAmountOfEvents = amountOfSets * eventsPerSet;
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the second event.")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the first event.")));
await producerClient.SendAsync(eventBatch);
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopProcessingAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task InitializePartition()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_InitializePartition
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
Task initializeEventHandler(PartitionInitializingEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
// If no checkpoint was found, start processing
// events enqueued now or in the future.
EventPosition startPositionWhenNoCheckpoint =
EventPosition.FromEnqueuedTime(DateTimeOffset.UtcNow);
args.DefaultStartingPosition = startPositionWhenNoCheckpoint;
}
catch (Exception ex)
{
Application.HandleInitializeException(args, ex);
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers for processing events and errors are
// not relevant for this sample; for illustration, they're
// delegating the implementation to the host application.
processor.PartitionInitializingAsync += initializeEventHandler;
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup
processor.PartitionInitializingAsync -= initializeEventHandler;
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ProcessByBatch()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_ProcessByBatch
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
const int EventsInBatch = 50;
var partitionEventBatches = new ConcurrentDictionary <string, List <EventData> >();
var checkpointNeeded = false;
async Task processEventHandler(ProcessEventArgs args)
{
try
{
string partition = args.Partition.PartitionId;
List <EventData> partitionBatch =
partitionEventBatches.GetOrAdd(
partition,
new List <EventData>());
partitionBatch.Add(args.Data);
if (partitionBatch.Count >= EventsInBatch)
{
await Application.ProcessEventBatchAsync(
partitionBatch,
args.Partition,
args.CancellationToken);
checkpointNeeded = true;
partitionBatch.Clear();
}
if (checkpointNeeded)
{
await args.UpdateCheckpointAsync();
checkpointNeeded = false;
}
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ErrorHandlerCancellationRecovery()
{
#region Snippet:EventHubs_Processor_Sample03_ErrorHandlerCancellationRecovery
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = _storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _eventHubScope.EventHubName;
/*@@*/ consumerGroup = _eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// This token is used to control processing,
// if signaled, then processing will be stopped.
using var cancellationSource = new CancellationTokenSource();
/*@@*/ cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
Task processEventHandler(ProcessEventArgs args)
{
try
{
// Process the event.
}
catch
{
// Handle the exception.
}
return(Task.CompletedTask);
}
async Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
// Always log the exception.
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation ?? "Unknown" }");
Debug.WriteLine($"\tPartition: { args.PartitionId ?? "None" }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
// If cancellation was requested, assume that
// it was in response to an application request
// and take no action.
if (args.CancellationToken.IsCancellationRequested)
{
return;
}
// If out of memory, signal for cancellation.
if (args.Exception is OutOfMemoryException)
{
cancellationSource.Cancel();
return;
}
// If processing stopped and this handler determined
// the error to be non-fatal, restart processing.
if ((!processor.IsRunning) &&
(!cancellationSource.IsCancellationRequested))
{
// To be safe, request that processing stop before
// requesting the start; this will ensure that any
// processor state is fully reset.
await processor.StopProcessingAsync();
await processor.StartProcessingAsync(cancellationSource.Token);
}
}
catch
{
// Handle the exception. If fatal, signal
// for cancellation.
}
}
try
{
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// Once processing has started, the delay will
// block to allow processing until cancellation
// is requested.
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task CheckpointByEventCount()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_CheckpointByEventCount
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
const int EventsBeforeCheckpoint = 25;
var partitionEventCount = new ConcurrentDictionary <string, int>();
async Task processEventHandler(ProcessEventArgs args)
{
try
{
await Application.ProcessEventAsync(
args.Data,
args.Partition,
args.CancellationToken);
// If the number of events that have been processed
// since the last checkpoint was created exceeds the
// checkpointing threshold, a new checkpoint will be
// created and the count reset.
string partition = args.Partition.PartitionId;
int eventsSinceLastCheckpoint = partitionEventCount.AddOrUpdate(
key: partition,
addValue: 1,
updateValueFactory: (_, currentCount) => currentCount + 1);
if (eventsSinceLastCheckpoint >= EventsBeforeCheckpoint)
{
await args.UpdateCheckpointAsync();
partitionEventCount[partition] = 0;
}
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task EventHandlerStopOnException()
{
#region Snippet:EventHubs_Processor_Sample03_EventHandlerStopOnException
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = _storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _eventHubScope.EventHubName;
/*@@*/ consumerGroup = _eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// This token is used to control processing,
// if signaled, then processing will be stopped.
using var cancellationSource = new CancellationTokenSource();
/*@@*/ cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
// Process the event.
}
catch
{
// Handle the exception. If fatal,
// signal for cancellation.
cancellationSource.Cancel();
}
return(Task.CompletedTask);
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
// Process the error, as appropriate for the
// application.
return(Task.CompletedTask);
}
try
{
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// Once processing has started, the delay will
// block to allow processing until cancellation
// is requested.
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// Sometimes the environment you are targeting uses a service version that's different from the one set by default by the
// SDK. The SDK targets the Azure cloud by default, and you may need to manually specify a different version in case you
// are trying to run in another platform.
//
// For instance, this will happen when you try to run the Event Processor Client on an Azure Stack Hub version 2002. The
// processor makes use of the Azure Storage SDK, which sets a default Azure Storage service version that's different from
// the ones expected by the Azure Stack Hub. This sample illustrates how to work around this issue by using a pipeline
// policy to request the Azure Storage SDK to change its service version.
//
// In order to do this, create a Blob Client Options instance and add a Storage Api Version Policy to it. These options
// will be passed to the Blob Container Client used by the processor. The implementation of the policy is provided at
// the end of this sample.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobClientOptions storageClientOptions = new BlobClientOptions();
storageClientOptions.AddPolicy(new StorageApiVersionPolicy(), HttpPipelinePosition.PerCall);
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName, storageClientOptions);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// For this example, we'll create a simple event handler that writes to the
// console each time it was invoked.
int eventsProcessed = 0;
Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
if (eventArgs.HasEvent)
{
++eventsProcessed;
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.EventBody.ToArray()) }");
}
return(Task.CompletedTask);
}
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route it to a single partition.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, Single Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Single Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Single Batch"))
};
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < expectedEvents.Count; ++index)
{
eventBatch.TryAdd(expectedEvents[index]);
}
await producer.SendAsync(eventBatch);
}
Console.WriteLine("Starting the Event Processor client...");
Console.WriteLine();
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
while ((!cancellationSource.IsCancellationRequested) && (eventsProcessed < expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task StopAsync()
{
await _client.StopProcessingAsync();
}
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogWarning("IoTMessage Service will be stopped.");
// Stop the processing
await eventProcessorClient.StopProcessingAsync();
}
public async Task ProcessWithHeartbeat()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_ProcessWithHeartbeat
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processorOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(250)
};
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName,
processorOptions);
async Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.HasEvent)
{
await Application.ProcessEventAndCheckpointAsync(
args.Data,
args.Partition,
args.CancellationToken);
}
await Application.SendHeartbeatAsync(args.CancellationToken);
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take slightly longer than the length of
// time defined as part of the MaximumWaitTime configured
// for the processor; in this example, 250 milliseconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// In this example, our Event Processor client will be configured to use a maximum wait time which is considered when the processor is reading events.
// When events are available, the Event Processor client will pass them to the ProcessEvent handler to be processed. When no events are available in any
// partition for longer than the specified maximum wait interval, the processor will invoke the ProcessEvent handler with a set of arguments that do not
// include an event. This allows your handler code to receive control and perform actions such as sending a heartbeat, emitting telemetry, or another
// operation specific to your application needs.
//
// For our processor, we'll specify a small maximum wait time value as part of the options.
EventProcessorClientOptions clientOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, clientOptions);
// For this example, we'll create a simple event handler that writes to the
// console each time it was invoked.
int eventIndex = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
// The "HasEvent" property of the arguments will be set if an event was available from the
// Event Hubs service. If so, the argument properties for the event is populated and checkpoints
// may be created.
//
// If the "HasEvent" property is unset, the event will be empty and checkpoints may not be created.
if (eventArgs.HasEvent)
{
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.EventBody.ToArray()) }");
}
// Simulate sending a heartbeat using a simple helper that writes a status to the
// console.
await SendHeartbeatAsync();
++eventIndex;
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
Console.WriteLine();
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
// We'll publish a batch of events for our processor to receive. We'll split the events into a couple of batches to
// increase the chance they'll be spread around to different partitions and introduce a delay between batches to
// allow for the handler to be invoked without an available event interleaved.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
await Task.Delay(250, cancellationSource.Token);
}
}
// We'll allow the Event Processor client to read and dispatch the events that we published, along with
// ensuring a few invocations with no event. Note that, due to non-determinism in the timing, we may or may
// not see all of the events from our batches read.
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count + 5))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public Task StopProcessingAsync(CancellationToken cancellationToken = default)
{
return(_client.StopProcessingAsync(cancellationToken));
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// With our events having been published, we'll create an Event Hub Processor to read them.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// When creating a handler for processing events, it is important to note that you are responsible for ensuring that exception handling
// takes place within your handler code. Should an exception go unhandled, the processor will allow it to bubble and will not attempt
// to route it through the exception handler.
int eventIndex = 0;
Task processEventHandler(ProcessEventArgs eventArgs)
{
// The event arguments contain a cancellation token that the Event Processor client uses to signal
// your handler that processing should stop when possible. This is most commonly used in the
// case that the event processor is stopping or has otherwise encountered an unrecoverable problem.
//
// Each of the handlers should respect cancellation as they are able in order to ensure that the
// Event Processor client is able to perform its operations efficiently.
//
// In the case of the process event handler, the Event Processor client must await the result in
// order to ensure that the ordering of events within a partition is maintained. This makes respecting
// the cancellation token important.
//
// Also of note, because the Event Processor client must await this handler, you are unable to safely
// perform operations on the client, such as stopping or starting. Doing so is likely to result in a
// deadlock unless it is carefully queued as a background task.
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
// For our example, we'll just track that the event was received and write its data to the
// console.
//
// Because there is no long-running or I/O operation, inspecting the cancellation
// token again does not make sense in this scenario. However, in real-world processing, it is
// highly recommended that you do so as you are able. It is also recommended that the cancellation
// token be passed to any asynchronous operations that are awaited in this handler.
++eventIndex;
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.Body.ToArray()) }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
// Because our example handler is running synchronously, we'll manually return a completed
// task.
return(Task.CompletedTask);
};
// The error handler is invoked when there is an exception observed within the Event Processor client; it is not invoked for
// exceptions in your handler code. The Event Processor client will make every effort to recover from exceptions and continue
// processing. Should an exception that cannot be recovered from is encountered, the processor will forfeit ownership of all partitions
// that it was processing so that work may redistributed.
//
// The exceptions surfaced to this handler may be fatal or non-fatal; because the processor may not be able to accurately predict
// whether an exception was fatal or whether its state was corrupted, this handler has responsibility for making the determination as to
// whether processing should be terminated or restarted. The handler may do so by calling Stop on the processor instance and then, if desired,
// calling Start on the processor.
//
// It is recommended that, for production scenarios, the decision be made by considering observations made by this error handler, the
// handler invoked when initializing processing for a partition, and the handler invoked when processing for a partition is stopped. Many
// developers will also include data from their monitoring platforms in this decision as well.
//
// As with event processing, should an exception occur in your code for this handler, processor will allow it to bubble and will not attempt
// further action.
//
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
// As with the process event handler, the event arguments contain a cancellation token used by the Event Processor client to signal
// that the operation should be canceled. The handler should respect cancellation as it is able in order to ensure that the Event
// Processor client is able to perform its operations efficiently.
//
// The process error handler is not awaited by the Event Processor client and is, instead, executed in a fire-and-forget manner. This
// means that you may safely interact with the Event Processor client, such as requesting that it stop processing.
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
// Because there is no long-running or I/O operation, inspecting the cancellation token again does not make sense in this scenario.
// However, in real-world processing, it is recommended that you do so as you are able without compromising your ability to capture
// and troubleshooting information.
//
// It is also recommended that the cancellation token be passed to any asynchronous operations that are awaited in this handler.
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// In order to begin processing, an explicit call must be made to the processor. This will instruct the processor to begin
// processing in the background, invoking your handlers when they are needed.
eventIndex = 0;
await processor.StartProcessingAsync();
// Because processing takes place in the background, we'll continue to wait until all of our events were
// read and handled before stopping. To ensure that we don't wait indefinitely should an unrecoverable
// error be encountered, we'll also add a timed cancellation.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(60));
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public override async Task CleanupAsync()
{
await EventProcessorClient.StopProcessingAsync();
await base.CleanupAsync();
}
/// <summary>
/// Run the sample async.
/// </summary>
/// <param name="config">The param is of type ConfigWrapper. This class reads values from local configuration file.</param>
/// <returns></returns>
private static async Task RunAsync(ConfigWrapper config)
{
IAzureMediaServicesClient client;
try
{
client = await Authentication.CreateMediaServicesClientAsync(config, UseInteractiveAuth);
}
catch (Exception e)
{
Console.Error.WriteLine("TIP: Make sure that you have filled out the appsettings.json or .env file before running this sample.");
Console.Error.WriteLine($"{e.Message}");
return;
}
// Set the polling interval for long running operations to 2 seconds.
// The default value is 30 seconds for the .NET client SDK
client.LongRunningOperationRetryTimeout = 2;
// Creating a unique suffix so that we don't have name collisions if you run the sample
// multiple times without cleaning up.
string uniqueness = Guid.NewGuid().ToString("N");
string jobName = $"job-{uniqueness}";
string locatorName = $"locator-{uniqueness}";
string outputAssetName = $"output-{uniqueness}";
bool stopEndpoint = false;
// In this sample, we use Event Grid to listen to the notifications through an Azure Event Hub.
// If you do not provide an Event Hub config in the settings, the sample will fall back to polling the job for status.
// For production ready code, it is always recommended to use Event Grid instead of polling on the Job status.
EventProcessorClient processorClient = null;
BlobContainerClient storageClient = null;
MediaServicesEventProcessor mediaEventProcessor = null;
try
{
// Ensure that you have the desired encoding Transform. This is really a one time setup operation.
Transform transform = await GetOrCreateTransformAsync(client, config.ResourceGroup, config.AccountName, AdaptiveStreamingTransformName);
// Output from the encoding Job must be written to an Asset, so let's create one
Asset outputAsset = await CreateOutputAssetAsync(client, config.ResourceGroup, config.AccountName, outputAssetName);
Job job = await SubmitJobAsync(client, config.ResourceGroup, config.AccountName, AdaptiveStreamingTransformName, outputAsset.Name, jobName);
try
{
// First we will try to process Job events through Event Hub in real-time. If this fails for any reason,
// we will fall-back on polling Job status instead.
// Please refer README for Event Hub and storage settings.
// A storage account is required to process the Event Hub events from the Event Grid subscription in this sample.
// Create a new host to process events from an Event Hub.
Console.WriteLine("Creating a new client to process events from an Event Hub...");
var credential = new DefaultAzureCredential();
var storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
config.StorageAccountName, config.StorageAccountKey);
var blobContainerName = config.StorageContainerName;
var eventHubsConnectionString = config.EventHubConnectionString;
var eventHubName = config.EventHubName;
var consumerGroup = config.EventHubConsumerGroup;
storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
processorClient = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// Create an AutoResetEvent to wait for the job to finish and pass it to EventProcessor so that it can be set when a final state event is received.
AutoResetEvent jobWaitingEvent = new(false);
// Create a Task list, adding a job waiting task and a timer task. Other tasks can be added too.
IList <Task> tasks = new List <Task>();
// Add a task to wait for the job to finish. jobWaitingEvent will be set when a final state is received by EventProcessor.
Task jobTask = Task.Run(() => jobWaitingEvent.WaitOne());
tasks.Add(jobTask);
// 30 minutes timeout.
var cancellationSource = new CancellationTokenSource();
var timeout = Task.Delay(30 * 60 * 1000, cancellationSource.Token);
tasks.Add(timeout);
mediaEventProcessor = new MediaServicesEventProcessor(jobName, jobWaitingEvent, null);
processorClient.ProcessEventAsync += mediaEventProcessor.ProcessEventsAsync;
processorClient.ProcessErrorAsync += mediaEventProcessor.ProcessErrorAsync;
await processorClient.StartProcessingAsync(cancellationSource.Token);
// Wait for any task to finish.
if (await Task.WhenAny(tasks) == jobTask)
{
// Job finished. Cancel the timer.
cancellationSource.Cancel();
// Get the latest status of the job.
job = await client.Jobs.GetAsync(config.ResourceGroup, config.AccountName, AdaptiveStreamingTransformName, jobName);
}
else
{
// Timeout happened, Something might go wrong with job events. Fall-back on polling instead.
jobWaitingEvent.Set();
throw new Exception("Timeout happened.");
}
}
catch (Exception)
{
Console.WriteLine("Warning: Failed to connect to Event Hub, please refer README for Event Hub and storage settings.");
// Polling is not a recommended best practice for production applications because of the latency it introduces.
// Overuse of this API may trigger throttling. Developers should instead use Event Grid.
Console.WriteLine("Polling job status...");
job = await WaitForJobToFinishAsync(client, config.ResourceGroup, config.AccountName, AdaptiveStreamingTransformName, jobName);
}
if (job.State == JobState.Finished)
{
// Create the content key policy that configures how the content key is delivered to end clients
// via the Key Delivery component of Azure Media Services.
ContentKeyPolicy policy = await GetOrCreateContentKeyPolicyAsync(client, config.ResourceGroup, config.AccountName, ContentKeyPolicyName);
// Because this sample sets StreamingLocator.StreamingPolicyName to "Predefined_MultiDrmCencStreaming" policy,
// two content keys get generated and set on the locator.
StreamingLocator locator = await CreateStreamingLocatorAsync(client, config.ResourceGroup, config.AccountName, outputAsset.Name, locatorName, policy.Name);
StreamingEndpoint streamingEndpoint = await client.StreamingEndpoints.GetAsync(config.ResourceGroup,
config.AccountName, DefaultStreamingEndpointName);
if (streamingEndpoint.ResourceState != StreamingEndpointResourceState.Running)
{
await client.StreamingEndpoints.StartAsync(config.ResourceGroup, config.AccountName, DefaultStreamingEndpointName);
// Since we started the endpoint, we should stop it in cleanup.
stopEndpoint = true;
}
string dashPath = await GetDASHStreamingUrlAsync(client, config.ResourceGroup, config.AccountName, locator.Name, streamingEndpoint);
Console.WriteLine();
Console.WriteLine("Copy and paste the following URL in your browser to play back the file in the Azure Media Player.");
Console.WriteLine("You can use Edge/IE11 for PlayReady.");
Console.WriteLine();
Console.WriteLine($"https://ampdemo.azureedge.net/?url={dashPath}");
Console.WriteLine();
}
Console.WriteLine("When finished testing press enter to cleanup.");
Console.Out.Flush();
Console.ReadLine();
}
catch (ErrorResponseException e)
{
Console.WriteLine("ErrorResponseException");
Console.WriteLine($"\tCode: {e.Body.Error.Code}");
Console.WriteLine($"\tMessage: {e.Body.Error.Message}");
Console.WriteLine();
Console.WriteLine("Exiting, cleanup may be necessary...");
Console.ReadLine();
}
finally
{
Console.WriteLine("Cleaning up...");
await CleanUpAsync(client, config.ResourceGroup, config.AccountName, AdaptiveStreamingTransformName, outputAssetName,
jobName, ContentKeyPolicyName, stopEndpoint, DefaultStreamingEndpointName);
if (processorClient != null)
{
Console.WriteLine("Job final state received, Stopping the event processor...");
await processorClient.StopProcessingAsync();
Console.WriteLine();
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processorClient.ProcessEventAsync -= mediaEventProcessor.ProcessEventsAsync;
processorClient.ProcessErrorAsync -= mediaEventProcessor.ProcessErrorAsync;
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// With our events having been published, we'll create an Event Hub Processor to read them.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// Create a simple handler for event processing.
int eventIndex = 0;
Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
++eventIndex;
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.EventBody.ToArray()) }");
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
// Because our example handler is running synchronously, we'll manually return a completed
// task.
return(Task.CompletedTask);
};
// The error handler is invoked when there is an exception observed within the Event Processor client; it is not invoked for
// exceptions in your handler code. The Event Processor client will make every effort to recover from exceptions and continue
// processing. Should an exception that cannot be recovered from is encountered, the processor will forfeit ownership of all partitions
// that it was processing so that work may redistributed.
//
// For this example, arbitrarily choose to restart processing when an Event Hubs service exception is encountered and simply
// log other exceptions to the console.
//
// It is important to note that this selection is for demonstration purposes only; it does not constitute the recommended course
// of action for service errors. Because the right approach for handling errors can vary greatly between different types of
// application, you will need to determine the error handling strategy that best suits your scenario.
async Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
// As with the process event handler, the event arguments contain a cancellation token used by the Event Processor client to signal
// that the operation should be canceled. The handler should respect cancellation as it is able in order to ensure that the Event
// Processor client is able to perform its operations efficiently.
//
// The process error handler is not awaited by the Event Processor client and is, instead, executed in a fire-and-forget manner. This
// means that you may safely interact with the Event Processor client, such as requesting that it stop processing.
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
// Because there is no long-running or I/O operation, inspecting the cancellation token again does not make sense in this scenario.
// However, in real-world processing, it is recommended that you do so as you are able without compromising your ability to capture
// and troubleshooting information.
//
// It is also recommended that the cancellation token be passed to any asynchronous operations that are awaited in this handler.
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
// We will not pass the cancellation token from the event arguments here, as it will be
// signaled when we request that the processor stop.
if (eventArgs.Exception is EventHubsException)
{
Console.WriteLine("Detected an service error. Restarting the processor...");
await processor.StopProcessingAsync();
await processor.StartProcessingAsync();
Console.WriteLine("Processor has been restarted....");
Console.WriteLine();
}
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// Unfortunately, because the handler is invoked when exceptions are encountered in the Event Processor client and not for the code
// developers write in the event handlers, there is no reliable way to force an exception. As a result, it is unlikely that you will
// be able to observe the error handler in action by just running the Event Processor.
//
// Instead, we will wait a short bit to allow processing to take place and then artificially trigger the event handler for illustration
// purposes.
await Task.Delay(TimeSpan.FromMilliseconds(250));
Console.WriteLine("Triggering the error handler...");
ProcessErrorEventArgs eventArgs = new ProcessErrorEventArgs("fake", "artificial invoke", new EventHubsException(true, eventHubName), cancellationSource.Token);
await processErrorHandler(eventArgs);
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task DefaultAzureCredential()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_DefaultAzureCredential
#if SNIPPET
var credential = new DefaultAzureCredential();
var storageEndpoint = "<< STORAGE ENDPOINT (likely similar to {your-account}.blob.core.windows.net) >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var fullyQualifiedNamespace = "<< NAMESPACE (likely similar to {your-namespace}.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var credential = EventHubsTestEnvironment.Instance.Credential;
var storageEndpoint = new BlobServiceClient(StorageTestEnvironment.Instance.StorageConnectionString).Uri.ToString();
var blobContainerName = storageScope.ContainerName;
var fullyQualifiedNamespace = EventHubsTestEnvironment.Instance.FullyQualifiedNamespace;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
#endif
var blobUriBuilder = new BlobUriBuilder(new Uri(storageEndpoint));
blobUriBuilder.BlobContainerName = blobContainerName;
var storageClient = new BlobContainerClient(
blobUriBuilder.ToUri(),
credential);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
fullyQualifiedNamespace,
eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ErrorHandlerCancellationRecovery()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample03_ErrorHandlerCancellationRecovery
#if SNIPPET
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
#endif
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// This token is used to control processing,
// if signaled, then processing will be stopped.
using var cancellationSource = new CancellationTokenSource();
#if !SNIPPET
cancellationSource.CancelAfter(TimeSpan.FromSeconds(10));
#endif
Task processEventHandler(ProcessEventArgs args)
{
try
{
// TODO:
// Process the event according to application needs.
}
catch
{
// TODO:
// Take action to handle the exception.
//
// It is important that all exceptions are
// handled and none are permitted to bubble up.
}
return(Task.CompletedTask);
}
async Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
// Always log the exception.
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation ?? "Unknown" }");
Debug.WriteLine($"\tPartition: { args.PartitionId ?? "None" }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
// If cancellation was requested, assume that
// it was in response to an application request
// and take no action.
if (args.CancellationToken.IsCancellationRequested)
{
return;
}
// Allow the application to handle the exception according to
// its business logic.
await HandleExceptionAsync(args.Exception, args.CancellationToken);
}
catch
{
// Handle the exception. If fatal, signal
// for cancellation.
}
}
try
{
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// Once processing has started, the delay will
// block to allow processing until cancellation
// is requested.
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task RunAsync(EventProcessorClient processor, CancellationToken ct)
{
EnsureArg.IsNotNull(processor);
// Reset previous checkpoints corresponding to an older source event hub (i.e. applicable if the source event hub changes)
await _checkpointClient.ResetCheckpointsAsync();
// Processes two types of events
// 1) Event hub events
// 2) Maximum wait events. These are generated when we have not received an event hub
// event for a certain time period and this event is used to flush events in the current window.
async Task ProcessEventHandler(ProcessEventArgs eventArgs)
{
IEventMessage evt;
if (eventArgs.HasEvent)
{
evt = EventMessageFactory.CreateEvent(eventArgs);
}
else
{
evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow);
}
await _eventConsumerService.ConsumeEvent(evt);
}
// todo: consider retry
Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs)
{
var exception = (EventHubsException)eventArgs.Exception;
string reason = $"EventHubError{exception.Reason}";
_logger.LogError(exception);
_logger.LogMetric(EventMetrics.HandledException(reason, ConnectorOperation.Setup), 1);
return(Task.CompletedTask);
}
async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs)
{
var partitionId = initArgs.PartitionId;
_logger.LogTrace($"Initializing partition {partitionId}");
try
{
var checkpoint = await _checkpointClient.GetCheckpointForPartitionAsync(partitionId);
initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed);
_logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}");
_logger.LogMetric(EventMetrics.EventHubPartitionInitialized(), 1);
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
_logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint");
_logger.LogError(ex);
}
}
processor.ProcessEventAsync += ProcessEventHandler;
processor.ProcessErrorAsync += ProcessErrorHandler;
processor.PartitionInitializingAsync += ProcessInitializingHandler;
try
{
Console.WriteLine($"Starting event hub processor at {DateTime.UtcNow}");
await processor.StartProcessingAsync(ct);
// Wait indefinitely until cancellation is requested
ct.WaitHandle.WaitOne();
await processor.StopProcessingAsync();
}
finally
{
processor.ProcessEventAsync -= ProcessEventHandler;
processor.ProcessErrorAsync -= ProcessErrorHandler;
processor.PartitionInitializingAsync -= ProcessInitializingHandler;
}
}
private async Task RunEventProcessorAsync()
{
var cred = new DefaultAzureCredential();
var storageClient = new BlobContainerClient(new UriBuilder("https", StorageAccount, -1, "test4").Uri, cred);
var options = new EventProcessorClientOptions()
{
PrefetchCount = this.PrefetchCount,
LoadBalancingStrategy = this.Greedy? LoadBalancingStrategy.Greedy : LoadBalancingStrategy.Balanced,
LoadBalancingUpdateInterval = TimeSpan.FromSeconds(10)
};
var processor = new EventProcessorClient(storageClient, this.ConsumerGroupName, NamespaceName, EntityName, cred, options);
var partitionEventCount = new ConcurrentDictionary <string, int>();
long messageCount = 0;
var sw = Stopwatch.StartNew();
var bagStart = sw.ElapsedMilliseconds;
Console.WriteLine($"Receiving from entity '{EntityName}' in namespace '{NamespaceName}'");
var timer = new Timer(state =>
{
var snapshot = Interlocked.Exchange(ref messageCount, 0);
var bagDuration = sw.ElapsedMilliseconds - bagStart;
bagStart = sw.ElapsedMilliseconds;
Console.ResetColor();
Console.WriteLine($"\nReceived {snapshot / (bagDuration / 1000.0)} msg/sec, {snapshot} in {bagDuration} ms");
}, null, 10000, 10000);
async Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return;
}
string partition = args.Partition.PartitionId;
if (!string.IsNullOrEmpty(args.Partition.PartitionId))
{
int color = Math.Abs(args.Partition.PartitionId.GetHashCode());
Console.BackgroundColor = ConsoleColor.Black;
Console.ForegroundColor = (ConsoleColor)((color % 14) + 1);
}
Console.Write("[]");
Interlocked.Increment(ref messageCount);
int eventsSinceLastCheckpoint = partitionEventCount.AddOrUpdate(
key: partition,
addValue: 1,
updateValueFactory: (_, currentCount) => currentCount + 1);
if (eventsSinceLastCheckpoint >= 50)
{
await args.UpdateCheckpointAsync();
partitionEventCount[partition] = 0;
}
}
catch
{
}
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
}
catch
{
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
Console.ReadKey();
}
catch (TaskCanceledException)
{
}
finally
{
await processor.StopProcessingAsync();
}
}
catch
{
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
await processor.StartProcessingAsync();
Console.ReadKey();
timer.Dispose();
await processor.StopProcessingAsync();
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// When the Event Processor client begins processing, it will take ownership over a set of Event Hub partitions to process. For each
// partition that the processor claims, the first step in processing is to initialize the partition. In order to allow developers to
// track partitions owned by the processor and to participate in the initialization, a "PartitionInitializing" event is available on
// the processor.
//
// When a partition is initialized, one of the decisions made is where in the partition's event stream to begin processing. In the case
// where a checkpoint exists for a partition, processing will begin at the next available event after the checkpoint. When no checkpoint
// is found for a partition, a default location is used.
//
// One of the common reasons that you may choose to participate in initialization is to influence where to begin processing when a checkpoint
// is not found, overriding the default. To achieve this, you'll set the associated property on the event arguments within the event handler.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// The handler for partition initialization will set the default position for all partitions, asking to begin reading at the latest
// point in the stream.
Task partitionInitializingHandler(PartitionInitializingEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
eventArgs.DefaultStartingPosition = EventPosition.Latest;
Console.WriteLine($"Initialized partition: { eventArgs.PartitionId }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while initializing partition: { eventArgs.PartitionId }. Message: { ex.Message }");
Console.WriteLine();
}
return(Task.CompletedTask);
}
// For this example, events will just be logged to the console.
Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.Body.ToArray()) }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
return(Task.CompletedTask);
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.PartitionInitializingAsync += partitionInitializingHandler;
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// In order to begin processing, an explicit call must be made to the processor. This will instruct the processor to begin
// processing in the background, invoking your handlers when they are needed.
await processor.StartProcessingAsync();
// Because processing takes place in the background, we'll wait for a small period of time and then trigger
// cancellation.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(12));
while (!cancellationSource.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.PartitionInitializingAsync -= partitionInitializingHandler;
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task ProcessEvents()
{
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample01_ProcessEvents
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _eventHubScope.EventHubName;
/*@@*/ consumerGroup = _eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
var partitionEventCount = new ConcurrentDictionary <string, int>();
async Task processEventHandler(ProcessEventArgs args)
{
try
{
// If the cancellation token is signaled, then the
// processor has been asked to stop. It will invoke
// this handler with any events that were in flight;
// these will not be lost if not processed.
//
// It is up to the handler to decide whether to take
// action to process the event or to cancel immediately.
if (args.CancellationToken.IsCancellationRequested)
{
return;
}
string partition = args.Partition.PartitionId;
byte[] eventBody = args.Data.EventBody.ToArray();
Debug.WriteLine($"Event from partition { partition } with length { eventBody.Length }.");
int eventsSinceLastCheckpoint = partitionEventCount.AddOrUpdate(
key: partition,
addValue: 1,
updateValueFactory: (_, currentCount) => currentCount + 1);
if (eventsSinceLastCheckpoint >= 50)
{
await args.UpdateCheckpointAsync();
partitionEventCount[partition] = 0;
}
}
catch
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be redirected to
// the error handler.
}
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
}
catch
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be handled in any
// way.
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// The processor will automatically attempt to recover from any
// failures, either transient or fatal, and continue processing.
// Errors in the processor's operation will be surfaced through
// its error handler.
//
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// In order to ensure efficient communication with the Event Hubs service and the best throughput possible for dispatching events to be processed,
// the Event Processor client is eagerly reading from each partition of the Event Hub and staging events. The processor will dispatch an event
// to the "ProcessEvent" handler immediately when one is available. Each call to the handler passes a single event and the context of the partition
// from which the event was read. This pattern is intended to allow developers to act on an event as soon as possible, and present a straightforward
// and understandable interface.
//
// This approach is optimized for scenarios where the processing of events can be performed quickly and without heavy resource costs. For scenarios
// where that is not the case, it may be advantageous to collect the events into batches and send them to be processed outside of the context
// of the "ProcessEvent" handler.
//
// In this example, our "ProcessEvent" handler will group events into batches by partition, sending them for downstream processing when the desired
// batch size was reached or when no event was available for more than a maximum wait time interval.
int desiredBatchSize = 3;
TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(150);
// The Event Processor client will preserve the order that events were enqueued in a partition by waiting for the "ProcessEvent" handler to
// complete before it is invoked with a new event for the same partition. However, partitions are processed concurrently, so the
// handler is likely to be executing for multiple partitions at the same time.
//
// To account for this, we'll use a concurrent dictionary to track batches, grouping them by partition.
ConcurrentDictionary <string, List <ProcessEventArgs> > eventBatches = new ConcurrentDictionary <string, List <ProcessEventArgs> >();
// Create our Event Processor client, specifying the maximum wait time as an option to ensure that
// our handler is invoked when no event was available.
EventProcessorClientOptions clientOptions = new EventProcessorClientOptions
{
MaximumWaitTime = maximumWaitTime
};
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, clientOptions);
// For this example, we'll create a simple event handler that writes to the
// console each time it was invoked.
int eventIndex = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
// Retrieve or create the active batch for the current partition.
List <ProcessEventArgs> currentBatch = eventBatches.GetOrAdd(eventArgs.Partition.PartitionId, _ => new List <ProcessEventArgs>());
bool sendBatchForProcessing = false;
// If there was an event emitted, add the event and check to see if the size of the batch has reached the desired
// size. If so, it will need to be sent.
//
// NOTE: There is a bug in the Event Hubs preview 6 library causing "HasEvents" to return the
// wrong value. We'll substitute a check against the "Data" property to work around it.
//
// if (eventArgs.HasEvents) {} is the preferred snippet.
//
if (eventArgs.Data != null)
{
currentBatch.Add(eventArgs);
sendBatchForProcessing = (currentBatch.Count >= desiredBatchSize);
}
else
{
// There was no event available within the interval requested by the maximum
// wait time, send the batch for processing if it contains any events.
sendBatchForProcessing = (currentBatch.Count > 0);
}
// It is important to be aware that no events for the partition will be processed until the handler returns,
// so you may wish to delegate processing to a downstream service or background task in order to maintain
// throughput.
//
// In this example, if the batch is to be sent, we'll delegate to simple helper method that will write
// to the console. If sending our batch completed successfully, we'll checkpoint using the last
// event and clear the batch.
if (sendBatchForProcessing)
{
await SendEventBatchAsync(currentBatch.Select(item => item.Data));
await currentBatch[currentBatch.Count - 1].UpdateCheckpointAsync();
currentBatch.Clear();
}
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
++eventIndex;
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
Console.WriteLine();
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
// We'll publish a batch of events for our processor to receive. We'll split the events into a couple of batches to
// increase the chance they'll be spread around to different partitions and introduce a delay between batches to
// allow for the handler to be invoked without an available event interleaved.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Seventh Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Seventh Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Seventh Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
await Task.Delay(250, cancellationSource.Token);
}
}
// We'll allow the Event Processor client to read and dispatch the events that we published, along with
// ensuring a few invocations with no event. Note that, due to non-determinism in the timing, we may or may
// not see all of the events from our batches read.
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count + 5))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task BasicEventProcessing()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_BasicEventProcessing
#if SNIPPET
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
#endif
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
string partition = args.Partition.PartitionId;
byte[] eventBody = args.Data.EventBody.ToArray();
Debug.WriteLine($"Event from partition { partition } with length { eventBody.Length }.");
}
catch
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be redirected to
// the error handler.
}
return(Task.CompletedTask);
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
}
catch (Exception ex)
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be handled in any
// way.
Application.HandleErrorException(args, ex);
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// The processor will automatically attempt to recover from any
// failures, either transient or fatal, and continue processing.
// Errors in the processor's operation will be surfaced through
// its error handler.
//
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
