public async Task ProcessUntilCanceled()
{
await using var eventHubScope = await EventHubScope.CreateAsync(2);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_ReadMe_ProcessUntilCanceled
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(45));
#if SNIPPET
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
#endif
Task processEventHandler(ProcessEventArgs eventArgs) => Task.CompletedTask;
Task processErrorHandler(ProcessErrorEventArgs eventArgs) => Task.CompletedTask;
var storageClient = new BlobContainerClient(storageConnectionString, blobContainerName);
var processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
await processor.StartProcessingAsync();
try
{
// The processor performs its work in the background; block until cancellation
// to allow processing to take place.
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected when the delay is canceled.
}
try
{
await processor.StopProcessingAsync();
}
finally
{
// To prevent leaks, the handlers should be removed when processing is complete.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task CreateReceiverWithEndOfStream()
{
var receiver = default(PartitionReceiver);
var partitionSender = default(PartitionSender);
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName);
var ehClient = EventHubClient.CreateFromConnectionString(connectionString);
try
{
// Randomly pick one of the available partitions.
var partitions = await this.GetPartitionsAsync(ehClient);
var partitionId = partitions[new Random().Next(partitions.Length)];
TestUtility.Log($"Randomly picked partition {partitionId}");
partitionSender = ehClient.CreatePartitionSender(partitionId);
// Send couple of messages before creating an EndOfStream receiver.
// We are not expecting to receive these messages would be sent before receiver creation.
for (int i = 0; i < 10; i++)
{
var ed = new EventData(new byte[1]);
await partitionSender.SendAsync(ed);
}
// Create a new receiver which will start reading from the end of the stream.
TestUtility.Log($"Creating a new receiver with offset EndOFStream");
receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, EventPosition.FromEnd());
// Attempt to receive the message. This should return only 1 message.
var receiveTask = receiver.ReceiveAsync(100);
// Send a new message which is expected to go to the end of stream.
// We are expecting to receive only this message.
// Wait 5 seconds before sending to avoid race.
await Task.Delay(5000);
var eventToReceive = new EventData(new byte[1]);
eventToReceive.Properties["stamp"] = Guid.NewGuid().ToString();
await partitionSender.SendAsync(eventToReceive);
// Complete asyncy receive task.
var receivedMessages = await receiveTask;
// We should have received only 1 message from this call.
Assert.True(receivedMessages.Count() == 1, $"Didn't receive 1 message. Received {receivedMessages.Count()} messages(s).");
// Check stamp.
Assert.True(receivedMessages.Single().Properties["stamp"].ToString() == eventToReceive.Properties["stamp"].ToString()
, "Stamps didn't match on the message sent and received!");
TestUtility.Log("Received correct message as expected.");
// Next receive on this partition shouldn't return any more messages.
receivedMessages = await receiver.ReceiveAsync(100, TimeSpan.FromSeconds(15));
Assert.True(receivedMessages == null, $"Received messages at the end.");
}
finally
{
await Task.WhenAll(
partitionSender.CloseAsync(),
receiver.CloseAsync(),
ehClient.CloseAsync());
}
}
}
public async Task SendBatchWithPartitionKey()
{
string targetPartitionKey = "this is the partition key";
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName);
var ehClient = EventHubClient.CreateFromConnectionString(connectionString);
var receiver = default(PartitionReceiver);
try
{
// Mark end of each partition so that we can start reading from there.
var partitionIds = await this.GetPartitionsAsync(ehClient);
var partitions = await TestUtility.DiscoverEndOfStreamForPartitionsAsync(ehClient, partitionIds);
// Send a batch of 2 messages.
using (var eventData1 = new EventData(Guid.NewGuid().ToByteArray()))
using (var eventData2 = new EventData(Guid.NewGuid().ToByteArray()))
{
await ehClient.SendAsync(new[] { eventData1, eventData2 }, targetPartitionKey);
}
// Now find out the partition where our messages landed.
var targetPartition = "";
foreach (var pId in partitionIds)
{
var pInfo = await ehClient.GetPartitionRuntimeInformationAsync(pId);
if (pInfo.LastEnqueuedOffset != partitions[pId])
{
targetPartition = pId;
TestUtility.Log($"Batch landed on partition {targetPartition}");
}
}
// Confirm that we identified the partition with our messages.
Assert.True(targetPartition != "", "None of the partition offsets moved.");
// Receive all messages from target partition.
receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, targetPartition, EventPosition.FromOffset(partitions[targetPartition]));
var messages = await ReceiveAllMessagesAsync(receiver);
// Validate 2 messages received.
Assert.True(messages.Count == 2, $"Received {messages.Count} messages instead of 2.");
// Validate both messages carry correct partition id.
Assert.True(messages[0].SystemProperties.PartitionKey == targetPartitionKey,
$"First message returned partition key value '{messages[0].SystemProperties.PartitionKey}'");
Assert.True(messages[1].SystemProperties.PartitionKey == targetPartitionKey,
$"Second message returned partition key value '{messages[1].SystemProperties.PartitionKey}'");
}
finally
{
await Task.WhenAll(
receiver?.CloseAsync(),
ehClient.CloseAsync());
}
}
}
public async Task CustomMetadata()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample04_CustomMetadata
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var producer = new EventHubBufferedProducerClient(connectionString, eventHubName);
// The failure handler is required and invoked after all allowable
// retries were applied.
producer.SendEventBatchFailedAsync += args =>
{
Debug.WriteLine($"Publishing failed for { args.EventBatch.Count } events. Error: '{ args.Exception.Message }'");
return(Task.CompletedTask);
};
// The success handler is optional.
producer.SendEventBatchSucceededAsync += args =>
{
Debug.WriteLine($"{ args.EventBatch.Count } events were published to partition: '{ args.PartitionId }.");
return(Task.CompletedTask);
};
try
{
var eventData = new EventData("Hello, Event Hubs!")
{
MessageId = "H1",
ContentType = "application/json"
};
eventData.Properties.Add("EventType", "com.microsoft.samples.hello-event");
eventData.Properties.Add("priority", 1);
eventData.Properties.Add("score", 9.0);
await producer.EnqueueEventAsync(eventData);
eventData = new EventData("Goodbye, Event Hubs!")
{
MessageId = "G1",
ContentType = "application/json"
};
eventData.Properties.Add("EventType", "com.microsoft.samples.goodbye-event");
eventData.Properties.Add("priority", "17");
eventData.Properties.Add("blob", true);
await producer.EnqueueEventAsync(eventData);
}
finally
{
// Closing the producer will flush any
// enqueued events that have not been published.
await producer.CloseAsync();
}
#endregion
}
public async Task DefaultAzureCredential()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_DefaultAzureCredential
TokenCredential credential = new DefaultAzureCredential();
var storageEndpoint = "<< STORAGE ENDPOINT (likely similar to {your-account}.blob.core.windows.net) >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageEndpoint = new BlobServiceClient(StorageTestEnvironment.Instance.StorageConnectionString).Uri.ToString();
/*@@*/ blobContainerName = storageScope.ContainerName;
var fullyQualifiedNamespace = "<< NAMESPACE (likely similar to {your-namespace}.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ fullyQualifiedNamespace = EventHubsTestEnvironment.Instance.FullyQualifiedNamespace;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
/*@@*/ credential = EventHubsTestEnvironment.Instance.Credential;
var blobUriBuilder = new BlobUriBuilder(new Uri(storageEndpoint));
blobUriBuilder.BlobContainerName = blobContainerName;
var storageClient = new BlobContainerClient(
blobUriBuilder.ToUri(),
credential);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
fullyQualifiedNamespace,
eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task EventProcessorWaitsMaximumReceiveWaitTimeForEvents(int maximumWaitTimeInSecs)
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var timestamps = new ConcurrentDictionary <string, List <DateTimeOffset> >();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
MaximumReceiveWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs)
},
onInitialize: (partitionContext, checkpointManager) =>
timestamps.TryAdd(partitionContext.PartitionId, new List <DateTimeOffset> {
DateTimeOffset.UtcNow
}),
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
timestamps.AddOrUpdate
(
// The key already exists, so the 'addValue' factory will never be called.
partitionContext.PartitionId,
partitionId => null,
(partitionId, list) =>
{
list.Add(DateTimeOffset.UtcNow);
return(list);
}
)
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize. We are waiting a few times the maximum
// wait time span so we can have enough samples.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(4000 *maximumWaitTimeInSecs);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
foreach (var kvp in timestamps)
{
var partitionId = kvp.Key;
var partitionTimestamps = kvp.Value;
Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more timestamp samples were expected.");
for (int index = 1; index < partitionTimestamps.Count; index++)
{
var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds;
Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short.");
Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long.");
++index;
}
}
}
}
}
public async Task LoadBalancingIsEnforcedWhenDistributionIsUneven()
{
var partitions = 10;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
ConcurrentDictionary <string, int> ownedPartitionsCount = new ConcurrentDictionary <string, int>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onInitialize: (partitionContext, checkpointManager) =>
ownedPartitionsCount.AddOrUpdate(checkpointManager.OwnerIdentifier, 1, (ownerId, value) => value + 1),
onClose: (partitionContext, checkpointManager, reason) =>
ownedPartitionsCount.AddOrUpdate(checkpointManager.OwnerIdentifier, 0, (ownerId, value) => value - 1)
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Assert all partitions have been claimed.
Assert.That(ownedPartitionsCount.ToArray().Single().Value, Is.EqualTo(partitions));
// Insert a new event processor into the hub so it can start stealing partitions.
hub.AddEventProcessors(1);
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Take a snapshot of the current partition balancing status.
var ownedPartitionsCountSnapshot = ownedPartitionsCount.ToArray().Select(kvp => kvp.Value);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
var minimumOwnedPartitionsCount = partitions / 2;
var maximumOwnedPartitionsCount = minimumOwnedPartitionsCount + 1;
foreach (var count in ownedPartitionsCountSnapshot)
{
Assert.That(count, Is.InRange(minimumOwnedPartitionsCount, maximumOwnedPartitionsCount));
}
Assert.That(ownedPartitionsCountSnapshot.Sum(), Is.EqualTo(partitions));
}
}
}
public async Task ProcessWithHeartbeat()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_ProcessWithHeartbeat
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processorOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(250)
};
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName,
processorOptions);
async Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.HasEvent)
{
await Application.ProcessEventAndCheckpointAsync(
args.Data,
args.Partition,
args.CancellationToken);
}
await Application.SendHeartbeatAsync(args.CancellationToken);
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take slightly longer than the length of
// time defined as part of the MaximumWaitTime configured
// for the processor; in this example, 250 milliseconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ReadPartitionWaitTime()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionWaitTime
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
int loopTicks = 0;
int maximumTicks = 10;
var options = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options))
{
if (partitionEvent.Data != null)
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
else
{
Debug.WriteLine("Wait time elapsed; no event was available.");
}
loopTicks++;
if (loopTicks >= maximumTicks)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task BasicEventProcessing()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_BasicEventProcessing
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
string partition = args.Partition.PartitionId;
byte[] eventBody = args.Data.EventBody.ToArray();
Debug.WriteLine($"Event from partition { partition } with length { eventBody.Length }.");
}
catch
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be redirected to
// the error handler.
}
return(Task.CompletedTask);
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
}
catch (Exception ex)
{
// It is very important that you always guard against
// exceptions in your handler code; the processor does
// not have enough understanding of your code to
// determine the correct action to take. Any
// exceptions from your handlers go uncaught by
// the processor and will NOT be handled in any
// way.
Application.HandleErrorException(args, ex);
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// The processor will automatically attempt to recover from any
// failures, either transient or fatal, and continue processing.
// Errors in the processor's operation will be surfaced through
// its error handler.
//
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task ProcessByBatch()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_ProcessByBatch
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
const int EventsInBatch = 50;
var partitionEventBatches = new ConcurrentDictionary <string, List <EventData> >();
var checkpointNeeded = false;
async Task processEventHandler(ProcessEventArgs args)
{
try
{
string partition = args.Partition.PartitionId;
List <EventData> partitionBatch =
partitionEventBatches.GetOrAdd(
partition,
new List <EventData>());
partitionBatch.Add(args.Data);
if (partitionBatch.Count >= EventsInBatch)
{
await Application.ProcessEventBatchAsync(
partitionBatch,
args.Partition,
args.CancellationToken);
checkpointNeeded = true;
partitionBatch.Clear();
}
if (checkpointNeeded)
{
await args.UpdateCheckpointAsync();
checkpointNeeded = false;
}
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task InitializePartition()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_InitializePartition
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
Task initializeEventHandler(PartitionInitializingEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
// If no checkpoint was found, start processing
// events enqueued now or in the future.
EventPosition startPositionWhenNoCheckpoint =
EventPosition.FromEnqueuedTime(DateTimeOffset.UtcNow);
args.DefaultStartingPosition = startPositionWhenNoCheckpoint;
}
catch (Exception ex)
{
Application.HandleInitializeException(args, ex);
}
return(Task.CompletedTask);
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers for processing events and errors are
// not relevant for this sample; for illustration, they're
// delegating the implementation to the host application.
processor.PartitionInitializingAsync += initializeEventHandler;
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup
processor.PartitionInitializingAsync -= initializeEventHandler;
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task CheckpointByEventCount()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_CheckpointByEventCount
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
const int EventsBeforeCheckpoint = 25;
var partitionEventCount = new ConcurrentDictionary <string, int>();
async Task processEventHandler(ProcessEventArgs args)
{
try
{
await Application.ProcessEventAsync(
args.Data,
args.Partition,
args.CancellationToken);
// If the number of events that have been processed
// since the last checkpoint was created exceeds the
// checkpointing threshold, a new checkpoint will be
// created and the count reset.
string partition = args.Partition.PartitionId;
int eventsSinceLastCheckpoint = partitionEventCount.AddOrUpdate(
key: partition,
addValue: 1,
updateValueFactory: (_, currentCount) => currentCount + 1);
if (eventsSinceLastCheckpoint >= EventsBeforeCheckpoint)
{
await args.UpdateCheckpointAsync();
partitionEventCount[partition] = 0;
}
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task CustomProcessorUse()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample08_CustomProcessorUse
#if SNIPPET
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
#else
var eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = Mock.Of <BlobContainerClient>();
#endif
var maximumBatchSize = 100;
var processor = new CustomProcessor(
storageClient,
maximumBatchSize,
consumerGroup,
eventHubsConnectionString,
eventHubName);
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// Starting the processor does not block when starting; delay
// until the cancellation token is signaled.
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// Stopping may take up to the length of time defined
// as the TryTimeout configured for the processor;
// By default, this is 60 seconds.
await processor.StopProcessingAsync();
}
#endregion
}
public async Task EventProcessorCanStartAgainAfterStopping()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
hub.AddEventProcessors(1);
// Send some events.
var expectedEventsCount = 20;
await using (var producer = client.CreateProducer())
{
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// We'll start and stop the event processors twice. This way, we can assert they will behave
// the same way both times, reprocessing all events in the second run.
for (int i = 0; i < 2; i++)
{
receivedEventsCount = 0;
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }.");
}
}
}
}
public async Task ReadPartitionTrackLastEnqueued()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
var options = new ReadEventOptions
{
TrackLastEnqueuedEventProperties = true
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options,
cancellationSource.Token))
{
LastEnqueuedEventProperties properties =
partitionEvent.Partition.ReadLastEnqueuedEventProperties();
Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }");
Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }");
Debug.WriteLine($"\tThe last offset is: { properties.Offset }");
Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC.");
Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC.");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
DateTimeOffset enqueuedTime;
await using (var producer = client.CreateProducer())
{
// Send a few dummy events. We are not expecting to receive these.
for (int i = 0; i < 30; i++)
{
await producer.SendAsync(dummyEvent);
}
// Wait a reasonable amount of time so the events are able to reach the service.
await Task.Delay(1000);
// Send the events we expect to receive.
enqueuedTime = DateTimeOffset.UtcNow;
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime)
},
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task ReadPartitionWithReceiver()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionWithReceiver
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition;
await using (var producer = new EventHubProducerClient(connectionString, eventHubName))
{
firstPartition = (await producer.GetPartitionIdsAsync()).First();
}
var receiver = new PartitionReceiver(
consumerGroup,
firstPartition,
EventPosition.Earliest,
connectionString,
eventHubName);
try
{
while (!cancellationSource.IsCancellationRequested)
{
int batchSize = 50;
TimeSpan waitTime = TimeSpan.FromSeconds(1);
IEnumerable <EventData> eventBatch = await receiver.ReceiveBatchAsync(
batchSize,
waitTime,
cancellationSource.Token);
foreach (EventData eventData in eventBatch)
{
byte[] eventBodyBytes = eventData.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { firstPartition }");
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await receiver.CloseAsync();
}
#endregion
}
public async Task EventProcessorCannotReceiveMoreThanMaximumMessageCountMessagesAtATime(int maximumMessageCount)
{
var partitions = 2;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var unexpectedMessageCount = -1;
// Send some events.
await using (var producer = client.CreateProducer())
{
var eventSet = Enumerable
.Range(0, 20 * maximumMessageCount)
.Select(index => new EventData(new byte[10]))
.ToList();
// Send one set per partition.
for (int i = 0; i < partitions; i++)
{
await producer.SendAsync(eventSet);
}
}
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
MaximumMessageCount = maximumMessageCount
},
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
// In case we find a message count greater than the allowed amount, we only store the first
// occurrence and ignore the subsequent ones.
if (eventsList.Count > maximumMessageCount)
{
Interlocked.CompareExchange(ref unexpectedMessageCount, eventsList.Count, -1);
}
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(unexpectedMessageCount, Is.EqualTo(-1), $"A set of { unexpectedMessageCount } events was received.");
}
}
}
public async Task EventHandlerStopOnException()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample03_EventHandlerStopOnException
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// This token is used to control processing,
// if signaled, then processing will be stopped.
using var cancellationSource = new CancellationTokenSource();
/*@@*/ cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
// Process the event.
}
catch
{
// Handle the exception. If fatal,
// signal for cancellation.
cancellationSource.Cancel();
}
return(Task.CompletedTask);
}
Task processErrorHandler(ProcessErrorEventArgs args)
{
// Process the error, as appropriate for the
// application.
return(Task.CompletedTask);
}
try
{
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// Once processing has started, the delay will
// block to allow processing until cancellation
// is requested.
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task SharedAccessSignature()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_SharedAccessSignature
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
//@@ var credential = new AzureSasCredential("<< SHARED ACCESS KEY STRING >>");
var fullyQualifiedNamespace = "<< NAMESPACE (likely similar to {your-namespace}.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ fullyQualifiedNamespace = EventHubsTestEnvironment.Instance.FullyQualifiedNamespace;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
/*@@*/
/*@@*/ var resource = $"amqps://{ EventHubsTestEnvironment.Instance.FullyQualifiedNamespace }/{ eventHubScope.EventHubName }".ToLowerInvariant();
/*@@*/ var signature = new SharedAccessSignature(resource, EventHubsTestEnvironment.Instance.SharedAccessKeyName, EventHubsTestEnvironment.Instance.SharedAccessKey);
/*@@*/ var credential = new AzureSasCredential(signature.Value);
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
fullyQualifiedNamespace,
eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task ErrorHandlerCancellationRecovery()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample03_ErrorHandlerCancellationRecovery
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
// This token is used to control processing,
// if signaled, then processing will be stopped.
using var cancellationSource = new CancellationTokenSource();
/*@@*/ cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
Task processEventHandler(ProcessEventArgs args)
{
try
{
// Process the event.
}
catch
{
// Handle the exception.
}
return(Task.CompletedTask);
}
async Task processErrorHandler(ProcessErrorEventArgs args)
{
try
{
// Always log the exception.
Debug.WriteLine("Error in the EventProcessorClient");
Debug.WriteLine($"\tOperation: { args.Operation ?? "Unknown" }");
Debug.WriteLine($"\tPartition: { args.PartitionId ?? "None" }");
Debug.WriteLine($"\tException: { args.Exception }");
Debug.WriteLine("");
// If cancellation was requested, assume that
// it was in response to an application request
// and take no action.
if (args.CancellationToken.IsCancellationRequested)
{
return;
}
// If out of memory, signal for cancellation.
if (args.Exception is OutOfMemoryException)
{
cancellationSource.Cancel();
return;
}
// If processing stopped and this handler determined
// the error to be non-fatal, restart processing.
if ((!processor.IsRunning) &&
(!cancellationSource.IsCancellationRequested))
{
// To be safe, request that processing stop before
// requesting the start; this will ensure that any
// processor state is fully reset.
await processor.StopProcessingAsync();
await processor.StartProcessingAsync(cancellationSource.Token);
}
}
catch
{
// Handle the exception. If fatal, signal
// for cancellation.
}
}
try
{
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// Once processing has started, the delay will
// block to allow processing until cancellation
// is requested.
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
finally
{
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
#endregion
}
public async Task FixtureSetUp()
{
_eventHubScope = await EventHubScope.CreateAsync(2);
}
public async Task NonexistentEntity()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName);
PartitionReceiver receiver = null;
// Rebuild connection string with a nonexistent entity.
var csb = new EventHubsConnectionStringBuilder(connectionString);
csb.EntityPath = Guid.NewGuid().ToString();
var ehClient = EventHubClient.CreateFromConnectionString(csb.ToString());
try
{
// GetRuntimeInformationAsync on a nonexistent entity.
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Getting entity information from a nonexistent entity.");
await ehClient.GetRuntimeInformationAsync();
});
// GetPartitionRuntimeInformationAsync on a nonexistent entity.
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Getting partition information from a nonexistent entity.");
await ehClient.GetPartitionRuntimeInformationAsync("0");
});
// Try sending.
PartitionSender sender = null;
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Sending an event to nonexistent entity.");
sender = ehClient.CreatePartitionSender("0");
await sender.SendAsync(new EventData(Encoding.UTF8.GetBytes("this send should fail.")));
});
await sender.CloseAsync();
// Try receiving.
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Receiving from nonexistent entity.");
receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromStart());
await receiver.ReceiveAsync(1);
});
await receiver.CloseAsync();
}
finally
{
await ehClient.CloseAsync();
}
// Try receiving on an nonexistent consumer group.
ehClient = EventHubClient.CreateFromConnectionString(connectionString);
try
{
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Receiving from nonexistent consumer group.");
receiver = ehClient.CreateReceiver(Guid.NewGuid().ToString(), "0", EventPosition.FromStart());
await receiver.ReceiveAsync(1);
});
await receiver.CloseAsync();
}
finally
{
await ehClient.CloseAsync();
}
}
}
public async Task ProcessEvents()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample06_ChooseStorageVersion
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClientOptions = new BlobClientOptions();
storageClientOptions.AddPolicy(
new StorageApiVersionPolicy(),
HttpPipelinePosition.PerCall);
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName,
storageClientOptions);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// The processor will automatically attempt to recover from any
// failures, either transient or fatal, and continue processing.
// Errors in the processor's operation will be surfaced through
// its error handler.
//
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
protected async Task SendWithEventDataBatch(
string partitionKey = null,
int maxPayloadSize = 1024,
int minimumNumberOfMessagesToSend = 1000)
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName);
var ehClient = EventHubClient.CreateFromConnectionString(connectionString);
var partitions = await this.GetPartitionsAsync(ehClient);
var receivers = new List <PartitionReceiver>();
// Create partition receivers starting from the end of the stream.
TestUtility.Log("Discovering end of stream on each partition.");
foreach (var partitionId in partitions)
{
var lastEvent = await ehClient.GetPartitionRuntimeInformationAsync(partitionId);
receivers.Add(ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, EventPosition.FromOffset(lastEvent.LastEnqueuedOffset)));
}
try
{
// Start receicing messages now.
var receiverTasks = new List <Task <List <EventData> > >();
foreach (var receiver in receivers)
{
receiverTasks.Add(ReceiveAllMessagesAsync(receiver));
}
// Create initial batcher.
EventDataBatch batcher = null;
// We will send a thousand messages where each message is 1K.
var totalSent = 0;
var rnd = new Random();
TestUtility.Log("Starting to send.");
do
{
if (batcher == null)
{
// Exercise both CreateBatch overloads.
if (partitionKey != null)
{
batcher = ehClient.CreateBatch(new BatchOptions()
{
PartitionKey = partitionKey
});
}
else
{
batcher = ehClient.CreateBatch();
}
}
// Send random body size.
var ed = new EventData(new byte[rnd.Next(0, maxPayloadSize)]);
if (!batcher.TryAdd(ed) || totalSent + batcher.Count >= minimumNumberOfMessagesToSend)
{
await ehClient.SendAsync(batcher);
totalSent += batcher.Count;
TestUtility.Log($"Sent {batcher.Count} messages in the batch.");
batcher = null;
}
} while (totalSent < minimumNumberOfMessagesToSend);
TestUtility.Log($"{totalSent} messages sent in total.");
var pReceived = await Task.WhenAll(receiverTasks);
var totalReceived = pReceived.Sum(p => p.Count);
TestUtility.Log($"{totalReceived} messages received in total.");
// Sent at least a message?
Assert.True(totalSent > 0, $"Client was not able to send any messages.");
// All messages received?
Assert.True(totalReceived == totalSent, $"Sent {totalSent}, but received {totalReceived} messages.");
if (partitionKey != null)
{
// Partition key is set then we expect all messages from the same partition.
Assert.True(pReceived.Count(p => p.Count > 0) == 1, "Received messsages from multiple partitions.");
// Find target partition.
var targetPartition = pReceived.Single(p => p.Count > 0);
// Validate partition key is delivered on all messages.
Assert.True(!targetPartition.Any(p => p.SystemProperties.PartitionKey != partitionKey), "Identified at least one event with a different partition key value.");
}
}
finally
{
await Task.WhenAll(receivers.Select(r => r.CloseAsync()));
await ehClient.CloseAsync();
}
}
}
public async Task ReadAllPartitions()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample07_ReadAllPartitions
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using (CancellationTokenSource cancellationSource = new CancellationTokenSource())
{
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
int eventsRead = 0;
int maximumEvents = 50;
IAsyncEnumerator <PartitionEvent> iterator =
consumer.ReadEventsAsync(cancellationSource.Token).GetAsyncEnumerator();
try
{
while (await iterator.MoveNextAsync())
{
PartitionEvent partitionEvent = iterator.Current;
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
eventsRead++;
if (eventsRead >= maximumEvents)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await iterator.DisposeAsync();
}
}
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
allReceivedEvents.AddOrUpdate
(
partitionContext.PartitionId,
partitionId => eventsList,
(partitionId, list) =>
{
list.AddRange(eventsList);
return(list);
}
);
}
}
);
hub.AddEventProcessors(1);
// Send some events.
var partitionIds = await client.GetPartitionIdsAsync();
var expectedEvents = new Dictionary <string, List <EventData> >();
foreach (var partitionId in partitionIds)
{
// Send a similar set of events for every partition.
expectedEvents[partitionId] = new List <EventData>
{
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
};
await using (var producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionId
}))
{
await producer.SendAsync(expectedEvents[partitionId]);
}
}
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results. Make sure we received every event in the correct partition processor,
// in the order they were sent.
foreach (var partitionId in partitionIds)
{
Assert.That(allReceivedEvents.TryGetValue(partitionId, out var partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");
var index = 0;
foreach (var receivedEvent in partitionReceivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
++index;
}
}
Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
public async Task FixtureSetUp()
{
_eventHubScope = await EventHubScope.CreateAsync(2);
_testId = Guid.NewGuid().ToString();
}
public async Task ProcessByBatch()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample07_ProcessByBatch_Usage
#if SNIPPET
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
#else
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = eventHubScope.EventHubName;
var consumerGroup = eventHubScope.ConsumerGroups.First();
#endif
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var checkpointStore = new BlobCheckpointStore(storageClient);
var maximumBatchSize = 100;
var processor = new SimpleBatchProcessor(
checkpointStore,
maximumBatchSize,
consumerGroup,
eventHubsConnectionString,
eventHubName);
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// There are no event handlers to set for the processor. All logic
// normally associated with the processor event handlers is
// implemented directly via method override in the custom processor.
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// Stopping may take up to the length of time defined
// as the TryTimeout configured for the processor;
// By default, this is 60 seconds.
await processor.StopProcessingAsync();
}
#endregion
}
