public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
DateTimeOffset enqueuedTime;
await using (EventHubProducer producer = client.CreateProducer())
{
// Send a few dummy events. We are not expecting to receive these.
for (int i = 0; i < 30; i++)
{
await producer.SendAsync(dummyEvent);
}
// Wait a reasonable amount of time so the events are able to reach the service.
await Task.Delay(1000);
// Send the events we expect to receive.
enqueuedTime = DateTimeOffset.UtcNow;
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
options: new EventProcessorOptions {
InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime)
},
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
// Send some events.
EventData lastEvent;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new InMemoryPartitionManager();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Any())
{
partitionContext.UpdateCheckpointAsync(eventsList.Last());
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
PartitionOwnership ownership = ownershipEnumerable.Single();
Assert.That(ownership.Offset.HasValue, Is.True);
Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));
Assert.That(ownership.SequenceNumber.HasValue, Is.True);
Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
}
}
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new InMemoryPartitionManager();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task EventProcessorCanStartAgainAfterStopping()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Send some events.
var expectedEventsCount = 20;
await using (EventHubProducer producer = client.CreateProducer())
{
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// We'll start and stop the event processors twice. This way, we can assert they will behave
// the same way both times, reprocessing all events in the second run.
for (int i = 0; i < 2; i++)
{
receivedEventsCount = 0;
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }.");
}
}
}
}
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
allReceivedEvents.AddOrUpdate
(
partitionContext.PartitionId,
partitionId => eventsList,
(partitionId, list) =>
{
list.AddRange(eventsList);
return(list);
}
);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Send some events.
var partitionIds = await client.GetPartitionIdsAsync();
var expectedEvents = new Dictionary <string, List <EventData> >();
foreach (var partitionId in partitionIds)
{
// Send a similar set of events for every partition.
expectedEvents[partitionId] = new List <EventData>
{
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
};
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionId
}))
{
await producer.SendAsync(expectedEvents[partitionId]);
}
}
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results. Make sure we received every event in the correct partition processor,
// in the order they were sent.
foreach (var partitionId in partitionIds)
{
Assert.That(allReceivedEvents.TryGetValue(partitionId, out List <EventData> partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");
var index = 0;
foreach (EventData receivedEvent in partitionReceivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
++index;
}
}
Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier
// of the first.
string firstPartition = (await client.GetPartitionIdsAsync()).First();
// In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = firstPartition
}))
{
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events, we will need to ask the consumer to perform an operation,
// because it opens its connection only when it needs to. The first receive that we ask of it will not see
// any events, but will allow the consumer to start watching the partition.
//
// Because the maximum wait time is specivied as zero, this call will return immediately and will not
// have consumed any events.
await consumer.ReceiveAsync(1, TimeSpan.Zero);
// Now that the consumer is watching the partition, let's publish a fair sized batch of events
// we would like for it to consume.
int eventBatchSize = 100;
EventData[] eventBatch = new EventData[eventBatchSize];
for (int index = 0; index < eventBatchSize; ++index)
{
eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"));
}
await producer.SendAsync(eventBatch);
Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
// To allow for throughput in our application, we will process the published events by consuming them in
// small batches with a small wait time. This will allow us to receive and process more quickly than blocking
// to wait on a larger batch size.
//
// The values that are used in this example are intended just for illustration and not as guidance for a recommended
// set of defaults. In a real-world application, determining the right balance of batch size and wait time to achieve
// the desired performance will often vary depending on the application and event data being consumed. It is an area
// where some experimentation in the application context may prove helpful.
//
// Our example will attempt to read about 1/5 of the batch at a time, which should result in the need for 5 batches to
// be consumed. Because publishing and receving events is asynchronous, the events that we published may not be
// immediately available for our consumer to see. To compensate, we will allow for a small number of extra attempts beyond
// the expected 5 to to be sure that we don't stop reading before we receive all of our events.
var receivedEvents = new List <EventData>();
int consumeBatchSize = (int)Math.Floor(eventBatchSize / 5.0f);
int maximumAttempts = 15;
int attempts = 0;
while ((receivedEvents.Count < eventBatchSize) && (++attempts < maximumAttempts))
{
// Each receive, we ask for the maximum amount of events that we would like in the batch and
// specify the maximum amount of time that we would like to wait for them.
//
// The batch of events will be returned to us when either we have read our maximum number of events
// or when the time that we asked to wait has elapsed. This means that a batch we receive may have
// between zero and the maximum we asked for, depending on what was available in the partition.
//
// For this attempt, we will ask to receive our computed batch size (1/5 of published events) and wait, at most,
// 25 milliseconds to receive them. We'll then process them and if we haven't gotten all that we published, we'll
// make another attempt until either we have them or we have tried for our maximum number of attempts.
IEnumerable <EventData> receivedBatch = await consumer.ReceiveAsync(consumeBatchSize, TimeSpan.FromMilliseconds(25));
receivedEvents.AddRange(receivedBatch);
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"Events Consumed: { receivedEvents.Count }");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
// At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
