public async Task StopAsyncCallsPartitionProcessorCloseAsyncWithShutdownReason()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
var closeCalls = new ConcurrentDictionary <string, int>();
var stopReasons = new ConcurrentDictionary <string, ProcessingStoppedReason>();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
onStop: stopContext =>
{
closeCalls.AddOrUpdate(stopContext.Context.PartitionId, 1, (partitionId, value) => value + 1);
stopReasons[stopContext.Context.PartitionId] = stopContext.Reason;
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// CloseAsync should have not been called when constructing the event processor or during processing initialization.
Assert.That(closeCalls.Keys, Is.Empty);
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
var partitionIds = await connection.GetPartitionIdsAsync(DefaultRetryPolicy);
foreach (var partitionId in partitionIds)
{
Assert.That(closeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: CloseAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: CloseAsync should have been called only once.");
Assert.That(stopReasons.TryGetValue(partitionId, out ProcessingStoppedReason reason), Is.True, $"{ partitionId }: processing stopped reason should have been set.");
Assert.That(reason, Is.EqualTo(ProcessingStoppedReason.Shutdown), $"{ partitionId }: unexpected processing stopped reason.");
}
Assert.That(closeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
public async Task StartAsyncCallsPartitionProcessorInitializeAsync()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
var initializeCalls = new ConcurrentDictionary <string, int>();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
onInitialize: initializationContext =>
initializeCalls.AddOrUpdate(initializationContext.Context.PartitionId, 1, (partitionId, value) => value + 1)
);
eventProcessorManager.AddEventProcessors(1);
// InitializeAsync should have not been called when constructing the event processors.
Assert.That(initializeCalls.Keys, Is.Empty);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// Validate results before calling stop. This way, we can make sure the initialize calls were
// triggered by start.
var partitionIds = await connection.GetPartitionIdsAsync(DefaultRetryPolicy);
foreach (var partitionId in partitionIds)
{
Assert.That(initializeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: InitializeAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: InitializeAsync should have been called only once.");
}
Assert.That(initializeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
}
}
}
public async Task PartitionProcessorCanCreateACheckpoint()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
// Send some events.
EventData lastEvent;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
var partitionId = (await connection.GetPartitionIdsAsync(DefaultRetryPolicy)).First();
await using (var producer = new EventHubProducerClient(connection))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString))
await using (var receiver = consumer.CreatePartitionReceiver(partitionId, EventPosition.Earliest))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new InMemoryPartitionManager();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
partitionManager,
onProcessEvent: processorEvent =>
{
if (processorEvent.Data != null)
{
processorEvent.UpdateCheckpointAsync();
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
PartitionOwnership ownership = ownershipEnumerable.Single();
Assert.That(ownership.Offset.HasValue, Is.True);
Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));
Assert.That(ownership.SequenceNumber.HasValue, Is.True);
Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
}
}
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await connection.GetPartitionIdsAsync(DefaultRetryPolicy)).First();
await using (var producer = new EventHubProducerClient(connectionString))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connection))
await using (var receiver = consumer.CreatePartitionReceiver(partitionId, EventPosition.Earliest))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new InMemoryPartitionManager();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName,
EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
partitionManager,
onProcessEvent: processorEvent =>
{
if (processorEvent.Data != null)
{
Interlocked.Increment(ref receivedEventsCount);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
onProcessEvent: processorEvent =>
{
if (processorEvent.Data != null)
{
allReceivedEvents.AddOrUpdate
(
processorEvent.Context.PartitionId,
partitionId => new List <EventData>()
{
processorEvent.Data
},
(partitionId, list) =>
{
list.Add(processorEvent.Data);
return(list);
}
);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Send some events.
var partitionIds = await connection.GetPartitionIdsAsync(DefaultRetryPolicy);
var expectedEvents = new Dictionary <string, List <EventData> >();
foreach (var partitionId in partitionIds)
{
// Send a similar set of events for every partition.
expectedEvents[partitionId] = new List <EventData>
{
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
};
await using (var producer = new EventHubProducerClient(connection))
{
await producer.SendAsync(expectedEvents[partitionId], new SendOptions { PartitionId = partitionId });
}
}
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results. Make sure we received every event with the correct partition context,
// in the order they were sent.
foreach (var partitionId in partitionIds)
{
Assert.That(allReceivedEvents.TryGetValue(partitionId, out List <EventData> partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");
var index = 0;
foreach (EventData receivedEvent in partitionReceivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
++index;
}
}
Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
