public async Task StopAsyncCallsPartitionProcessorCloseAsyncWithShutdownReason()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var closeCalls = new ConcurrentDictionary <string, int>();
var closeReasons = new ConcurrentDictionary <string, PartitionProcessorCloseReason>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onClose: (partitionContext, checkpointManager, reason) =>
{
closeCalls.AddOrUpdate(partitionContext.PartitionId, 1, (partitionId, value) => value + 1);
closeReasons[partitionContext.PartitionId] = reason;
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// CloseAsync should have not been called when constructing the event processor or initializing the partition processors.
Assert.That(closeCalls.Keys, Is.Empty);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
var partitionIds = await client.GetPartitionIdsAsync();
foreach (var partitionId in partitionIds)
{
Assert.That(closeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: CloseAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: CloseAsync should have been called only once.");
Assert.That(closeReasons.TryGetValue(partitionId, out var reason), Is.True, $"{ partitionId }: close reason should have been set.");
Assert.That(reason, Is.EqualTo(PartitionProcessorCloseReason.Shutdown), $"{ partitionId }: unexpected close reason.");
}
Assert.That(closeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
public async Task PartitionDistributionIsEvenAfterLoadBalancing(int partitions, int eventProcessors)
{
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
ConcurrentDictionary <string, int> ownedPartitionsCount = new ConcurrentDictionary <string, int>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onInitialize: (partitionContext, checkpointManager) =>
ownedPartitionsCount.AddOrUpdate(checkpointManager.OwnerIdentifier, 1, (ownerId, value) => value + 1),
onClose: (partitionContext, checkpointManager, reason) =>
ownedPartitionsCount.AddOrUpdate(checkpointManager.OwnerIdentifier, 0, (ownerId, value) => value - 1)
);
hub.AddEventProcessors(eventProcessors);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Take a snapshot of the current partition balancing status.
var ownedPartitionsCountSnapshot = ownedPartitionsCount.ToArray().Select(kvp => kvp.Value);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
var minimumOwnedPartitionsCount = partitions / eventProcessors;
var maximumOwnedPartitionsCount = minimumOwnedPartitionsCount + 1;
foreach (var count in ownedPartitionsCountSnapshot)
{
Assert.That(count, Is.InRange(minimumOwnedPartitionsCount, maximumOwnedPartitionsCount));
}
Assert.That(ownedPartitionsCountSnapshot.Sum(), Is.EqualTo(partitions));
}
}
}
public async Task StartAsyncCallsPartitionProcessorInitializeAsync()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var initializeCalls = new ConcurrentDictionary <string, int>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onInitialize: (partitionContext, checkpointManager) =>
initializeCalls.AddOrUpdate(partitionContext.PartitionId, 1, (partitionId, value) => value + 1)
);
hub.AddEventProcessors(1);
// InitializeAsync should have not been called when constructing the event processors.
Assert.That(initializeCalls.Keys, Is.Empty);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Validate results before calling stop. This way, we can make sure the initialize calls were
// triggered by start.
var partitionIds = await client.GetPartitionIdsAsync();
foreach (var partitionId in partitionIds)
{
Assert.That(initializeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: InitializeAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: InitializeAsync should have been called only once.");
}
Assert.That(initializeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
// Stop the event processors.
await hub.StopAllAsync();
}
}
}
public async Task StopAsyncDoesNothingWhenEventProcessorIsNotRunning()
{
var partitions = 1;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int closeCallsCount = 0;
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onClose: (partitionContext, checkpointManager, reason) =>
Interlocked.Increment(ref closeCallsCount)
);
hub.AddEventProcessors(1);
// Calling StopAsync before starting the event processors shouldn't have any effect.
Assert.That(async() => await hub.StopAllAsync(), Throws.Nothing);
Assert.That(closeCallsCount, Is.EqualTo(0));
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// We should be able to call StopAsync again without getting an exception.
Assert.That(async() => await hub.StopAllAsync(), Throws.Nothing);
// Validate results.
Assert.That(closeCallsCount, Is.EqualTo(partitions));
}
}
}
public async Task PartitionProcessorProcessEventsAsyncIsCalledWithNoEvents()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var receivedEventSets = new ConcurrentBag <IEnumerable <EventData> >();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
receivedEventSets.Add(events)
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(receivedEventSets, Is.Not.Empty);
Assert.That(receivedEventSets.Any(set => (set == null || set.Any())), Is.False);
}
}
}
public async Task EventProcessorCannotReceiveMoreThanMaximumMessageCountMessagesAtATime(int maximumMessageCount)
{
var partitions = 2;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var unexpectedMessageCount = -1;
// Send some events.
await using (var producer = client.CreateProducer())
{
var eventSet = Enumerable
.Range(0, 20 * maximumMessageCount)
.Select(index => new EventData(new byte[10]))
.ToList();
// Send one set per partition.
for (int i = 0; i < partitions; i++)
{
await producer.SendAsync(eventSet);
}
}
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
MaximumMessageCount = maximumMessageCount
},
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
// In case we find a message count greater than the allowed amount, we only store the first
// occurrence and ignore the subsequent ones.
if (eventsList.Count > maximumMessageCount)
{
Interlocked.CompareExchange(ref unexpectedMessageCount, eventsList.Count, -1);
}
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(unexpectedMessageCount, Is.EqualTo(-1), $"A set of { unexpectedMessageCount } events was received.");
}
}
}
public async Task EventProcessorWaitsMaximumReceiveWaitTimeForEvents(int maximumWaitTimeInSecs)
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var timestamps = new ConcurrentDictionary <string, List <DateTimeOffset> >();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
MaximumReceiveWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs)
},
onInitialize: (partitionContext, checkpointManager) =>
timestamps.TryAdd(partitionContext.PartitionId, new List <DateTimeOffset> {
DateTimeOffset.UtcNow
}),
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
timestamps.AddOrUpdate
(
// The key already exists, so the 'addValue' factory will never be called.
partitionContext.PartitionId,
partitionId => null,
(partitionId, list) =>
{
list.Add(DateTimeOffset.UtcNow);
return(list);
}
)
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize. We are waiting a few times the maximum
// wait time span so we can have enough samples.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(4000 *maximumWaitTimeInSecs);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
foreach (var kvp in timestamps)
{
var partitionId = kvp.Key;
var partitionTimestamps = kvp.Value;
Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more timestamp samples were expected.");
for (int index = 1; index < partitionTimestamps.Count; index++)
{
var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds;
Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short.");
Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long.");
++index;
}
}
}
}
}
public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
DateTimeOffset enqueuedTime;
await using (var producer = client.CreateProducer())
{
// Send a few dummy events. We are not expecting to receive these.
for (int i = 0; i < 30; i++)
{
await producer.SendAsync(dummyEvent);
}
// Wait a reasonable amount of time so the events are able to reach the service.
await Task.Delay(1000);
// Send the events we expect to receive.
enqueuedTime = DateTimeOffset.UtcNow;
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
new EventProcessorOptions {
InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime)
},
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task EventProcessorCanStartAgainAfterStopping()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
hub.AddEventProcessors(1);
// Send some events.
var expectedEventsCount = 20;
await using (var producer = client.CreateProducer())
{
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// We'll start and stop the event processors twice. This way, we can assert they will behave
// the same way both times, reprocessing all events in the second run.
for (int i = 0; i < 2; i++)
{
receivedEventsCount = 0;
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount), $"Events should match in iteration { i + 1 }.");
}
}
}
}
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, checkpointManager, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
allReceivedEvents.AddOrUpdate
(
partitionContext.PartitionId,
partitionId => eventsList,
(partitionId, list) =>
{
list.AddRange(eventsList);
return(list);
}
);
}
}
);
hub.AddEventProcessors(1);
// Send some events.
var partitionIds = await client.GetPartitionIdsAsync();
var expectedEvents = new Dictionary <string, List <EventData> >();
foreach (var partitionId in partitionIds)
{
// Send a similar set of events for every partition.
expectedEvents[partitionId] = new List <EventData>
{
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
};
await using (var producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionId
}))
{
await producer.SendAsync(expectedEvents[partitionId]);
}
}
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results. Make sure we received every event in the correct partition processor,
// in the order they were sent.
foreach (var partitionId in partitionIds)
{
Assert.That(allReceivedEvents.TryGetValue(partitionId, out var partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");
var index = 0;
foreach (var receivedEvent in partitionReceivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
++index;
}
}
Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
