public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
{
var partitionManager = new MockCheckPointStorage();
IEnumerable <PartitionOwnership> ownership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(ownership, Is.Not.Null.And.Empty);
}
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount));
Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True);
}
public async Task RunLoadBalancingAsyncClaimsAllClaimablePartitions()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the loadbalancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by loadbalancer.
Assert.That(completeOwnership.Count(), Is.EqualTo(NumberOfPartitions));
}
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing()
{
string fullyQualifiedNamespace = "namespace";
string eventHubName = "eventHub";
var mockStorage = new MockCheckPointStorage();
var mockConsumer = new Mock<EventHubConsumerClient>("cg", Mock.Of<EventHubConnection>(), default);
var mockProcessor = new Mock<EventProcessorClient>(mockStorage, "cg", fullyQualifiedNamespace, eventHubName, Mock.Of<Func<EventHubConnection>>(), default, default) { CallBase = true };
public async Task RelinquishOwnershipAsyncRelinquishesPartitionOwnershipOtherClientsConsiderThemClaimableImmediately()
{
const int NumberOfPartitions = 3;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer1 = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
var loadbalancer2 = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Ownership should start empty.
var completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
Assert.That(completeOwnership.Count(), Is.EqualTo(0));
// Start the loadbalancer so that it claims a random partition until none are left.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer1.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await partitionManager.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup);
// All partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
// Stopping the loadbalancer should relinquish all partition ownership.
await loadbalancer1.RelinquishOwnershipAsync(CancellationToken.None);
completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// No partitions are owned by loadbalancer1.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer1.OwnerIdentifier)), Is.EqualTo(0));
// Start loadbalancer2 so that the loadbalancer claims a random partition until none are left.
// All partitions should be immediately claimable even though they were just claimed by the loadbalancer1.
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer2.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
completeOwnership = await partitionManager.ListOwnershipAsync(loadbalancer1.FullyQualifiedNamespace, loadbalancer1.EventHubName, loadbalancer1.ConsumerGroup);
// All partitions are owned by loadbalancer2.
Assert.That(completeOwnership.Count(p => p.OwnerIdentifier.Equals(loadbalancer2.OwnerIdentifier)), Is.EqualTo(NumberOfPartitions));
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"{ i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
// The ETags must have been set by the partition manager.
var eTags = ownershipList.Select(ownership => ownership.ETag).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"{ i }",
eTag: i % 2 == 1 ? eTags[i] : null
));
}
IEnumerable <PartitionOwnership> claimedOwnership = await partitionManager.ClaimOwnershipAsync(ownershipList);
Assert.That(claimedOwnership, Is.Not.Null);
Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount));
Assert.That(claimedOwnership.OrderBy(ownership => int.Parse(ownership.PartitionId)).SequenceEqual(ownershipList.Where(ownership => int.Parse(ownership.PartitionId) % 2 == 1)), Is.True);
}
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace1",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
// ETag must have been set by the partition manager.
var eTag = firstOwnership.ETag;
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace2",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedOwnership1, Is.Not.Null);
Assert.That(storedOwnership1.Count, Is.EqualTo(1));
Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership));
Assert.That(storedOwnership2, Is.Not.Null);
Assert.That(storedOwnership2.Count, Is.EqualTo(1));
Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces()
{
var partitionManager = new MockCheckPointStorage();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace1", "eventHubName", "consumerGroup", "partitionId", 10, 20));
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace2", "eventHubName", "consumerGroup", "partitionId", 10, 20));
IEnumerable <Checkpoint> storedCheckpointsList1 = await partitionManager.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <Checkpoint> storedCheckpointsList2 = await partitionManager.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList1, Is.Not.Null);
Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1));
Assert.That(storedCheckpointsList2, Is.Not.Null);
Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1));
}
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag)
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
var partitionManager = new MockCheckPointStorage();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20));
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20));
IEnumerable <Checkpoint> storedCheckpointsList = await partitionManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList, Is.Not.Null);
Assert.That(storedCheckpointsList.Count, Is.EqualTo(2));
Checkpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1");
Checkpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2");
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
public async Task VerifiesEventProcessorLogs()
{
const int NumberOfPartitions = 4;
const int MinimumpartitionCount = NumberOfPartitions / 2;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by a different loadbalancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var completeOwnership = CreatePartitionOwnership(partitionIds.Skip(1), loadbalancer2Id);
// Seed the partitionManager with the owned partitions.
await partitionManager.ClaimOwnershipAsync(completeOwnership);
var mockLog = new Mock <PartitionLoadBalancerEventSource>();
loadbalancer.Logger = mockLog.Object;
for (int i = 0; i < NumberOfPartitions; i++)
{
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
}
await loadbalancer.RelinquishOwnershipAsync(CancellationToken.None);
mockLog.Verify(m => m.RenewOwnershipStart(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.RenewOwnershipComplete(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ClaimOwnershipStart(It.Is <string>(p => partitionIds.Contains(p))));
mockLog.Verify(m => m.MinimumPartitionsPerEventProcessor(MinimumpartitionCount));
mockLog.Verify(m => m.CurrentOwnershipCount(MinimumpartitionCount, loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.StealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.ShouldStealPartition(loadbalancer.OwnerIdentifier));
mockLog.Verify(m => m.UnclaimedPartitions(It.Is <HashSet <string> >(p => p.Overlaps(partitionIds))));
}
public async Task FirstOwnershipClaimSucceeds()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(ownership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(ownership));
}
public async Task RunLoadBalancingAsyncStealsPartitionsWhenThisLoadbalancerOwnsLessThanMinPartitionsAndOtherLoadbalancerOwnsMaxPartitions()
{
const int MinimumpartitionCount = 4;
const int MaximumpartitionCount = 5;
const int NumberOfPartitions = 12;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create more partitions owned by this loadbalancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount - 1);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create more partitions owned by a different loadbalancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create more partitions owned by a different loadbalancer above the MaximumPartitionCount.
var loadbalancer3Id = Guid.NewGuid().ToString();
var stealablePartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MaximumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(stealablePartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the partitionManager with the owned partitions.
await partitionManager.ClaimOwnershipAsync(completeOwnership);
// Get owned partitions.
var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
var ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.False);
// Verify loadbalancer 3 has stealable partitions.
Assert.That(ownedByloadbalancer3.Count(), Is.EqualTo(MaximumpartitionCount));
// Start the loadbalancer to steal owership from of a when ownedPartitionCount == MinimumOwnedPartitionsCount but a loadbalancer owns > MaximumPartitionCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
ownedByloadbalancer3 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer3Id);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Any(owned => stealablePartitionIds.Contains(int.Parse(owned.PartitionId))), Is.True);
// Verify loadbalancer 3 now does not own > MaximumPartitionCount.
Assert.That(ownedByloadbalancer3.Count(), Is.LessThan(MaximumpartitionCount));
}
public async Task RunLoadBalancingAsyncClaimsPartitionsWhenOwnedEqualsMinimumOwnedPartitionsCount()
{
const int MinimumpartitionCount = 4;
const int NumberOfPartitions = 13;
var partitionIds = Enumerable.Range(1, NumberOfPartitions).Select(p => p.ToString()).ToArray();
var partitionManager = new MockCheckPointStorage((s) => Console.WriteLine(s));
var loadbalancer = new PartitionLoadBalancer(
partitionManager, Guid.NewGuid().ToString(), ConsumerGroup, FullyQualifiedNamespace, EventHubName, TimeSpan.FromMinutes(1));
// Create partitions owned by this loadbalancer.
var loadbalancer1PartitionIds = Enumerable.Range(1, MinimumpartitionCount);
var completeOwnership = CreatePartitionOwnership(loadbalancer1PartitionIds.Select(i => i.ToString()), loadbalancer.OwnerIdentifier);
// Create partitions owned by a different loadbalancer.
var loadbalancer2Id = Guid.NewGuid().ToString();
var loadbalancer2PartitionIds = Enumerable.Range(loadbalancer1PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer2PartitionIds.Select(i => i.ToString()), loadbalancer2Id));
// Create partitions owned by a different loadbalancer.
var loadbalancer3Id = Guid.NewGuid().ToString();
var loadbalancer3PartitionIds = Enumerable.Range(loadbalancer2PartitionIds.Max() + 1, MinimumpartitionCount);
completeOwnership = completeOwnership
.Concat(CreatePartitionOwnership(loadbalancer3PartitionIds.Select(i => i.ToString()), loadbalancer3Id));
// Seed the partitionManager with all partitions.
await partitionManager.ClaimOwnershipAsync(completeOwnership);
var claimablePartitionIds = partitionIds.Except(completeOwnership.Select(p => p.PartitionId));
// Get owned partitions.
var totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
var ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify owned partitionIds match the owned partitions.
Assert.That(ownedByloadbalancer1.Count(), Is.EqualTo(MinimumpartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.False);
// Start the loadbalancer to claim owership from of a Partition even though ownedPartitionCount == MinimumOwnedPartitionsCount.
await loadbalancer.RunLoadBalancingAsync(partitionIds, CancellationToken.None);
// Get owned partitions.
totalOwnedPartitions = await partitionManager.ListOwnershipAsync(loadbalancer.FullyQualifiedNamespace, loadbalancer.EventHubName, loadbalancer.ConsumerGroup);
ownedByloadbalancer1 = totalOwnedPartitions.Where(p => p.OwnerIdentifier == loadbalancer.OwnerIdentifier);
// Verify that we took ownership of the additional partition.
Assert.That(ownedByloadbalancer1.Count(), Is.GreaterThan(MinimumpartitionCount));
Assert.That(ownedByloadbalancer1.Any(owned => claimablePartitionIds.Contains(owned.PartitionId)), Is.True);
}
public async Task PartitionProcessorCanCreateACheckpoint()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
// Send some events.
EventData lastEvent;
await using (var producer = new EventHubProducerClient(connection))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
using (var dummyBatch = await producer.CreateBatchAsync())
{
for (int i = 0; i < dummyEventsCount; i++)
{
dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy.")));
}
await producer.SendAsync(dummyBatch);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
Assert.Fail("Convert to iterator.");
//receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new MockCheckPointStorage();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
partitionManager,
onProcessEvent: eventArgs =>
{
if (eventArgs.Data != null)
{
eventArgs.UpdateCheckpointAsync();
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
}
}
}
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing()
{
var mockStorage = new MockCheckPointStorage();
var mockConsumer = new Mock <EventHubConsumerClient>("cg", Mock.Of <EventHubConnection>(), default);
var mockProcessor = new Mock <EventProcessorClient>(mockStorage, "cg", "ns", "eh", Mock.Of <Func <EventHubConnection> >(), default, default)
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing()
{
var mockStorage = new MockCheckPointStorage();
var mockConsumer = new Mock <EventHubConsumerClient>("cg", Mock.Of <EventHubConnection>(), default);
var mockProcessor = new Mock <EventProcessorClient>(mockStorage, "cg", "ns", "eh", Mock.Of <Func <EventHubConnection> >(), default)
{
CallBase = true
};
using ClientDiagnosticListener listener = new ClientDiagnosticListener(DiagnosticSourceName);
var completionSource = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var processEventCalls = 0;
mockConsumer
.Setup(consumer => consumer.ReadEventsFromPartitionAsync(
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <ReadEventOptions>(),
It.IsAny <CancellationToken>()))
.Returns <string, EventPosition, ReadEventOptions, CancellationToken>((partitionId, position, options, token) =>
{
async IAsyncEnumerable <PartitionEvent> mockPartitionEventEnumerable()
{
var context = new MockPartitionContext(partitionId);
yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>())
{
Properties = { { "Diagnostic-Id", "id" } }
}));
yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>())
{
Properties = { { "Diagnostic-Id", "id2" } }
}));
while (!completionSource.Task.IsCompleted && !token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1));
yield return(new PartitionEvent());
}
yield break;
};
return(mockPartitionEventEnumerable());
});
mockProcessor
.Setup(processor => processor.CreateConsumer(
It.IsAny <string>(),
It.IsAny <EventHubConnection>(),
It.IsAny <EventHubConsumerClientOptions>()))
.Returns(mockConsumer.Object);
mockProcessor.Object.ProcessEventAsync += eventArgs =>
{
if (++processEventCalls == 2)
{
completionSource.SetResult(null);
}
return(Task.CompletedTask);
};
// RunPartitionProcessingAsync does not invoke the error handler, but we are setting it here in case
// this fact changes in the future.
mockProcessor.Object.ProcessErrorAsync += eventArgs => Task.CompletedTask;
// Start processing and wait for the consumer to be invoked. Set a cancellation for backup to ensure
// that the test completes deterministically.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(15));
using var partitionProcessingTask = Task.Run(() => mockProcessor.Object.RunPartitionProcessingAsync("pid", default, cancellationSource.Token));
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
string partitionId;
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
long?checkpointedSequenceNumber = default;
await using (var producer = new EventHubProducerClient(connectionString))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connection))
{
partitionId = (await consumer.GetPartitionIdsAsync()).First();
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
using (var dummyBatch = await producer.CreateBatchAsync())
{
for (int i = 0; i < dummyEventsCount; i++)
{
dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy.")));
}
await producer.SendAsync(dummyBatch);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
Assert.Fail("Convert to iterator");
//receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
using (var dummyBatch = await producer.CreateBatchAsync())
{
for (int i = 0; i < expectedEventsCount; i++)
{
dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy.")));
}
await producer.SendAsync(dummyBatch);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new MockCheckPointStorage();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName,
EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
partitionManager,
onProcessEvent: eventArgs =>
{
if (eventArgs.Data != null)
{
Interlocked.Increment(ref receivedEventsCount);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
