public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces()
{
var storageManager = new MockCheckPointStorage();
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace1",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
}, mockEvent);
await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace2",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
}, mockEvent);
IEnumerable <EventProcessorCheckpoint> storedCheckpointsList1 = await storageManager.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <EventProcessorCheckpoint> storedCheckpointsList2 = await storageManager.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList1, Is.Not.Null);
Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1));
Assert.That(storedCheckpointsList2, Is.Not.Null);
Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1));
}
public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
{
var partitionManager = new MockCheckPointStorage();
IEnumerable <PartitionOwnership> ownership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(ownership, Is.Not.Null.And.Empty);
}
public async Task CheckpointUpdateUpdatesOwnershipInformation()
{
var partitionManager = new MockCheckPointStorage();
var originalOwnership = new PartitionOwnership
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)));
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
originalOwnership
});
// ETag must have been set by the partition manager.
DateTimeOffset?originalLastModifiedTime = originalOwnership.LastModifiedTime;
var originalETag = originalOwnership.ETag;
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", 10, 20));
// Make sure the ownership has changed, even though the instance should be the same.
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));
Assert.That(originalOwnership.Offset, Is.EqualTo(10));
Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(20));
Assert.That(originalOwnership.LastModifiedTime, Is.GreaterThan(originalLastModifiedTime));
Assert.That(originalOwnership.ETag, Is.Not.EqualTo(originalETag));
}
public async Task CheckpointUpdateFailsWhenOwnerChanges()
{
var partitionManager = new MockCheckPointStorage();
var originalOwnership = new PartitionOwnership
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier1", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow);
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
originalOwnership
});
// ETag must have been set by the partition manager.
DateTimeOffset?originalLastModifiedTime = originalOwnership.LastModifiedTime;
var originalETag = originalOwnership.ETag;
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier2", "partitionId", 10, 20));
// Make sure the ownership hasn't changed.
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));
Assert.That(originalOwnership.OwnerIdentifier, Is.EqualTo("ownerIdentifier1"));
Assert.That(originalOwnership.Offset, Is.EqualTo(1));
Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(2));
Assert.That(originalOwnership.LastModifiedTime, Is.EqualTo(originalLastModifiedTime));
Assert.That(originalOwnership.ETag, Is.EqualTo(originalETag));
}
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount));
Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True);
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
var storageManager = new MockCheckPointStorage();
await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId1",
Offset = 10,
SequenceNumber = 20
});
await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId2",
Offset = 10,
SequenceNumber = 20
});
IEnumerable <EventProcessorCheckpoint> storedCheckpointsList = await storageManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList, Is.Not.Null);
Assert.That(storedCheckpointsList.Count, Is.EqualTo(2));
EventProcessorCheckpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1");
EventProcessorCheckpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2");
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
public async Task CheckpointUpdateFailsWhenAssociatedOwnershipDoesNotExist()
{
var partitionManager = new MockCheckPointStorage();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset : 10, sequenceNumber : 20));
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership, Is.Empty);
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
// The ETags must have been set by the partition manager.
var eTags = ownershipList.Select(ownership => ownership.ETag).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }",
offset: i,
eTag: i % 2 == 1 ? eTags[i] : null
));
}
IEnumerable <PartitionOwnership> claimedOwnership = await partitionManager.ClaimOwnershipAsync(ownershipList);
Assert.That(claimedOwnership, Is.Not.Null);
Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount));
Assert.That(claimedOwnership.OrderBy(ownership => ownership.Offset).SequenceEqual(ownershipList.Where(ownership => ownership.Offset % 2 == 1)), Is.True);
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
var storageManager = new MockCheckPointStorage();
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{ i }"
});
}
await storageManager.ClaimOwnershipAsync(ownershipList);
// The versions must have been set by the storage manager.
var versions = ownershipList.Select(ownership => ownership.Version).ToList();
ownershipList.Clear();
// Use a valid version when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{ i }",
Version = i % 2 == 1 ? versions[i] : null
});
}
IEnumerable <EventProcessorPartitionOwnership> claimedOwnership = await storageManager.ClaimOwnershipAsync(ownershipList);
Assert.That(claimedOwnership, Is.Not.Null);
Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount));
Assert.That(claimedOwnership.OrderBy(ownership => int.Parse(ownership.PartitionId)).SequenceEqual(ownershipList.Where(ownership => int.Parse(ownership.PartitionId) % 2 == 1)), Is.True);
}
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace1",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
// ETag must have been set by the partition manager.
var eTag = firstOwnership.ETag;
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace2",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedOwnership1, Is.Not.Null);
Assert.That(storedOwnership1.Count, Is.EqualTo(1));
Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership));
Assert.That(storedOwnership2, Is.Not.Null);
Assert.That(storedOwnership2.Count, Is.EqualTo(1));
Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership));
}
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces()
{
var storageManager = new MockCheckPointStorage();
var ownershipList = new List <EventProcessorPartitionOwnership>();
var firstOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace1",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
ownershipList.Add(firstOwnership);
await storageManager.ClaimOwnershipAsync(ownershipList);
// Version must have been set by the storage manager.
var version = firstOwnership.Version;
ownershipList.Clear();
var secondOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace2",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = version
};
ownershipList.Add(secondOwnership);
await storageManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <EventProcessorPartitionOwnership> storedOwnership1 = await storageManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <EventProcessorPartitionOwnership> storedOwnership2 = await storageManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedOwnership1, Is.Not.Null);
Assert.That(storedOwnership1.Count, Is.EqualTo(1));
Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership));
Assert.That(storedOwnership2, Is.Not.Null);
Assert.That(storedOwnership2.Count, Is.EqualTo(1));
Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership));
}
public async Task OwnershipClaimSucceedsWhenETagIsValid()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 1
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
// ETag must have been set by the partition manager.
var eTag = firstOwnership.ETag;
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 2,
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(secondOwnership));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces()
{
var partitionManager = new MockCheckPointStorage();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace1", "eventHubName", "consumerGroup", "partitionId", 10, 20));
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace2", "eventHubName", "consumerGroup", "partitionId", 10, 20));
IEnumerable <Checkpoint> storedCheckpointsList1 = await partitionManager.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <Checkpoint> storedCheckpointsList2 = await partitionManager.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList1, Is.Not.Null);
Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1));
Assert.That(storedCheckpointsList2, Is.Not.Null);
Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1));
}
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag)
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
var partitionManager = new MockCheckPointStorage();
var ownership1 = new PartitionOwnership
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", offset: 1);
var ownership2 = new PartitionOwnership
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId2", offset: 1);
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
ownership1,
ownership2
});
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", 10, 20));
Assert.That(ownership1.Offset, Is.EqualTo(10));
Assert.That(ownership2.Offset, Is.EqualTo(1));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
var partitionManager = new MockCheckPointStorage();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20));
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20));
IEnumerable <Checkpoint> storedCheckpointsList = await partitionManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList, Is.Not.Null);
Assert.That(storedCheckpointsList.Count, Is.EqualTo(2));
Checkpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1");
Checkpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2");
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
public async Task OwnershipClaimFailsWhenVersionIsInvalid(string version)
{
var storageManager = new MockCheckPointStorage();
var ownershipList = new List <EventProcessorPartitionOwnership>();
var firstOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
ownershipList.Add(firstOwnership);
await storageManager.ClaimOwnershipAsync(ownershipList);
ownershipList.Clear();
var secondOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = version
};
ownershipList.Add(secondOwnership);
await storageManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <EventProcessorPartitionOwnership> storedOwnership = await storageManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership));
}
public async Task FirstOwnershipClaimSucceeds()
{
var partitionManager = new MockCheckPointStorage();
var ownershipList = new List <PartitionOwnership>();
var ownership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(ownership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(ownership));
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await connection.GetPartitionIdsAsync(DefaultRetryPolicy)).First();
await using (var producer = new EventHubProducerClient(connectionString))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connection))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
// TODO: Convert to iterator
//receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new MockCheckPointStorage();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(connection.FullyQualifiedNamespace, connection.EventHubName,
EventHubConsumerClient.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
partitionManager,
onProcessEvent: eventArgs =>
{
if (eventArgs.Data != null)
{
Interlocked.Increment(ref receivedEventsCount);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task PartitionProcessorCanCreateACheckpoint()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
// Send some events.
EventData lastEvent;
await using (var producer = new EventHubProducerClient(connection))
await using (var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
using (var dummyBatch = await producer.CreateBatchAsync())
{
for (int i = 0; i < dummyEventsCount; i++)
{
dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I'm dummy.")));
}
await producer.SendAsync(dummyBatch);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
Assert.Fail("Convert to iterator.");
//receivedEvents.AddRange(await receiver.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a storage manager so we can retrieve the created checkpoint from it.
var storageManager = new MockCheckPointStorage();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
storageManager,
onProcessEvent: eventArgs =>
{
if (eventArgs.Data != null)
{
eventArgs.UpdateCheckpointAsync();
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await storageManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
}
}
}
public async Task ProcessorDoesNotProcessCheckpointedEventsAgain()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var firstEventBatch = Enumerable
.Range(0, 20)
.Select(index => new EventData(Encoding.UTF8.GetBytes($"First event batch: { index }")))
.ToList();
var secondEventBatch = Enumerable
.Range(0, 10)
.Select(index => new EventData(Encoding.UTF8.GetBytes($"Second event batch: { index }")))
.ToList();
var completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var firstBatchReceivedEventsCount = 0;
// Send the first batch of events and checkpoint after the last one.
var checkpointStorage = new MockCheckPointStorage();
var firstProcessor = new EventProcessorClient(checkpointStorage, EventHubConsumerClient.DefaultConsumerGroupName,
TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, () => new EventHubConnection(connectionString), default);
firstProcessor.ProcessEventAsync += async eventArgs =>
{
if (++firstBatchReceivedEventsCount == firstEventBatch.Count)
{
await eventArgs.UpdateCheckpointAsync();
completionSource.SetResult(true);
}
};
firstProcessor.ProcessErrorAsync += eventArgs => Task.CompletedTask;
await using (var producer = new EventHubProducerClient(connectionString))
{
using var batch = await producer.CreateBatchAsync();
foreach (var eventData in firstEventBatch)
{
batch.TryAdd(eventData);
}
await producer.SendAsync(batch);
}
// Establish timed cancellation to ensure that the test doesn't hang.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromMinutes(5));
await firstProcessor.StartProcessingAsync(cancellationSource.Token);
while (!completionSource.Task.IsCompleted &&
!cancellationSource.IsCancellationRequested)
{
await Task.Delay(25);
}
await firstProcessor.StopProcessingAsync(cancellationSource.Token);
// Send the second batch of events. Only the new events should be read by the second processor.
await using (var producer = new EventHubProducerClient(connectionString))
{
using var batch = await producer.CreateBatchAsync();
foreach (var eventData in secondEventBatch)
{
batch.TryAdd(eventData);
}
await producer.SendAsync(batch);
}
completionSource = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var secondBatchReceivedEvents = new List <EventData>();
var secondProcessor = new EventProcessorClient(checkpointStorage, EventHubConsumerClient.DefaultConsumerGroupName,
TestEnvironment.FullyQualifiedNamespace, scope.EventHubName, () => new EventHubConnection(connectionString), default);
secondProcessor.ProcessEventAsync += eventArgs =>
{
secondBatchReceivedEvents.Add(eventArgs.Data);
if (secondBatchReceivedEvents.Count == firstEventBatch.Count)
{
completionSource.SetResult(true);
}
return(Task.CompletedTask);
};
var wasErrorHandlerCalled = false;
secondProcessor.ProcessErrorAsync += eventArgs =>
{
wasErrorHandlerCalled = true;
return(Task.CompletedTask);
};
await secondProcessor.StartProcessingAsync(cancellationSource.Token);
while (!completionSource.Task.IsCompleted &&
!cancellationSource.IsCancellationRequested)
{
await Task.Delay(25);
}
await secondProcessor.StopProcessingAsync(cancellationSource.Token);
Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The processors should have stopped without cancellation.");
Assert.That(wasErrorHandlerCalled, Is.False, "No errors should have happened while resuming from checkpoint.");
var index = 0;
foreach (var eventData in secondBatchReceivedEvents)
{
Assert.That(eventData.IsEquivalentTo(secondEventBatch[index]), Is.True, "The received and sent event datas do not match.");
index++;
}
Assert.That(index, Is.EqualTo(secondEventBatch.Count), $"The second processor did not receive the expected amount of events.");
}
}
public async Task PartitionProcessorCanCreateACheckpoint()
{
await using EventHubScope scope = await EventHubScope.CreateAsync(1);
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using var connection = new EventHubConnection(connectionString);
// Send some events.
EventData lastEvent;
await using var producer = new EventHubProducerClient(connection);
await using var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString);
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
using var dummyBatch = await producer.CreateBatchAsync();
for (int i = 0; i < dummyEventsCount; i++)
{
dummyBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(eventBody)));
}
await producer.SendAsync(dummyBatch);
var receivedEvents = new List <EventData>();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
await foreach (var evt in consumer.ReadEventsAsync(new ReadEventOptions {
MaximumWaitTime = TimeSpan.FromSeconds(30)
}, cancellationSource.Token))
{
receivedEvents.Add(evt.Data);
if (receivedEvents.Count == dummyEventsCount)
{
break;
}
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
// Create a storage manager so we can retrieve the created checkpoint from it.
var storageManager = new MockCheckPointStorage();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
storageManager,
onProcessEvent: eventArgs =>
{
if (eventArgs.Data != null)
{
eventArgs.UpdateCheckpointAsync();
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <EventProcessorPartitionOwnership> ownershipEnumerable = await storageManager.ListOwnershipAsync(connection.FullyQualifiedNamespace, connection.EventHubName, EventHubConsumerClient.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
}
