public async Task BlobStorageManagerCanGetACheckpoint()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
Assert.That(async() => await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partition", default), Throws.Nothing);
}
}
public async Task ListOwnershipFailsWhenContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
Assert.That(async() => await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default), Throws.InstanceOf <RequestFailedException>());
}
}
public async Task GetCheckpointReturnsNullIfTheContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var checkpoints = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partition", default);
Assert.That(checkpoints, Is.Null);
}
}
public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownership = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(ownership, Is.Not.Null.And.Empty);
}
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20, default);
await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20, default);
var storedCheckpoint1 = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId1", default);
var storedCheckpoint2 = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId2", default);
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
}
public async Task OwnershipClaimSucceedsWhenVersionIsValid()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var firstOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
await checkpointStore.ClaimOwnershipAsync(new[] { firstOwnership }, default);
// The first ownership's version should have been set by the checkpoint store.
var secondOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = firstOwnership.Version
};
await checkpointStore.ClaimOwnershipAsync(new[] { secondOwnership }, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null);
Assert.That(storedOwnershipList.Count, Is.EqualTo(1));
Assert.That(storedOwnershipList.Single().IsEquivalentTo(secondOwnership), Is.True);
}
}
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownershipCount = 5;
for (var i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"partitionId { i }"
});
}
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null);
Assert.That(storedOwnershipList.Count, Is.EqualTo(ownershipCount));
var index = 0;
foreach (EventProcessorPartitionOwnership ownership in storedOwnershipList.OrderBy(ownership => ownership.PartitionId))
{
Assert.That(ownership.IsEquivalentTo(ownershipList[index]), Is.True, $"Ownership of partition '{ ownership.PartitionId }' should be equivalent.");
++index;
}
}
}
public async Task FixtureSetUp()
{
_eventHubScope = await EventHubScope.CreateAsync(2);
_storageScope = await StorageScope.CreateAsync();
}
public async Task CheckpointUpdatesAnExistingBlob()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
};
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
// Calling update should create the checkpoint.
await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default);
var blobCount = 0;
var storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default);
await foreach (var blob in containerClient.GetBlobsAsync())
{
++blobCount;
if (blobCount > 1)
{
break;
}
}
Assert.That(blobCount, Is.EqualTo(1));
Assert.That(storedCheckpoint, Is.Not.Null);
Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false)));
// Calling update again should update the existing checkpoint.
mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 50,
sequenceNumber: 60);
await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default);
blobCount = 0;
storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default);
await foreach (var blob in containerClient.GetBlobsAsync())
{
++blobCount;
if (blobCount > 1)
{
break;
}
}
Assert.That(blobCount, Is.EqualTo(1));
Assert.That(storedCheckpoint, Is.Not.Null);
Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false)));
}
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{i}"
});
}
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
// The versions must have been set by the checkpoint store.
var versions = ownershipList.Select(ownership => ownership.Version).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{i}",
Version = i % 2 == 1 ? versions[i] : null
});
}
var claimedOwnershipList = await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var expectedOwnership = ownershipList.Where(ownership => int.Parse(ownership.PartitionId) % 2 == 1);
Assert.That(claimedOwnershipList, Is.Not.Null);
Assert.That(claimedOwnershipList.Count, Is.EqualTo(expectedClaimedCount));
var index = 0;
foreach (EventProcessorPartitionOwnership ownership in claimedOwnershipList.OrderBy(ownership => ownership.PartitionId))
{
Assert.That(ownership.IsEquivalentTo(expectedOwnership.ElementAt(index)), Is.True, $"Ownership of partition '{ ownership.PartitionId }' should be equivalent.");
++index;
}
}
}
