public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces()
{
await using (var storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient);
await checkpointStore.UpdateCheckpointAsync(new Checkpoint
("namespace1", "eventHubName", "consumerGroup", "partitionId", 10, 20));
await checkpointStore.UpdateCheckpointAsync(new Checkpoint
("namespace2", "eventHubName", "consumerGroup", "partitionId", 10, 20));
IEnumerable <Checkpoint> storedCheckpointsList1 = await checkpointStore.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <Checkpoint> storedCheckpointsList2 = await checkpointStore.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedCheckpointsList1, Is.Not.Null);
Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1));
Assert.That(storedCheckpointsList2, Is.Not.Null);
Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1));
}
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
await checkpointStore.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20), default);
await checkpointStore.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20), default);
IEnumerable <Checkpoint> storedCheckpointsList = await checkpointStore.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedCheckpointsList, Is.Not.Null);
Assert.That(storedCheckpointsList.Count, Is.EqualTo(2));
Checkpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1");
Checkpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2");
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
}
public async Task CheckpointUpdateFailsWhenContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
};
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(checkpoint, mockEvent, default), Throws.InstanceOf <RequestFailedException>());
}
}
public async Task BlobPartitionManagerCanUpdateCheckpoint()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
var ownershipList = new List <PartitionOwnership>
{
// Make sure the ownership exists beforehand so we hit all storage SDK calls in the checkpoint store.
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
)
};
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var checkpoint = new Checkpoint("namespace", "eventHubName", "consumerGroup", "partitionId", 10, 20);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(checkpoint, default), Throws.Nothing);
}
}
public async Task UpdateCheckpointLogsCheckpointUpdated()
{
var checkpoint = new Checkpoint(FullyQualifiedNamespace, EventHubName, ConsumerGroup, PartitionId, 0L, 0L);
var blobList = new List <BlobItem> {
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var target = new BlobsCheckpointStore(new MockBlobContainerClient()
{
Blobs = blobList
},
new BasicRetryPolicy(new EventHubsRetryOptions()));
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.UpdateCheckpointAsync(checkpoint, new CancellationToken());
mockLog.Verify(m => m.CheckpointUpdated(PartitionId));
}
public void UpdateCheckpointLogsErrorsWhenTheBlobDoesNotExist()
{
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
PartitionId = PartitionId,
};
var expectedException = new DllNotFoundException("BOOM!");
var mockLog = new Mock <BlobEventStoreEventSource>();
var mockContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client =>
{
client.UploadBlobException = expectedException;
});
var target = new BlobsCheckpointStore(mockContainerClient, DefaultRetryPolicy);
target.Logger = mockLog.Object;
Assert.That(async() => await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None), Throws.Exception.EqualTo(expectedException));
mockLog.Verify(log => log.UpdateCheckpointError(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, expectedException.Message));
}
internal virtual async Task CheckpointAsync(string partitionId, EventData checkpointEvent, CancellationToken cancellationToken = default)
{
await _checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint()
{
PartitionId = partitionId,
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace
}, checkpointEvent, cancellationToken).ConfigureAwait(false);
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId1"
}, mockEvent, default);
await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId2"
}, mockEvent, default);
IEnumerable <EventProcessorCheckpoint> storedCheckpointsList = await checkpointStore.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedCheckpointsList, Is.Not.Null);
Assert.That(storedCheckpointsList.Count, Is.EqualTo(2));
EventProcessorCheckpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1");
EventProcessorCheckpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2");
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherNamespaces()
{
await using (var storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace1",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId",
Offset = 10,
SequenceNumber = 20
}, default);
await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace2",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId",
Offset = 10,
SequenceNumber = 20
}, default);
IEnumerable<EventProcessorCheckpoint> storedCheckpointsList1 = await checkpointStore.ListCheckpointsAsync("namespace1", "eventHubName", "consumerGroup", default);
IEnumerable<EventProcessorCheckpoint> storedCheckpointsList2 = await checkpointStore.ListCheckpointsAsync("namespace2", "eventHubName", "consumerGroup", default);
Assert.That(storedCheckpointsList1, Is.Not.Null);
Assert.That(storedCheckpointsList1.Count, Is.EqualTo(1));
Assert.That(storedCheckpointsList2, Is.Not.Null);
Assert.That(storedCheckpointsList2.Count, Is.EqualTo(1));
}
}
public async Task CheckpointUpdateFailsWhenContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(new Checkpoint
("namespace", "eventHubName", "consumerGroup", "partitionId", offset: 10, sequenceNumber: 20), default), Throws.InstanceOf <RequestFailedException>());
}
}
public void UpdateCheckpointForMissingCheckpointThrowsAndLogsCheckpointUpdateError()
{
var checkpoint = new Checkpoint(FullyQualifiedNamespace, EventHubName, ConsumerGroup, PartitionId, 0L, 0L);
var ex = new RequestFailedException(404, BlobErrorCode.ContainerNotFound.ToString(), BlobErrorCode.ContainerNotFound.ToString(), null);
var target = new BlobsCheckpointStore(new MockBlobContainerClient(blobClientUploadBlobException: ex),
new BasicRetryPolicy(new EventHubsRetryOptions()));
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
Assert.ThrowsAsync <RequestFailedException>(async() => await target.UpdateCheckpointAsync(checkpoint, new CancellationToken()));
mockLog.Verify(m => m.CheckpointUpdateError(PartitionId, It.Is <string>(s => s.Contains(BlobErrorCode.ContainerNotFound.ToString()))));
}
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobExists()
{
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
PartitionId = PartitionId,
};
var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{MatchingEtag}"""), DateTime.UtcNow);
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var mockContainerClient = new MockBlobContainerClient()
{
Blobs = blobList
};
mockContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client =>
{
client.BlobInfo = blobInfo;
client.UploadBlobException = new Exception("Upload should not be called");
});
var target = new BlobsCheckpointStore(mockContainerClient, DefaultRetryPolicy);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None);
mockLog.Verify(log => log.UpdateCheckpointStart(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup));
mockLog.Verify(log => log.UpdateCheckpointComplete(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup));
}
public void UpdateCheckpointForMissingContainerLogsCheckpointUpdateError()
{
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
PartitionId = PartitionId
};
var ex = new RequestFailedException(404, BlobErrorCode.ContainerNotFound.ToString(), BlobErrorCode.ContainerNotFound.ToString(), null);
var mockBlobContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client => client.UploadBlobException = ex);
var target = new BlobsCheckpointStore(mockBlobContainerClient, DefaultRetryPolicy);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
Assert.That(async() => await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None), Throws.InstanceOf <RequestFailedException>());
mockLog.Verify(m => m.UpdateCheckpointError(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, ex.Message));
}
public async Task BlobStorageManagerCanUpdateCheckpoint()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy);
var ownershipList = new List <EventProcessorPartitionOwnership>
{
// Make sure the ownership exists beforehand so we hit all storage SDK calls in the checkpoint store.
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
}
};
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
};
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(checkpoint, mockEvent, default), Throws.Nothing);
}
}