protected Mock <DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
{
var mockFileSystemClient = new Mock <DataLakeFileSystemClient>();
mockFileSystemClient.SetupGet(x => x.Name).Returns(ContainerName);
mockFileSystemClient.SetupGet(x => x.AccountName).Returns(AccountUri);
mockFileSystemClient
.Setup(x => x.GetDirectoryClient(It.IsAny <String>()))
.Returns <string>(BuildMockDataLakeDirectoryClient <DataLakeDirectoryClient>);
mockFileSystemClient
.Setup(x => x.GetFileClient(It.IsAny <String>()))
.Returns <string>(BuildMockDataLakeDirectoryClient <DataLakeFileClient>);
mockFileSystemClient
.Setup(x => x.GetPaths(It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <CancellationToken>()))
.Returns((string path, bool recursive, bool userPrinciaplName, CancellationToken token) =>
{
var pathLength = 1 + (path?.Length ?? 0);
var items = TestData
// Include all files starting with the test path, or root paths if the test path is null
.Where(x => x.Name.StartsWith(path ?? string.Empty) && x.Name != path)
// Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains
// directory separator to exclude sub dirs
.Where(x => recursive || !x.Name.Substring(pathLength > x.Name.Length ? x.Name.Length : pathLength).Contains('/'))
.ToList()
.AsReadOnly();
var page = Page <PathItem> .FromValues(items, null, Mock.Of <Response>());
return(Pageable <PathItem> .FromPages(new[] { page }));
});
return(mockFileSystemClient);
}
public StorageCheckpointTests()
{
_blobContainerClient = Substitute.For <BlobContainerClient>();
_storageCheckpointOptions = new StorageCheckpointOptions()
{
BlobPrefix = "Normalization",
CheckpointBatchCount = "5"
};
_eventHubClientOptions = new EventHubClientOptions();
_eventHubNamespaceFQDN = "test.servicebus.windows.net";
_eventHubName = "devicedata";
// Blob path corresponds to current event hub name
_blobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix}/checkpoint/";
_blobPath = $"{_blobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{_blobPath}1"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}10"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
_logger = Substitute.For <ITelemetryLogger>();
}
public void CanCreatePageableFromPages()
{
var pageable = Pageable <byte> .FromPages(new[]
{
Page <byte> .FromValues(new byte[] { 1, 2 }, null, null),
Page <byte> .FromValues(new byte[] { 3, 4 }, null, null)
});
Assert.AreEqual(new byte[] { 1, 2, 3, 4 }, pageable.ToArray());
}
public override Pageable <T> GetValues(CancellationToken cancellationToken = default)
{
if (!HasValue)
{
throw new InvalidOperationException("The operation has not completed yet.");
}
return(Pageable <T> .FromPages(new[]
{
_firstPage,
_secondPage
}));
}
public async Task GivenDifferentAppType_WhenResetCheckpointsAsyncCalled_ThenCheckpointsOfOtherAppsAreNotDeleted()
{
_eventHubClientOptions.EventHubNamespaceFQDN = _eventHubNamespaceFQDN;
_eventHubClientOptions.EventHubName = "newdevicedata";
_storageCheckpointOptions.BlobPrefix = "MeasurementToFhir";
var fhirconvBlobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix }/checkpoint/";
var fhirconvBlobPath = $"{fhirconvBlobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}1"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}10"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that we are processing events for a different app type and the source changed, verify that the previous checkpoints corresponding to this app are deleted.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(3).DeleteBlobAsync(null);
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}1");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}10");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}20");
// Given that we are processing events for a different app type, verify that the checkpoints corresponding to the other apps are not deleted.
_blobContainerClient.Received(0).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}1");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}10");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}20");
}
public void CanCreatePageableFromPage_MultiplePage()
{
var pageable = Pageable <byte> .FromPages(new[]
{
Page <byte> .FromValues(new byte[] { 1, 2 }, "X", null),
Page <byte> .FromValues(new byte[] { 3, 4 }, null, null)
});
var pages = pageable.AsPages(continuationToken: null);
var page = pages.First();
Assert.AreEqual(new byte[] { 1, 2 }, page.Values);
Assert.AreEqual("X", page.ContinuationToken);
pages = pageable.AsPages(continuationToken: "X");
page = pages.First();
Assert.AreEqual(new byte[] { 3, 4 }, page.Values);
Assert.Null(page.ContinuationToken);
}