public static Page <BlobHierarchyItem> GetSegmentsInYearFunc(
string continuation,
int?pageSizeHint)
=> new BlobHierarchyItemPage(new List <BlobHierarchyItem>
{
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/01/16/2300/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/02/2300/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/03/0000/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/03/1800/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/03/2000/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/03/2200/meta.json", false, null)),
BlobsModelFactory.BlobHierarchyItem(
null,
BlobsModelFactory.BlobItem("idx/segments/2020/03/05/1700/meta.json", false, null)),
});
public async Task ListCheckpointsAsync_LogsOnInvalidCheckpoints()
{
var testLoggerProvider = new TestLoggerProvider();
Mock <BlobContainerClient> containerClientMock = new Mock <BlobContainerClient>(MockBehavior.Strict);
containerClientMock.Setup(c => c.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(AsyncPageable <BlobItem> .FromPages(new[]
{
Page <BlobItem> .FromValues(new[]
{
BlobsModelFactory.BlobItem("testnamespace/testeventhubname/testconsumergroup/checkpoint/0", false, BlobsModelFactory.BlobItemProperties(false), metadata: new Dictionary <string, string>())
}, null, Mock.Of <Response>())
}));
BlobsCheckpointStore store = new BlobsCheckpointStore(
containerClientMock.Object,
new BasicRetryPolicy(new EventHubsRetryOptions()),
_functionId,
testLoggerProvider.CreateLogger("TestLogger")
);
await store.ListCheckpointsAsync(_namespace, _eventHubName, _consumerGroup, CancellationToken.None);
var warning = testLoggerProvider.GetAllLogMessages().Single(p => p.Level == Extensions.Logging.LogLevel.Warning);
var expectedWarning = "Function 'EventHubsTriggerFunction': An invalid checkpoint was found for partition: '0' of " +
"FullyQualifiedNamespace: 'TestNamespace'; EventHubName: 'TestEventHubName'; ConsumerGroup: 'TestConsumerGroup'. " +
"This checkpoint is not valid and will be ignored.";
Assert.AreEqual(expectedWarning, warning.FormattedMessage);
testLoggerProvider.ClearAllLogMessages();
}
public async Task ListOwnershipLogsStartAndComplete()
{
var blobList = new List <BlobItem> {
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var target = new BlobsCheckpointStore(new MockBlobContainerClient()
{
Blobs = blobList
},
new BasicRetryPolicy(new EventHubsRetryOptions()));
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, new CancellationToken());
mockLog.Verify(m => m.ListOwnershipAsyncStart(FullyQualifiedNamespace, EventHubName, ConsumerGroup));
mockLog.Verify(m => m.ListOwnershipAsyncComplete(FullyQualifiedNamespace, EventHubName, ConsumerGroup, blobList.Count));
}
private string CreateBlobAndUploadToContainer(Mock <BlobContainerClient> containerMock, List <BlobItem> blobItems, string blobContent = "test", DateTimeOffset lastModified = default)
{
string blobName = Path.GetRandomFileName().Replace(".", "");
Mock <BlobBaseClient> item = new Mock <BlobBaseClient>();
if (lastModified == default)
{
lastModified = DateTimeOffset.UtcNow;
}
var blobProperties = BlobsModelFactory.BlobProperties(lastModified: lastModified);
item.Setup(x => x.GetPropertiesAsync(null, It.IsAny <CancellationToken>())).ReturnsAsync(Response.FromValue(blobProperties, null));
item.Setup(x => x.Name).Returns(blobName);
BlobItemProperties blobItemProperties = BlobsModelFactory.BlobItemProperties(true, lastModified: lastModified);
BlobItem blobItem = BlobsModelFactory.BlobItem(
name: blobName,
properties: blobItemProperties
);
blobItems.Add(blobItem);
Mock <BlobClient> blobClientMock = new Mock <BlobClient>();
blobClientMock.Setup(x => x.Name).Returns(blobName);
blobClientMock.Setup(x => x.Download(It.IsAny <CancellationToken>())).Returns(() =>
Response.FromValue(BlobsModelFactory.BlobDownloadInfo(content: new MemoryStream(Encoding.UTF8.GetBytes(blobContent))), null));
blobClientMock.Setup(x => x.GetProperties(It.IsAny <BlobRequestConditions>(), It.IsAny <CancellationToken>()))
.Returns(Response.FromValue(blobProperties, null));
containerMock.Setup(x => x.GetBlobClient(blobName)).Returns(blobClientMock.Object);
return(blobName);
}
public StorageCheckpointTests()
{
_blobContainerClient = Substitute.For <BlobContainerClient>();
_storageCheckpointOptions = new StorageCheckpointOptions()
{
BlobPrefix = "Normalization",
CheckpointBatchCount = "5"
};
_eventHubClientOptions = new EventHubClientOptions();
_eventHubNamespaceFQDN = "test.servicebus.windows.net";
_eventHubName = "devicedata";
// Blob path corresponds to current event hub name
_blobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix}/checkpoint/";
_blobPath = $"{_blobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{_blobPath}1"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}10"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
_logger = Substitute.For <ITelemetryLogger>();
}
public async Task GetCheckpointLogsInvalidCheckpoint()
{
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/0",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var target = new BlobsCheckpointStore(new MockBlobContainerClient()
{
Blobs = blobList
}, DefaultRetryPolicy);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.ListCheckpointsAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, CancellationToken.None);
mockLog.Verify(m => m.InvalidCheckpointFound("0", FullyQualifiedNamespace, EventHubName, ConsumerGroup));
}
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobExists()
{
var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{MatchingEtag}"""), DateTime.UtcNow);
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var mockContainerClient = new MockBlobContainerClient()
{
Blobs = blobList
};
mockContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client =>
{
client.BlobInfo = blobInfo;
client.UploadBlobException = new Exception("Upload should not be called");
});
var target = new BlobCheckpointStoreInternal(mockContainerClient);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.UpdateCheckpointAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, PartitionId, 0, default, CancellationToken.None);
public static async IAsyncEnumerator <BlobItem> GetTestValues()
{
yield return(BlobsModelFactory.BlobItem("file1", false, null));
yield return(BlobsModelFactory.BlobItem("file2", false, null));
await Task.CompletedTask;
}
public async Task GivenDifferentAppType_WhenResetCheckpointsAsyncCalled_ThenCheckpointsOfOtherAppsAreNotDeleted()
{
_eventHubClientOptions.EventHubNamespaceFQDN = _eventHubNamespaceFQDN;
_eventHubClientOptions.EventHubName = "newdevicedata";
_storageCheckpointOptions.BlobPrefix = "MeasurementToFhir";
var fhirconvBlobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix }/checkpoint/";
var fhirconvBlobPath = $"{fhirconvBlobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}1"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}10"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that we are processing events for a different app type and the source changed, verify that the previous checkpoints corresponding to this app are deleted.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(3).DeleteBlobAsync(null);
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}1");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}10");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}20");
// Given that we are processing events for a different app type, verify that the checkpoints corresponding to the other apps are not deleted.
_blobContainerClient.Received(0).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}1");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}10");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}20");
}
public async Task JobFinishedHandleAsyncShouldReturnTrueAndNotLogWhenNoErrors()
{
// Arrange
this.eventJobFinishedToPublish.Data = JObject.FromObject(this.outputJobFinishedData);
outputJobFinishedData.CorrelationData.TryGetValue("outputAssetContainer", out var containerUri);
EventGridEvent publishedEvent = null;
string blobName = "covfefe";
BlobItemProperties blobProps = BlobsModelFactory.BlobItemProperties(true);
var blobArray = new BlobItem[]
{
BlobsModelFactory.BlobItem(blobName, false, blobProps)
};
var blobsList = new List <BlobItem>(blobArray);
// Arrange Mocks
Mock.Get(storageService)
.Setup(x => x.ListBlobsAsync(It.IsAny <Uri>(), It.IsAny <StorageClientProviderContext>()))
.ReturnsAsync(blobsList);
Mock.Get(this.eventGridPublisher)
.Setup(x => x.PublishEventToTopic(It.IsAny <EventGridEvent>()))
.Callback <EventGridEvent>((ege) => publishedEvent = ege)
.ReturnsAsync(true);
// Act
var handleAsyncResult = await this.handler.HandleAsync(this.eventJobFinishedToPublish).ConfigureAwait(false);
// Assert
handleAsyncResult.ShouldBe(true, "handleAsync should always return true");
publishedEvent.Data.ShouldBeOfType <ResponseEncodeSuccessDTO>();
var responseEncodeSuccessData = (ResponseEncodeSuccessDTO)publishedEvent.Data;
for (int i = 0; i < blobArray.Length; i++)
{
responseEncodeSuccessData.Outputs[i].BlobUri.ShouldBe(new BlobUriBuilder(new Uri(containerUri))
{
BlobName = blobArray[i].Name
}.ToString());
}
}
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobDoesNotExist()
{
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
PartitionId = PartitionId,
};
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var mockBlobContainerClient = new MockBlobContainerClient()
{
Blobs = blobList
};
mockBlobContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", _ => { });
var target = new BlobsCheckpointStore(mockBlobContainerClient, DefaultRetryPolicy);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None);
mockLog.Verify(log => log.UpdateCheckpointStart(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup));
mockLog.Verify(log => log.UpdateCheckpointComplete(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup));
}
public BlobsList(IEnumerable <string> names)
{
_items = names
.Select(name => BlobsModelFactory.BlobItem(name))
.ToList();
}
