public static Page <BlobHierarchyItem> GetSegmentsInYearFunc( string continuation, int?pageSizeHint) => new BlobHierarchyItemPage(new List <BlobHierarchyItem> { BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/01/16/2300/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/02/2300/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/03/0000/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/03/1800/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/03/2000/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/03/2200/meta.json", false, null)), BlobsModelFactory.BlobHierarchyItem( null, BlobsModelFactory.BlobItem("idx/segments/2020/03/05/1700/meta.json", false, null)), });
public async Task ListCheckpointsAsync_LogsOnInvalidCheckpoints() { var testLoggerProvider = new TestLoggerProvider(); Mock <BlobContainerClient> containerClientMock = new Mock <BlobContainerClient>(MockBehavior.Strict); containerClientMock.Setup(c => c.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(AsyncPageable <BlobItem> .FromPages(new[] { Page <BlobItem> .FromValues(new[] { BlobsModelFactory.BlobItem("testnamespace/testeventhubname/testconsumergroup/checkpoint/0", false, BlobsModelFactory.BlobItemProperties(false), metadata: new Dictionary <string, string>()) }, null, Mock.Of <Response>()) })); BlobsCheckpointStore store = new BlobsCheckpointStore( containerClientMock.Object, new BasicRetryPolicy(new EventHubsRetryOptions()), _functionId, testLoggerProvider.CreateLogger("TestLogger") ); await store.ListCheckpointsAsync(_namespace, _eventHubName, _consumerGroup, CancellationToken.None); var warning = testLoggerProvider.GetAllLogMessages().Single(p => p.Level == Extensions.Logging.LogLevel.Warning); var expectedWarning = "Function 'EventHubsTriggerFunction': An invalid checkpoint was found for partition: '0' of " + "FullyQualifiedNamespace: 'TestNamespace'; EventHubName: 'TestEventHubName'; ConsumerGroup: 'TestConsumerGroup'. " + "This checkpoint is not valid and will be ignored."; Assert.AreEqual(expectedWarning, warning.FormattedMessage); testLoggerProvider.ClearAllLogMessages(); }
public async Task ListOwnershipLogsStartAndComplete() { var blobList = new List <BlobItem> { BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}", false, BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)), "snapshot", new Dictionary <string, string> { { BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() } }) }; var target = new BlobsCheckpointStore(new MockBlobContainerClient() { Blobs = blobList }, new BasicRetryPolicy(new EventHubsRetryOptions())); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; await target.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, new CancellationToken()); mockLog.Verify(m => m.ListOwnershipAsyncStart(FullyQualifiedNamespace, EventHubName, ConsumerGroup)); mockLog.Verify(m => m.ListOwnershipAsyncComplete(FullyQualifiedNamespace, EventHubName, ConsumerGroup, blobList.Count)); }
private string CreateBlobAndUploadToContainer(Mock <BlobContainerClient> containerMock, List <BlobItem> blobItems, string blobContent = "test", DateTimeOffset lastModified = default) { string blobName = Path.GetRandomFileName().Replace(".", ""); Mock <BlobBaseClient> item = new Mock <BlobBaseClient>(); if (lastModified == default) { lastModified = DateTimeOffset.UtcNow; } var blobProperties = BlobsModelFactory.BlobProperties(lastModified: lastModified); item.Setup(x => x.GetPropertiesAsync(null, It.IsAny <CancellationToken>())).ReturnsAsync(Response.FromValue(blobProperties, null)); item.Setup(x => x.Name).Returns(blobName); BlobItemProperties blobItemProperties = BlobsModelFactory.BlobItemProperties(true, lastModified: lastModified); BlobItem blobItem = BlobsModelFactory.BlobItem( name: blobName, properties: blobItemProperties ); blobItems.Add(blobItem); Mock <BlobClient> blobClientMock = new Mock <BlobClient>(); blobClientMock.Setup(x => x.Name).Returns(blobName); blobClientMock.Setup(x => x.Download(It.IsAny <CancellationToken>())).Returns(() => Response.FromValue(BlobsModelFactory.BlobDownloadInfo(content: new MemoryStream(Encoding.UTF8.GetBytes(blobContent))), null)); blobClientMock.Setup(x => x.GetProperties(It.IsAny <BlobRequestConditions>(), It.IsAny <CancellationToken>())) .Returns(Response.FromValue(blobProperties, null)); containerMock.Setup(x => x.GetBlobClient(blobName)).Returns(blobClientMock.Object); return(blobName); }
public StorageCheckpointTests() { _blobContainerClient = Substitute.For <BlobContainerClient>(); _storageCheckpointOptions = new StorageCheckpointOptions() { BlobPrefix = "Normalization", CheckpointBatchCount = "5" }; _eventHubClientOptions = new EventHubClientOptions(); _eventHubNamespaceFQDN = "test.servicebus.windows.net"; _eventHubName = "devicedata"; // Blob path corresponds to current event hub name _blobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix}/checkpoint/"; _blobPath = $"{_blobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/"; IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>() { BlobsModelFactory.BlobItem(name: $"{_blobPath}1"), BlobsModelFactory.BlobItem(name: $"{_blobPath}10"), BlobsModelFactory.BlobItem(name: $"{_blobPath}20") }; var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>()); var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems }); _blobContainerClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None) .Returns(mockPageableBlobItems); _logger = Substitute.For <ITelemetryLogger>(); }
public async Task GetCheckpointLogsInvalidCheckpoint() { var blobList = new List <BlobItem> { BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/0", false, BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)), "snapshot", new Dictionary <string, string> { { BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() } }) }; var target = new BlobsCheckpointStore(new MockBlobContainerClient() { Blobs = blobList }, DefaultRetryPolicy); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; await target.ListCheckpointsAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, CancellationToken.None); mockLog.Verify(m => m.InvalidCheckpointFound("0", FullyQualifiedNamespace, EventHubName, ConsumerGroup)); }
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobExists() { var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{MatchingEtag}"""), DateTime.UtcNow); var blobList = new List <BlobItem> { BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}", false, BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)), "snapshot", new Dictionary <string, string> { { BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() } }) }; var mockContainerClient = new MockBlobContainerClient() { Blobs = blobList }; mockContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client => { client.BlobInfo = blobInfo; client.UploadBlobException = new Exception("Upload should not be called"); }); var target = new BlobCheckpointStoreInternal(mockContainerClient); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; await target.UpdateCheckpointAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, PartitionId, 0, default, CancellationToken.None);
public static async IAsyncEnumerator <BlobItem> GetTestValues() { yield return(BlobsModelFactory.BlobItem("file1", false, null)); yield return(BlobsModelFactory.BlobItem("file2", false, null)); await Task.CompletedTask; }
public async Task GivenDifferentAppType_WhenResetCheckpointsAsyncCalled_ThenCheckpointsOfOtherAppsAreNotDeleted() { _eventHubClientOptions.EventHubNamespaceFQDN = _eventHubNamespaceFQDN; _eventHubClientOptions.EventHubName = "newdevicedata"; _storageCheckpointOptions.BlobPrefix = "MeasurementToFhir"; var fhirconvBlobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix }/checkpoint/"; var fhirconvBlobPath = $"{fhirconvBlobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/"; IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>() { BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}1"), BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}10"), BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}20") }; var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>()); var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems }); _blobContainerClient.GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None) .Returns(mockPageableBlobItems); var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger); await storageClient.ResetCheckpointsAsync(); // Given that we are processing events for a different app type and the source changed, verify that the previous checkpoints corresponding to this app are deleted. _blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None); await _blobContainerClient.ReceivedWithAnyArgs(3).DeleteBlobAsync(null); await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}1"); await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}10"); await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}20"); // Given that we are processing events for a different app type, verify that the checkpoints corresponding to the other apps are not deleted. _blobContainerClient.Received(0).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None); await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}1"); await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}10"); await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}20"); }
public async Task JobFinishedHandleAsyncShouldReturnTrueAndNotLogWhenNoErrors() { // Arrange this.eventJobFinishedToPublish.Data = JObject.FromObject(this.outputJobFinishedData); outputJobFinishedData.CorrelationData.TryGetValue("outputAssetContainer", out var containerUri); EventGridEvent publishedEvent = null; string blobName = "covfefe"; BlobItemProperties blobProps = BlobsModelFactory.BlobItemProperties(true); var blobArray = new BlobItem[] { BlobsModelFactory.BlobItem(blobName, false, blobProps) }; var blobsList = new List <BlobItem>(blobArray); // Arrange Mocks Mock.Get(storageService) .Setup(x => x.ListBlobsAsync(It.IsAny <Uri>(), It.IsAny <StorageClientProviderContext>())) .ReturnsAsync(blobsList); Mock.Get(this.eventGridPublisher) .Setup(x => x.PublishEventToTopic(It.IsAny <EventGridEvent>())) .Callback <EventGridEvent>((ege) => publishedEvent = ege) .ReturnsAsync(true); // Act var handleAsyncResult = await this.handler.HandleAsync(this.eventJobFinishedToPublish).ConfigureAwait(false); // Assert handleAsyncResult.ShouldBe(true, "handleAsync should always return true"); publishedEvent.Data.ShouldBeOfType <ResponseEncodeSuccessDTO>(); var responseEncodeSuccessData = (ResponseEncodeSuccessDTO)publishedEvent.Data; for (int i = 0; i < blobArray.Length; i++) { responseEncodeSuccessData.Outputs[i].BlobUri.ShouldBe(new BlobUriBuilder(new Uri(containerUri)) { BlobName = blobArray[i].Name }.ToString()); } }
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobDoesNotExist() { var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = FullyQualifiedNamespace, EventHubName = EventHubName, ConsumerGroup = ConsumerGroup, PartitionId = PartitionId, }; var blobList = new List <BlobItem> { BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}", false, BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)), "snapshot", new Dictionary <string, string> { { BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() } }) }; var mockBlobContainerClient = new MockBlobContainerClient() { Blobs = blobList }; mockBlobContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", _ => { }); var target = new BlobsCheckpointStore(mockBlobContainerClient, DefaultRetryPolicy); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None); mockLog.Verify(log => log.UpdateCheckpointStart(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup)); mockLog.Verify(log => log.UpdateCheckpointComplete(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup)); }
public BlobsList(IEnumerable <string> names) { _items = names .Select(name => BlobsModelFactory.BlobItem(name)) .ToList(); }