public StorageCheckpointTests() { _blobContainerClient = Substitute.For <BlobContainerClient>(); _storageCheckpointOptions = new StorageCheckpointOptions() { BlobPrefix = "Normalization", CheckpointBatchCount = "5" }; _eventHubClientOptions = new EventHubClientOptions(); _eventHubNamespaceFQDN = "test.servicebus.windows.net"; _eventHubName = "devicedata"; // Blob path corresponds to current event hub name _blobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix}/checkpoint/"; _blobPath = $"{_blobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/"; IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>() { BlobsModelFactory.BlobItem(name: $"{_blobPath}1"), BlobsModelFactory.BlobItem(name: $"{_blobPath}10"), BlobsModelFactory.BlobItem(name: $"{_blobPath}20") }; var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>()); var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems }); _blobContainerClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None) .Returns(mockPageableBlobItems); _logger = Substitute.For <ITelemetryLogger>(); }
public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider) { var applicationType = GetConsoleApplicationType(); var storageOptions = new StorageCheckpointOptions(); Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); var checkpointContainerOptions = new BlobContainerClientOptions(); Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions); var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>(); var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventProcessorOptions = new EventHubClientOptions(); if (applicationType == _normalizationAppType) { Configuration.GetSection("InputEventHub").Bind(eventProcessorOptions); } else if (applicationType == _measurementToFhirAppType) { Configuration.GetSection("NormalizationEventHub").Bind(eventProcessorOptions); } storageOptions.BlobPrefix = $"{applicationType}/{storageOptions.BlobPrefix}"; var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, eventProcessorOptions, logger); return(checkpointClient); }
public static async Task Main() { var config = GetEnvironmentConfig(); var eventHubName = GetEventHubName(config); var eventHubOptions = GetEventHubInfo(config, eventHubName); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName); var serviceCollection = GetRequiredServiceCollection(config, eventHubName); var serviceProvider = serviceCollection.BuildServiceProvider(); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var blobContainerClientFactory = new BlobContainerClientFactory(); var eventConsumers = GetEventConsumers(config, eventHubName, serviceProvider, blobContainerClientFactory, logger); var storageOptions = new StorageCheckpointOptions(); config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); var checkpointContainerOptions = new BlobContainerClientOptions(); config.GetSection("CheckpointStorage").Bind(checkpointContainerOptions); var storageCheckpointClient = GetStorageCheckpointClient(blobContainerClientFactory, checkpointContainerOptions, storageOptions, logger, eventHubName); var eventConsumerService = new EventConsumerService(eventConsumers, logger); var incomingEventReader = GetEventProcessorClient(storageCheckpointClient.GetBlobContainerClient(), eventHubOptions); var eventHubReader = GetEventProcessor(config, eventConsumerService, storageCheckpointClient, logger); System.Console.WriteLine($"Reading from event hub: {eventHubName}"); var ct = new CancellationToken(); await eventHubReader.RunAsync(incomingEventReader, ct); }
public static async Task Main() { var config = GetEnvironmentConfig(); // determine which event hub to read from var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME"); if (eventHub == null) { eventHub = config.GetSection("Console:EventHub").Value; } System.Console.WriteLine($"Reading from event hub: {eventHub}"); System.Console.WriteLine($"Logs and Metrics will be written to Application Insights"); var eventHubOptions = GetEventHubInfo(config, eventHub); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName); var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var serviceProvider = GetRequiredServiceProvider(config, eventHub); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); var storageOptions = new StorageCheckpointOptions(); config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); storageOptions.BlobPrefix = eventHub; var checkpointClient = new StorageCheckpointClient(storageOptions, logger); var eventConsumerService = new EventConsumerService(eventConsumers, logger); var ct = new CancellationToken(); string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName); var eventProcessorClientOptions = new EventProcessorClientOptions(); eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60); EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); await eventHubReader.RunAsync(client, ct); }
public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider) { var applicationType = GetConsoleApplicationType(); var storageOptions = new StorageCheckpointOptions(); Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); var checkpointContainerOptions = new BlobContainerClientOptions(); Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions); var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>(); var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); storageOptions.BlobPrefix = applicationType; var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, logger); return(checkpointClient); }
public static StorageCheckpointClient GetStorageCheckpointClient(BlobContainerClientFactory factory, BlobContainerClientOptions containerOptions, StorageCheckpointOptions options, ITelemetryLogger logger, string prefix) { var checkpointBlobClient = factory.CreateStorageClient(containerOptions); options.BlobPrefix = prefix; var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, options, logger); return(checkpointClient); }