public StorageCheckpointTests()
{
_blobContainerClient = Substitute.For <BlobContainerClient>();
_storageCheckpointOptions = new StorageCheckpointOptions()
{
BlobPrefix = "Normalization",
CheckpointBatchCount = "5"
};
_eventHubClientOptions = new EventHubClientOptions();
_eventHubNamespaceFQDN = "test.servicebus.windows.net";
_eventHubName = "devicedata";
// Blob path corresponds to current event hub name
_blobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix}/checkpoint/";
_blobPath = $"{_blobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{_blobPath}1"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}10"),
BlobsModelFactory.BlobItem(name: $"{_blobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
_logger = Substitute.For <ITelemetryLogger>();
}
public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
var storageOptions = new StorageCheckpointOptions();
Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>();
var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventProcessorOptions = new EventHubClientOptions();
if (applicationType == _normalizationAppType)
{
Configuration.GetSection("InputEventHub").Bind(eventProcessorOptions);
}
else if (applicationType == _measurementToFhirAppType)
{
Configuration.GetSection("NormalizationEventHub").Bind(eventProcessorOptions);
}
storageOptions.BlobPrefix = $"{applicationType}/{storageOptions.BlobPrefix}";
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, eventProcessorOptions, logger);
return(checkpointClient);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
var eventHubName = GetEventHubName(config);
var eventHubOptions = GetEventHubInfo(config, eventHubName);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var serviceCollection = GetRequiredServiceCollection(config, eventHubName);
var serviceProvider = serviceCollection.BuildServiceProvider();
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var blobContainerClientFactory = new BlobContainerClientFactory();
var eventConsumers = GetEventConsumers(config, eventHubName, serviceProvider, blobContainerClientFactory, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
config.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var storageCheckpointClient = GetStorageCheckpointClient(blobContainerClientFactory, checkpointContainerOptions, storageOptions, logger, eventHubName);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var incomingEventReader = GetEventProcessorClient(storageCheckpointClient.GetBlobContainerClient(), eventHubOptions);
var eventHubReader = GetEventProcessor(config, eventConsumerService, storageCheckpointClient, logger);
System.Console.WriteLine($"Reading from event hub: {eventHubName}");
var ct = new CancellationToken();
await eventHubReader.RunAsync(incomingEventReader, ct);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
// determine which event hub to read from
var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME");
if (eventHub == null)
{
eventHub = config.GetSection("Console:EventHub").Value;
}
System.Console.WriteLine($"Reading from event hub: {eventHub}");
System.Console.WriteLine($"Logs and Metrics will be written to Application Insights");
var eventHubOptions = GetEventHubInfo(config, eventHub);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var serviceProvider = GetRequiredServiceProvider(config, eventHub);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
storageOptions.BlobPrefix = eventHub;
var checkpointClient = new StorageCheckpointClient(storageOptions, logger);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var ct = new CancellationToken();
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName);
var eventProcessorClientOptions = new EventProcessorClientOptions();
eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60);
EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
await eventHubReader.RunAsync(client, ct);
}
public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
var storageOptions = new StorageCheckpointOptions();
Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>();
var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
storageOptions.BlobPrefix = applicationType;
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, logger);
return(checkpointClient);
}
public static StorageCheckpointClient GetStorageCheckpointClient(BlobContainerClientFactory factory, BlobContainerClientOptions containerOptions, StorageCheckpointOptions options, ITelemetryLogger logger, string prefix)
{
var checkpointBlobClient = factory.CreateStorageClient(containerOptions);
options.BlobPrefix = prefix;
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, options, logger);
return(checkpointClient);
}
