public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
var storageOptions = new StorageCheckpointOptions();
Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>();
var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventProcessorOptions = new EventHubClientOptions();
if (applicationType == _normalizationAppType)
{
Configuration.GetSection("InputEventHub").Bind(eventProcessorOptions);
}
else if (applicationType == _measurementToFhirAppType)
{
Configuration.GetSection("NormalizationEventHub").Bind(eventProcessorOptions);
}
storageOptions.BlobPrefix = $"{applicationType}/{storageOptions.BlobPrefix}";
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, eventProcessorOptions, logger);
return(checkpointClient);
}
public static StorageCheckpointClient GetStorageCheckpointClient(BlobContainerClientFactory factory, BlobContainerClientOptions containerOptions, StorageCheckpointOptions options, ITelemetryLogger logger, string prefix)
{
var checkpointBlobClient = factory.CreateStorageClient(containerOptions);
options.BlobPrefix = prefix;
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, options, logger);
return(checkpointClient);
}
public async Task GivenUnchangedEventHubOptions_WhenResetCheckpointsAsyncCalled_ThenNoCheckpointsAreDeleted()
{
_eventHubClientOptions.EventHubNamespaceFQDN = _eventHubNamespaceFQDN;
_eventHubClientOptions.EventHubName = _eventHubName;
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that the source event hub didn't change, verify that no checkpoint deletions occured.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(0).DeleteBlobAsync(null);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
// determine which event hub to read from
var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME");
if (eventHub == null)
{
eventHub = config.GetSection("Console:EventHub").Value;
}
System.Console.WriteLine($"Reading from event hub: {eventHub}");
System.Console.WriteLine($"Logs and Metrics will be written to Application Insights");
var eventHubOptions = GetEventHubInfo(config, eventHub);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var serviceProvider = GetRequiredServiceProvider(config, eventHub);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
storageOptions.BlobPrefix = eventHub;
var checkpointClient = new StorageCheckpointClient(storageOptions, logger);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var ct = new CancellationToken();
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName);
var eventProcessorClientOptions = new EventProcessorClientOptions();
eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60);
EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
await eventHubReader.RunAsync(client, ct);
}
public async Task GivenUnchangedEventHubOptionsWithConnectionString_WhenResetCheckpointsAsyncCalled_ThenNoCheckpointsAreDeleted()
{
var eventHubClientOptions = new EventHubClientOptions()
{
AuthenticationType = AuthenticationType.ConnectionString,
ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=test123;EntityPath=devicedata"
};
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that the source event hub didn't change, verify that no checkpoint deletions occured.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(0).DeleteBlobAsync(null);
}
public async Task GivenUpdatedEventHubNamespaceAndEventHubName_WhenResetCheckpointsAsyncCalled_ThenPreviousCheckpointsAreDeleted()
{
_eventHubClientOptions.EventHubNamespaceFQDN = "newtest.servicebus.windows.net";
_eventHubClientOptions.EventHubName = "newdevicedata";
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that the event hub namespace and the event hub name changed and is therefore a new source, verify that the checkpoints corresponding to the old source will be deleted.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(3).DeleteBlobAsync(null);
await _blobContainerClient.Received(1).DeleteBlobAsync($"{_blobPath}1");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{_blobPath}10");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{_blobPath}20");
}
public async Task GivenDifferentAppType_WhenResetCheckpointsAsyncCalled_ThenCheckpointsOfOtherAppsAreNotDeleted()
{
_eventHubClientOptions.EventHubNamespaceFQDN = _eventHubNamespaceFQDN;
_eventHubClientOptions.EventHubName = "newdevicedata";
_storageCheckpointOptions.BlobPrefix = "MeasurementToFhir";
var fhirconvBlobCheckpointPrefix = $"{_storageCheckpointOptions.BlobPrefix }/checkpoint/";
var fhirconvBlobPath = $"{fhirconvBlobCheckpointPrefix}{_eventHubNamespaceFQDN}/{_eventHubName}/";
IReadOnlyList <BlobItem> mockBlobItems = new List <BlobItem>()
{
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}1"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}10"),
BlobsModelFactory.BlobItem(name: $"{fhirconvBlobPath}20")
};
var mockPageBlobItems = Page <BlobItem> .FromValues(mockBlobItems, "continuationToken", Substitute.For <Response>());
var mockPageableBlobItems = Pageable <BlobItem> .FromPages(new[] { mockPageBlobItems });
_blobContainerClient.GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None)
.Returns(mockPageableBlobItems);
var storageClient = new StorageCheckpointClient(_blobContainerClient, _storageCheckpointOptions, _eventHubClientOptions, _logger);
await storageClient.ResetCheckpointsAsync();
// Given that we are processing events for a different app type and the source changed, verify that the previous checkpoints corresponding to this app are deleted.
_blobContainerClient.Received(1).GetBlobs(states: BlobStates.All, prefix: fhirconvBlobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.ReceivedWithAnyArgs(3).DeleteBlobAsync(null);
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}1");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}10");
await _blobContainerClient.Received(1).DeleteBlobAsync($"{fhirconvBlobPath}20");
// Given that we are processing events for a different app type, verify that the checkpoints corresponding to the other apps are not deleted.
_blobContainerClient.Received(0).GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None);
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}1");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}10");
await _blobContainerClient.Received(0).DeleteBlobAsync($"{_blobPath}20");
}
public virtual StorageCheckpointClient ResolveCheckpointClient(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
var storageOptions = new StorageCheckpointOptions();
Configuration.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
Configuration.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var factory = serviceProvider.GetRequiredService <BlobContainerClientFactory>();
var checkpointBlobClient = factory.CreateStorageClient(checkpointContainerOptions);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
storageOptions.BlobPrefix = applicationType;
var checkpointClient = new StorageCheckpointClient(checkpointBlobClient, storageOptions, logger);
return(checkpointClient);
}
