public virtual List <IEventConsumer> ResolveEventConsumers(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
string template;
var eventConsumers = new List <IEventConsumer>();
var templateManager = serviceProvider.GetRequiredService <TemplateManager>();
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
if (applicationType == _normalizationAppType)
{
template = Configuration.GetSection("Template:DeviceContent").Value;
var collector = ResolveEventCollector(serviceProvider);
var collectionContentFactory = serviceProvider.GetRequiredService <CollectionTemplateFactory <IContentTemplate, IContentTemplate> >();
var deviceDataNormalization = new Normalize.Processor(template, templateManager, collector, logger, collectionContentFactory);
eventConsumers.Add(deviceDataNormalization);
EventMetrics.SetConnectorOperation(ConnectorOperation.Normalization);
}
else if (applicationType == _measurementToFhirAppType)
{
template = Configuration.GetSection("Template:FhirMapping").Value;
var importService = serviceProvider.GetRequiredService <MeasurementFhirImportService>();
var measurementCollectionToFhir = new MeasurementCollectionToFhir.Processor(template, templateManager, importService, logger);
eventConsumers.Add(measurementCollectionToFhir);
EventMetrics.SetConnectorOperation(ConnectorOperation.FHIRConversion);
}
else
{
throw new Exception($"Unable to determine template from application type {applicationType}");
}
return(eventConsumers);
}
public async Task SetCheckpointAsync(IEventMessage eventArgs)
{
EnsureArg.IsNotNull(eventArgs);
EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);
try
{
var partitionId = eventArgs.PartitionId;
var checkpoint = new Checkpoint();
checkpoint.LastProcessed = eventArgs.EnqueuedTime;
checkpoint.Id = partitionId;
checkpoint.Prefix = _blobPath;
_checkpoints[partitionId] = checkpoint;
var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1);
if (count >= _lastCheckpointMaxCount)
{
await PublishCheckpointAsync(partitionId);
_log.LogMetric(EventMetrics.EventWatermark(partitionId, eventArgs.EnqueuedTime.UtcDateTime), 1);
_lastCheckpointTracker[partitionId] = 0;
}
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
_log.LogError(new Exception($"Unable to set checkpoint. {ex.Message}"));
}
}
protected virtual async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs)
{
var partitionId = initArgs.PartitionId;
Logger.LogTrace($"Initializing partition {partitionId}");
if (initArgs.CancellationToken.IsCancellationRequested)
{
// Log the condition where the initialization handler is called, an the PartitionInitializingEventArgs contains a cancellation request.
Logger.LogTrace($"PartitionInitializingEventArgs contain a cancellation request {partitionId}");
}
try
{
var checkpoint = await CheckpointClient.GetCheckpointForPartitionAsync(partitionId, initArgs.CancellationToken);
initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed);
Logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}");
Logger.LogMetric(EventMetrics.EventHubPartitionInitialized(partitionId), 1);
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
Logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint");
EventHubExceptionProcessor.ProcessException(ex, Logger, errorMetricName: EventHubErrorCode.EventHubPartitionInitFailed.ToString());
}
}
public async Task SetCheckpointAsync(IEventMessage eventArgs, IEnumerable <KeyValuePair <Metric, double> > metrics = null)
{
EnsureArg.IsNotNull(eventArgs);
EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);
try
{
var partitionId = eventArgs.PartitionId;
var checkpoint = new Checkpoint
{
LastProcessed = eventArgs.EnqueuedTime,
Id = partitionId,
Prefix = _blobPath,
};
await UpdateCheckpointAsync(checkpoint);
_logger.LogMetric(EventMetrics.EventWatermark(partitionId), 1);
if (metrics != null)
{
foreach (var metric in metrics)
{
_logger.LogMetric(metric.Key, metric.Value);
}
}
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to set checkpoint. {ex.Message}", ex));
}
}
private void LogDataFreshness(string partitionId, string triggerReason, IEnumerable <IEventMessage> events = null)
{
// To determine the data freshness per partition (i.e. latest event data processed in a partition), use the enqueued time of the last event for the batch.
// If no events were flushed for the partition (eg: trigger reason is ThresholdWaitReached - due to receival of MaxTimeEvent), then use the current timestamp.
var eventTimestampLastProcessed = events?.Any() ?? false?events.Last().EnqueuedTime.UtcDateTime : DateTime.UtcNow;
_logger.LogMetric(EventMetrics.EventTimestampLastProcessedPerPartition(partitionId, triggerReason), double.Parse(eventTimestampLastProcessed.ToString("yyyyMMddHHmmss")));
}
public async Task SetCheckpointAsync(IEventMessage eventArgs, IEnumerable <KeyValuePair <Metric, double> > metrics = null)
{
EnsureArg.IsNotNull(eventArgs);
EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);
try
{
var partitionId = eventArgs.PartitionId;
var checkpoint = new Checkpoint
{
LastProcessed = eventArgs.EnqueuedTime,
Id = partitionId,
Prefix = _blobPath,
};
_checkpoints[partitionId] = checkpoint;
var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1);
if (metrics != null)
{
_postCheckpointMetrics.TryGetValue(partitionId, out var partitionExists);
if (partitionExists == null)
{
_postCheckpointMetrics[partitionId] = new List <KeyValuePair <Metric, double> >();
}
foreach (var metric in metrics)
{
_postCheckpointMetrics[partitionId].Add(metric);
}
}
if (count >= _lastCheckpointMaxCount)
{
await PublishCheckpointAsync(partitionId);
_logger.LogMetric(EventMetrics.EventWatermark(partitionId), 1);
_lastCheckpointTracker[partitionId] = 0;
_postCheckpointMetrics.TryGetValue(partitionId, out var postCheckpointMetrics);
postCheckpointMetrics?.ForEach(m =>
{
_logger.LogMetric(m.Key, m.Value);
});
postCheckpointMetrics?.Clear();
}
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to set checkpoint. {ex.Message}", ex));
}
}
public static void ProcessException(
Exception exception,
ITelemetryLogger logger,
string errorMetricName = null)
{
EnsureArg.IsNotNull(logger, nameof(logger));
var(customException, errorName) = CustomizeException(exception);
logger.LogError(customException);
errorMetricName = customException.Equals(exception) ? errorMetricName ?? $"{ErrorType.EventHubError}{errorName}" : customException.GetType().Name;
_exceptionTelemetryProcessor.LogExceptionMetric(customException, logger, EventMetrics.HandledException(errorMetricName, ConnectorOperation.Setup));
}
public Processor(
[Blob("template/%Template:FhirMapping%", FileAccess.Read)] string templateDefinition,
ITemplateManager templateManager,
[MeasurementFhirImport] MeasurementFhirImportService measurementImportService,
ITelemetryLogger logger)
{
_templateDefinition = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition));
_templateManager = EnsureArg.IsNotNull(templateManager, nameof(templateManager));
_measurementImportService = EnsureArg.IsNotNull(measurementImportService, nameof(measurementImportService));
_logger = EnsureArg.IsNotNull(logger, nameof(logger));
_retryPolicy = CreateRetryPolicy(logger);
EventMetrics.SetConnectorOperation(ConnectorOperation.FHIRConversion);
}
public virtual IEventProcessingMetricMeters ResolveEventProcessingMetricMeters(IServiceProvider serviceProvider)
{
var applicationType = GetConsoleApplicationType();
if (applicationType == _normalizationAppType)
{
Metric processingMetric = EventMetrics.EventsConsumed(EventMetricDefinition.DeviceIngressSizeBytes);
var meter = new Events.Common.EventProcessingMeter(processingMetric);
var meters = new EventProcessingMetricMeters(new List <IEventProcessingMeter>()
{
meter
});
return(meters);
}
return(null);
}
// flush a fixed number of events
public Task <List <IEventMessage> > Flush(int numEvents)
{
var count = 0;
var events = new List <IEventMessage>();
while (count < numEvents)
{
if (_partition.TryDequeue(out var dequeuedEvent))
{
events.Add(dequeuedEvent);
count++;
}
}
_logger.LogTrace($"Flushed {events.Count} events on partition {_partitionId}");
_logger.LogMetric(EventMetrics.EventsFlushed(_partitionId), events.Count);
return(Task.FromResult(events));
}
public Processor(
string templateDefinition,
ITemplateManager templateManager,
IEnumerableAsyncCollector <IMeasurement> collector,
ITelemetryLogger logger,
CollectionTemplateFactory <IContentTemplate, IContentTemplate> collectionTemplateFactory,
NormalizationExceptionTelemetryProcessor exceptionTelemetryProcessor)
{
_templateDefinition = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition));
_templateManager = EnsureArg.IsNotNull(templateManager, nameof(templateManager));
_collector = EnsureArg.IsNotNull(collector, nameof(collector));
_logger = EnsureArg.IsNotNull(logger, nameof(logger));
_retryPolicy = CreateRetryPolicy(logger);
_collectionTemplateFactory = EnsureArg.IsNotNull(collectionTemplateFactory, nameof(collectionTemplateFactory));
_exceptionTelemetryProcessor = EnsureArg.IsNotNull(exceptionTelemetryProcessor, nameof(exceptionTelemetryProcessor));
EventMetrics.SetConnectorOperation(ConnectorOperation.Normalization);
}
/// <summary>
/// Deletes the previously recorded checkpoints if the current checkpoint blob path (corresponding to the input event hub) has changed.
/// </summary>
/// <param name="cancellationToken">A cancellation token</param>
/// <returns><see cref="Task"/></returns>
public async Task ResetCheckpointsAsync(CancellationToken cancellationToken = default)
{
try
{
_logger.LogTrace($"Entering {nameof(ResetCheckpointsAsync)}...");
var hasEventHubChanged = false;
var blobs = _storageClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: cancellationToken);
foreach (BlobItem blob in blobs)
{
if (!blob.Name.Contains(_blobPath, StringComparison.OrdinalIgnoreCase))
{
try
{
await _storageClient.DeleteBlobAsync(blob.Name, cancellationToken : cancellationToken);
_logger.LogTrace($"Blob checkpoint path changed to {_blobPath}. Deleted checkpoint {blob.Name}.");
hasEventHubChanged = true;
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to delete checkpoint {blob.Name} with error {ex.Message}", ex));
}
}
}
if (blobs.Count() == 0 || hasEventHubChanged)
{
_logger.LogMetric(EventMetrics.EventHubChanged(_blobPath.Replace(_blobCheckpointPrefix, string.Empty)), 1);
}
_logger.LogTrace($"Exiting {nameof(ResetCheckpointsAsync)}.");
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to reset checkpoints. {ex.Message}", ex));
}
}
// flush up to a date time
public Task <List <IEventMessage> > Flush(DateTime dateTime)
{
var events = new List <IEventMessage>();
while (_partition.TryPeek(out var eventData))
{
var enqueuedUtc = eventData.EnqueuedTime.UtcDateTime;
if (enqueuedUtc <= dateTime)
{
_partition.TryDequeue(out var dequeuedEvent);
events.Add(dequeuedEvent);
}
else
{
break;
}
}
_logger.LogTrace($"Flushed {events.Count} events up to {dateTime} on partition {_partitionId}");
_logger.LogMetric(EventMetrics.EventsFlushed(_partitionId), events.Count);
return(Task.FromResult(events));
}
public async Task RunAsync(EventProcessorClient processor, CancellationToken ct)
{
EnsureArg.IsNotNull(processor);
// Processes two types of events
// 1) Event hub events
// 2) Maximum wait events. These are generated when we have not received an event hub
// event for a certain time period and this event is used to flush events in the current window.
async Task ProcessEventHandler(ProcessEventArgs eventArgs)
{
IEventMessage evt;
if (eventArgs.HasEvent)
{
evt = EventMessageFactory.CreateEvent(eventArgs);
}
else
{
evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow);
}
await _eventConsumerService.ConsumeEvent(evt);
}
// todo: consider retry
Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs)
{
_logger.LogError(eventArgs.Exception);
return(Task.CompletedTask);
}
async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs)
{
var partitionId = initArgs.PartitionId;
_logger.LogTrace($"Initializing partition {partitionId}");
try
{
var checkpoint = await _checkpointClient.GetCheckpointForPartitionAsync(partitionId);
initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed);
_logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}");
_logger.LogMetric(EventMetrics.EventHubPartitionInitialized(), 1);
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
_logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint");
_logger.LogError(ex);
}
}
processor.ProcessEventAsync += ProcessEventHandler;
processor.ProcessErrorAsync += ProcessErrorHandler;
processor.PartitionInitializingAsync += ProcessInitializingHandler;
try
{
Console.WriteLine($"Starting event hub processor at {DateTime.UtcNow}");
await processor.StartProcessingAsync();
while (!ct.IsCancellationRequested)
{
}
await processor.StopProcessingAsync();
}
finally
{
processor.ProcessEventAsync -= ProcessEventHandler;
processor.ProcessErrorAsync -= ProcessErrorHandler;
processor.PartitionInitializingAsync -= ProcessInitializingHandler;
}
}