public async Task ProcessAsync(IEnumerable <EventData> data, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer = null)
{
EnsureArg.IsNotNull(data, nameof(data));
EnsureArg.IsNotNull(collector, nameof(collector));
await StartConsumer(StartProducer(data), collector, errorConsumer ?? ProcessErrorAsync).ConfigureAwait(false);
}
public MeasurementEventNormalizationServiceTests()
{
_template = Substitute.For <IContentTemplate>();
_consumer = Substitute.For <IEnumerableAsyncCollector <IMeasurement> >();
_converter = Substitute.For <IConverter <EventData, JToken> >();
_logger = Substitute.For <ITelemetryLogger>();
_exceptionTelemetryProcessor = Substitute.For <IExceptionTelemetryProcessor>();
}
public MeasurementToEventMessageAsyncCollectorTests()
{
_eventHubService = Substitute.For <IEventHubMessageService>();
_hashCodeFactory = Substitute.For <IHashCodeFactory>();
_telemetryLogger = Substitute.For <ITelemetryLogger>();
_measurementCollector = new MeasurementToEventMessageAsyncCollector(_eventHubService, _hashCodeFactory, _telemetryLogger);
_hashCodeGenerator = Substitute.For <IHashCodeGenerator>();
_hashCodeGenerator.GenerateHashCode(Arg.Any <string>()).Returns("123");
_hashCodeFactory.CreateDeterministicHashCodeGenerator().Returns(_hashCodeGenerator);
}
private async Task StartConsumer(ISourceBlock <EventData> producer, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer)
{
// Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed
var exceptions = new ConcurrentBag <Exception>();
var cts = new CancellationTokenSource();
var transformingConsumer = new TransformManyBlock <EventData, (string, IMeasurement)>(
async evt =>
{
var createdMeasurements = new List <(string, IMeasurement)>();
try
{
string partitionId = evt.SystemProperties.PartitionKey;
var deviceEventProcessingLatency = DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc;
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatency(partitionId),
deviceEventProcessingLatency.TotalSeconds);
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatencyMs(partitionId),
deviceEventProcessingLatency.TotalMilliseconds);
var token = _converter.Convert(evt);
try
{
foreach (var measurement in _contentTemplate.GetMeasurements(token))
{
measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
createdMeasurements.Add((partitionId, measurement));
}
}
catch (Exception ex)
{
// Translate all Normalization Mapping exceptions into a common type for easy identification.
throw new NormalizationDataMappingException(ex);
}
}
catch (Exception ex)
{
if (await errorConsumer(ex, evt).ConfigureAwait(false))
{
exceptions.Add(ex);
}
}
return(createdMeasurements);
});
var asyncCollectorConsumer = new ActionBlock <(string, IMeasurement)[]>(
public Processor(
string templateDefinition,
ITemplateManager templateManager,
IEnumerableAsyncCollector <IMeasurement> collector,
ITelemetryLogger logger,
CollectionTemplateFactory <IContentTemplate, IContentTemplate> collectionTemplateFactory)
{
_templateDefinition = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition));
_templateManager = EnsureArg.IsNotNull(templateManager, nameof(templateManager));
_collector = EnsureArg.IsNotNull(collector, nameof(collector));
_logger = EnsureArg.IsNotNull(logger, nameof(logger));
_retryPolicy = CreateRetryPolicy(logger);
_collectionTemplateFactory = EnsureArg.IsNotNull(collectionTemplateFactory, nameof(collectionTemplateFactory));
_exceptionTelemetryProcessor = new NormalizationExceptionTelemetryProcessor();
}
