private async Task StartConsumer(ISourceBlock <EventData> producer, IAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer)
{
// Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed
var exceptions = new ConcurrentBag <Exception>();
var cts = new CancellationTokenSource();
var consumer = new ActionBlock <EventData>(
async evt =>
{
try
{
_log.LogMetric(Metrics.DeviceEventProcessingLatency, (DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc).TotalSeconds);
var token = _converter.Convert(evt);
foreach (var measurement in _contentTemplate.GetMeasurements(token))
{
measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
await collector.AddAsync(measurement).ConfigureAwait(false);
_log.LogMetric(Metrics.NormalizedEvent, 1);
}
}
catch (OperationCanceledException)
{
cts.Cancel();
throw;
}
#pragma warning disable CA1031
catch (Exception ex)
{
if (await errorConsumer(ex, evt).ConfigureAwait(false))
{
exceptions.Add(ex);
}
}
#pragma warning restore CA1031
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _maxParallelism, SingleProducerConstrained = true, CancellationToken = cts.Token
});
_ = producer.LinkTo(consumer, new DataflowLinkOptions {
PropagateCompletion = true
});
await consumer.Completion
.ContinueWith(
task =>
{
if (!exceptions.IsEmpty)
{
throw new AggregateException(exceptions);
}
},
cts.Token,
AsyncContinueOnSuccess,
TaskScheduler.Current)
.ConfigureAwait(false);
}
private async Task StartConsumer(ISourceBlock <EventData> producer, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer)
{
// Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed
var exceptions = new ConcurrentBag <Exception>();
var cts = new CancellationTokenSource();
var transformingConsumer = new TransformManyBlock <EventData, (string, IMeasurement)>(
async evt =>
{
var createdMeasurements = new List <(string, IMeasurement)>();
try
{
string partitionId = evt.SystemProperties.PartitionKey;
var deviceEventProcessingLatency = DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc;
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatency(partitionId),
deviceEventProcessingLatency.TotalSeconds);
_log.LogMetric(
IomtMetrics.DeviceEventProcessingLatencyMs(partitionId),
deviceEventProcessingLatency.TotalMilliseconds);
var token = _converter.Convert(evt);
try
{
foreach (var measurement in _contentTemplate.GetMeasurements(token))
{
measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
createdMeasurements.Add((partitionId, measurement));
}
}
catch (Exception ex)
{
// Translate all Normalization Mapping exceptions into a common type for easy identification.
throw new NormalizationDataMappingException(ex);
}
}
catch (Exception ex)
{
if (await errorConsumer(ex, evt).ConfigureAwait(false))
{
exceptions.Add(ex);
}
}
return(createdMeasurements);
});
var asyncCollectorConsumer = new ActionBlock <(string, IMeasurement)[]>(
