private async Task StartConsumer(ISourceBlock <EventData> producer, IAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer) { // Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed var exceptions = new ConcurrentBag <Exception>(); var cts = new CancellationTokenSource(); var consumer = new ActionBlock <EventData>( async evt => { try { _log.LogMetric(Metrics.DeviceEventProcessingLatency, (DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc).TotalSeconds); var token = _converter.Convert(evt); foreach (var measurement in _contentTemplate.GetMeasurements(token)) { measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc; await collector.AddAsync(measurement).ConfigureAwait(false); _log.LogMetric(Metrics.NormalizedEvent, 1); } } catch (OperationCanceledException) { cts.Cancel(); throw; } #pragma warning disable CA1031 catch (Exception ex) { if (await errorConsumer(ex, evt).ConfigureAwait(false)) { exceptions.Add(ex); } } #pragma warning restore CA1031 }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _maxParallelism, SingleProducerConstrained = true, CancellationToken = cts.Token }); _ = producer.LinkTo(consumer, new DataflowLinkOptions { PropagateCompletion = true }); await consumer.Completion .ContinueWith( task => { if (!exceptions.IsEmpty) { throw new AggregateException(exceptions); } }, cts.Token, AsyncContinueOnSuccess, TaskScheduler.Current) .ConfigureAwait(false); }
private async Task StartConsumer(ISourceBlock <EventData> producer, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer) { // Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed var exceptions = new ConcurrentBag <Exception>(); var cts = new CancellationTokenSource(); var transformingConsumer = new TransformManyBlock <EventData, (string, IMeasurement)>( async evt => { var createdMeasurements = new List <(string, IMeasurement)>(); try { string partitionId = evt.SystemProperties.PartitionKey; var deviceEventProcessingLatency = DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc; _log.LogMetric( IomtMetrics.DeviceEventProcessingLatency(partitionId), deviceEventProcessingLatency.TotalSeconds); _log.LogMetric( IomtMetrics.DeviceEventProcessingLatencyMs(partitionId), deviceEventProcessingLatency.TotalMilliseconds); var token = _converter.Convert(evt); try { foreach (var measurement in _contentTemplate.GetMeasurements(token)) { measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc; createdMeasurements.Add((partitionId, measurement)); } } catch (Exception ex) { // Translate all Normalization Mapping exceptions into a common type for easy identification. throw new NormalizationDataMappingException(ex); } } catch (Exception ex) { if (await errorConsumer(ex, evt).ConfigureAwait(false)) { exceptions.Add(ex); } } return(createdMeasurements); }); var asyncCollectorConsumer = new ActionBlock <(string, IMeasurement)[]>(