public async Task ProcessAsync(IEnumerable <EventData> data, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer = null) { EnsureArg.IsNotNull(data, nameof(data)); EnsureArg.IsNotNull(collector, nameof(collector)); await StartConsumer(StartProducer(data), collector, errorConsumer ?? ProcessErrorAsync).ConfigureAwait(false); }
public MeasurementEventNormalizationServiceTests() { _template = Substitute.For <IContentTemplate>(); _consumer = Substitute.For <IEnumerableAsyncCollector <IMeasurement> >(); _converter = Substitute.For <IConverter <EventData, JToken> >(); _logger = Substitute.For <ITelemetryLogger>(); _exceptionTelemetryProcessor = Substitute.For <IExceptionTelemetryProcessor>(); }
public MeasurementToEventMessageAsyncCollectorTests() { _eventHubService = Substitute.For <IEventHubMessageService>(); _hashCodeFactory = Substitute.For <IHashCodeFactory>(); _telemetryLogger = Substitute.For <ITelemetryLogger>(); _measurementCollector = new MeasurementToEventMessageAsyncCollector(_eventHubService, _hashCodeFactory, _telemetryLogger); _hashCodeGenerator = Substitute.For <IHashCodeGenerator>(); _hashCodeGenerator.GenerateHashCode(Arg.Any <string>()).Returns("123"); _hashCodeFactory.CreateDeterministicHashCodeGenerator().Returns(_hashCodeGenerator); }
private async Task StartConsumer(ISourceBlock <EventData> producer, IEnumerableAsyncCollector <IMeasurement> collector, Func <Exception, EventData, Task <bool> > errorConsumer) { // Collect non operation canceled exceptions as they occur to ensure the entire data stream is processed var exceptions = new ConcurrentBag <Exception>(); var cts = new CancellationTokenSource(); var transformingConsumer = new TransformManyBlock <EventData, (string, IMeasurement)>( async evt => { var createdMeasurements = new List <(string, IMeasurement)>(); try { string partitionId = evt.SystemProperties.PartitionKey; var deviceEventProcessingLatency = DateTime.UtcNow - evt.SystemProperties.EnqueuedTimeUtc; _log.LogMetric( IomtMetrics.DeviceEventProcessingLatency(partitionId), deviceEventProcessingLatency.TotalSeconds); _log.LogMetric( IomtMetrics.DeviceEventProcessingLatencyMs(partitionId), deviceEventProcessingLatency.TotalMilliseconds); var token = _converter.Convert(evt); try { foreach (var measurement in _contentTemplate.GetMeasurements(token)) { measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc; createdMeasurements.Add((partitionId, measurement)); } } catch (Exception ex) { // Translate all Normalization Mapping exceptions into a common type for easy identification. throw new NormalizationDataMappingException(ex); } } catch (Exception ex) { if (await errorConsumer(ex, evt).ConfigureAwait(false)) { exceptions.Add(ex); } } return(createdMeasurements); }); var asyncCollectorConsumer = new ActionBlock <(string, IMeasurement)[]>(
public Processor( string templateDefinition, ITemplateManager templateManager, IEnumerableAsyncCollector <IMeasurement> collector, ITelemetryLogger logger, CollectionTemplateFactory <IContentTemplate, IContentTemplate> collectionTemplateFactory) { _templateDefinition = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition)); _templateManager = EnsureArg.IsNotNull(templateManager, nameof(templateManager)); _collector = EnsureArg.IsNotNull(collector, nameof(collector)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _retryPolicy = CreateRetryPolicy(logger); _collectionTemplateFactory = EnsureArg.IsNotNull(collectionTemplateFactory, nameof(collectionTemplateFactory)); _exceptionTelemetryProcessor = new NormalizationExceptionTelemetryProcessor(); }