コード例 #1
0
        public virtual List <IEventConsumer> ResolveEventConsumers(IServiceProvider serviceProvider)
        {
            var applicationType = GetConsoleApplicationType();

            string template;
            var    eventConsumers  = new List <IEventConsumer>();
            var    templateManager = serviceProvider.GetRequiredService <TemplateManager>();
            var    logger          = serviceProvider.GetRequiredService <ITelemetryLogger>();

            if (applicationType == _normalizationAppType)
            {
                template = Configuration.GetSection("Template:DeviceContent").Value;
                var collector = ResolveEventCollector(serviceProvider);
                var collectionContentFactory = serviceProvider.GetRequiredService <CollectionTemplateFactory <IContentTemplate, IContentTemplate> >();
                var deviceDataNormalization  = new Normalize.Processor(template, templateManager, collector, logger, collectionContentFactory);
                eventConsumers.Add(deviceDataNormalization);
                EventMetrics.SetConnectorOperation(ConnectorOperation.Normalization);
            }
            else if (applicationType == _measurementToFhirAppType)
            {
                template = Configuration.GetSection("Template:FhirMapping").Value;
                var importService = serviceProvider.GetRequiredService <MeasurementFhirImportService>();
                var measurementCollectionToFhir = new MeasurementCollectionToFhir.Processor(template, templateManager, importService, logger);
                eventConsumers.Add(measurementCollectionToFhir);
                EventMetrics.SetConnectorOperation(ConnectorOperation.FHIRConversion);
            }
            else
            {
                throw new Exception($"Unable to determine template from application type {applicationType}");
            }

            return(eventConsumers);
        }
コード例 #2
0
        public async Task SetCheckpointAsync(IEventMessage eventArgs)
        {
            EnsureArg.IsNotNull(eventArgs);
            EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);

            try
            {
                var partitionId = eventArgs.PartitionId;
                var checkpoint  = new Checkpoint();
                checkpoint.LastProcessed = eventArgs.EnqueuedTime;
                checkpoint.Id            = partitionId;
                checkpoint.Prefix        = _blobPath;

                _checkpoints[partitionId] = checkpoint;
                var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1);

                if (count >= _lastCheckpointMaxCount)
                {
                    await PublishCheckpointAsync(partitionId);

                    _log.LogMetric(EventMetrics.EventWatermark(partitionId, eventArgs.EnqueuedTime.UtcDateTime), 1);
                    _lastCheckpointTracker[partitionId] = 0;
                }
            }
#pragma warning disable CA1031
            catch (Exception ex)
#pragma warning restore CA1031
            {
                _log.LogError(new Exception($"Unable to set checkpoint. {ex.Message}"));
            }
        }
コード例 #3
0
        protected virtual async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs)
        {
            var partitionId = initArgs.PartitionId;

            Logger.LogTrace($"Initializing partition {partitionId}");

            if (initArgs.CancellationToken.IsCancellationRequested)
            {
                // Log the condition where the initialization handler is called, an the PartitionInitializingEventArgs contains a cancellation request.
                Logger.LogTrace($"PartitionInitializingEventArgs contain a cancellation request {partitionId}");
            }

            try
            {
                var checkpoint = await CheckpointClient.GetCheckpointForPartitionAsync(partitionId, initArgs.CancellationToken);

                initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed);
                Logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}");
                Logger.LogMetric(EventMetrics.EventHubPartitionInitialized(partitionId), 1);
            }
#pragma warning disable CA1031
            catch (Exception ex)
#pragma warning restore CA1031
            {
                Logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint");

                EventHubExceptionProcessor.ProcessException(ex, Logger, errorMetricName: EventHubErrorCode.EventHubPartitionInitFailed.ToString());
            }
        }
コード例 #4
0
        public async Task SetCheckpointAsync(IEventMessage eventArgs, IEnumerable <KeyValuePair <Metric, double> > metrics = null)
        {
            EnsureArg.IsNotNull(eventArgs);
            EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);

            try
            {
                var partitionId = eventArgs.PartitionId;
                var checkpoint  = new Checkpoint
                {
                    LastProcessed = eventArgs.EnqueuedTime,
                    Id            = partitionId,
                    Prefix        = _blobPath,
                };

                await UpdateCheckpointAsync(checkpoint);

                _logger.LogMetric(EventMetrics.EventWatermark(partitionId), 1);

                if (metrics != null)
                {
                    foreach (var metric in metrics)
                    {
                        _logger.LogMetric(metric.Key, metric.Value);
                    }
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(new StorageCheckpointClientException($"Unable to set checkpoint. {ex.Message}", ex));
            }
        }
コード例 #5
0
        private void LogDataFreshness(string partitionId, string triggerReason, IEnumerable <IEventMessage> events = null)
        {
            // To determine the data freshness per partition (i.e. latest event data processed in a partition), use the enqueued time of the last event for the batch.
            // If no events were flushed for the partition (eg: trigger reason is ThresholdWaitReached - due to receival of MaxTimeEvent), then use the current timestamp.
            var eventTimestampLastProcessed = events?.Any() ?? false?events.Last().EnqueuedTime.UtcDateTime : DateTime.UtcNow;

            _logger.LogMetric(EventMetrics.EventTimestampLastProcessedPerPartition(partitionId, triggerReason), double.Parse(eventTimestampLastProcessed.ToString("yyyyMMddHHmmss")));
        }
コード例 #6
0
        public async Task SetCheckpointAsync(IEventMessage eventArgs, IEnumerable <KeyValuePair <Metric, double> > metrics = null)
        {
            EnsureArg.IsNotNull(eventArgs);
            EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId);

            try
            {
                var partitionId = eventArgs.PartitionId;
                var checkpoint  = new Checkpoint
                {
                    LastProcessed = eventArgs.EnqueuedTime,
                    Id            = partitionId,
                    Prefix        = _blobPath,
                };

                _checkpoints[partitionId] = checkpoint;
                var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1);

                if (metrics != null)
                {
                    _postCheckpointMetrics.TryGetValue(partitionId, out var partitionExists);

                    if (partitionExists == null)
                    {
                        _postCheckpointMetrics[partitionId] = new List <KeyValuePair <Metric, double> >();
                    }

                    foreach (var metric in metrics)
                    {
                        _postCheckpointMetrics[partitionId].Add(metric);
                    }
                }

                if (count >= _lastCheckpointMaxCount)
                {
                    await PublishCheckpointAsync(partitionId);

                    _logger.LogMetric(EventMetrics.EventWatermark(partitionId), 1);
                    _lastCheckpointTracker[partitionId] = 0;

                    _postCheckpointMetrics.TryGetValue(partitionId, out var postCheckpointMetrics);
                    postCheckpointMetrics?.ForEach(m =>
                    {
                        _logger.LogMetric(m.Key, m.Value);
                    });

                    postCheckpointMetrics?.Clear();
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(new StorageCheckpointClientException($"Unable to set checkpoint. {ex.Message}", ex));
            }
        }
コード例 #7
0
        public static void ProcessException(
            Exception exception,
            ITelemetryLogger logger,
            string errorMetricName = null)
        {
            EnsureArg.IsNotNull(logger, nameof(logger));

            var(customException, errorName) = CustomizeException(exception);

            logger.LogError(customException);

            errorMetricName = customException.Equals(exception) ? errorMetricName ?? $"{ErrorType.EventHubError}{errorName}" : customException.GetType().Name;
            _exceptionTelemetryProcessor.LogExceptionMetric(customException, logger, EventMetrics.HandledException(errorMetricName, ConnectorOperation.Setup));
        }
コード例 #8
0
        public Processor(
            [Blob("template/%Template:FhirMapping%", FileAccess.Read)] string templateDefinition,
            ITemplateManager templateManager,
            [MeasurementFhirImport] MeasurementFhirImportService measurementImportService,
            ITelemetryLogger logger)
        {
            _templateDefinition       = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition));
            _templateManager          = EnsureArg.IsNotNull(templateManager, nameof(templateManager));
            _measurementImportService = EnsureArg.IsNotNull(measurementImportService, nameof(measurementImportService));
            _logger      = EnsureArg.IsNotNull(logger, nameof(logger));
            _retryPolicy = CreateRetryPolicy(logger);

            EventMetrics.SetConnectorOperation(ConnectorOperation.FHIRConversion);
        }
コード例 #9
0
        public virtual IEventProcessingMetricMeters ResolveEventProcessingMetricMeters(IServiceProvider serviceProvider)
        {
            var applicationType = GetConsoleApplicationType();

            if (applicationType == _normalizationAppType)
            {
                Metric processingMetric = EventMetrics.EventsConsumed(EventMetricDefinition.DeviceIngressSizeBytes);
                var    meter            = new Events.Common.EventProcessingMeter(processingMetric);
                var    meters           = new EventProcessingMetricMeters(new List <IEventProcessingMeter>()
                {
                    meter
                });
                return(meters);
            }

            return(null);
        }
コード例 #10
0
        // flush a fixed number of events
        public Task <List <IEventMessage> > Flush(int numEvents)
        {
            var count  = 0;
            var events = new List <IEventMessage>();

            while (count < numEvents)
            {
                if (_partition.TryDequeue(out var dequeuedEvent))
                {
                    events.Add(dequeuedEvent);
                    count++;
                }
            }

            _logger.LogTrace($"Flushed {events.Count} events on partition {_partitionId}");
            _logger.LogMetric(EventMetrics.EventsFlushed(_partitionId), events.Count);
            return(Task.FromResult(events));
        }
コード例 #11
0
        public Processor(
            string templateDefinition,
            ITemplateManager templateManager,
            IEnumerableAsyncCollector <IMeasurement> collector,
            ITelemetryLogger logger,
            CollectionTemplateFactory <IContentTemplate, IContentTemplate> collectionTemplateFactory,
            NormalizationExceptionTelemetryProcessor exceptionTelemetryProcessor)
        {
            _templateDefinition          = EnsureArg.IsNotNullOrWhiteSpace(templateDefinition, nameof(templateDefinition));
            _templateManager             = EnsureArg.IsNotNull(templateManager, nameof(templateManager));
            _collector                   = EnsureArg.IsNotNull(collector, nameof(collector));
            _logger                      = EnsureArg.IsNotNull(logger, nameof(logger));
            _retryPolicy                 = CreateRetryPolicy(logger);
            _collectionTemplateFactory   = EnsureArg.IsNotNull(collectionTemplateFactory, nameof(collectionTemplateFactory));
            _exceptionTelemetryProcessor = EnsureArg.IsNotNull(exceptionTelemetryProcessor, nameof(exceptionTelemetryProcessor));

            EventMetrics.SetConnectorOperation(ConnectorOperation.Normalization);
        }
コード例 #12
0
        /// <summary>
        /// Deletes the previously recorded checkpoints if the current checkpoint blob path (corresponding to the input event hub) has changed.
        /// </summary>
        /// <param name="cancellationToken">A cancellation token</param>
        /// <returns><see cref="Task"/></returns>
        public async Task ResetCheckpointsAsync(CancellationToken cancellationToken = default)
        {
            try
            {
                _logger.LogTrace($"Entering {nameof(ResetCheckpointsAsync)}...");

                var hasEventHubChanged = false;

                var blobs = _storageClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: cancellationToken);

                foreach (BlobItem blob in blobs)
                {
                    if (!blob.Name.Contains(_blobPath, StringComparison.OrdinalIgnoreCase))
                    {
                        try
                        {
                            await _storageClient.DeleteBlobAsync(blob.Name, cancellationToken : cancellationToken);

                            _logger.LogTrace($"Blob checkpoint path changed to {_blobPath}. Deleted checkpoint {blob.Name}.");
                            hasEventHubChanged = true;
                        }
                        catch (Exception ex)
                        {
                            _logger.LogError(new StorageCheckpointClientException($"Unable to delete checkpoint {blob.Name} with error {ex.Message}", ex));
                        }
                    }
                }

                if (blobs.Count() == 0 || hasEventHubChanged)
                {
                    _logger.LogMetric(EventMetrics.EventHubChanged(_blobPath.Replace(_blobCheckpointPrefix, string.Empty)), 1);
                }

                _logger.LogTrace($"Exiting {nameof(ResetCheckpointsAsync)}.");
            }
            catch (Exception ex)
            {
                _logger.LogError(new StorageCheckpointClientException($"Unable to reset checkpoints. {ex.Message}", ex));
            }
        }
コード例 #13
0
        // flush up to a date time
        public Task <List <IEventMessage> > Flush(DateTime dateTime)
        {
            var events = new List <IEventMessage>();

            while (_partition.TryPeek(out var eventData))
            {
                var enqueuedUtc = eventData.EnqueuedTime.UtcDateTime;
                if (enqueuedUtc <= dateTime)
                {
                    _partition.TryDequeue(out var dequeuedEvent);
                    events.Add(dequeuedEvent);
                }
                else
                {
                    break;
                }
            }

            _logger.LogTrace($"Flushed {events.Count} events up to {dateTime} on partition {_partitionId}");
            _logger.LogMetric(EventMetrics.EventsFlushed(_partitionId), events.Count);
            return(Task.FromResult(events));
        }
コード例 #14
0
        public async Task RunAsync(EventProcessorClient processor, CancellationToken ct)
        {
            EnsureArg.IsNotNull(processor);

            // Processes two types of events
            // 1) Event hub events
            // 2) Maximum wait events. These are generated when we have not received an event hub
            //    event for a certain time period and this event is used to flush events in the current window.
            async Task ProcessEventHandler(ProcessEventArgs eventArgs)
            {
                IEventMessage evt;

                if (eventArgs.HasEvent)
                {
                    evt = EventMessageFactory.CreateEvent(eventArgs);
                }
                else
                {
                    evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow);
                }

                await _eventConsumerService.ConsumeEvent(evt);
            }

            // todo: consider retry
            Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs)
            {
                _logger.LogError(eventArgs.Exception);
                return(Task.CompletedTask);
            }

            async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs)
            {
                var partitionId = initArgs.PartitionId;

                _logger.LogTrace($"Initializing partition {partitionId}");

                try
                {
                    var checkpoint = await _checkpointClient.GetCheckpointForPartitionAsync(partitionId);

                    initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed);
                    _logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}");
                    _logger.LogMetric(EventMetrics.EventHubPartitionInitialized(), 1);
                }
#pragma warning disable CA1031
                catch (Exception ex)
#pragma warning restore CA1031
                {
                    _logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint");
                    _logger.LogError(ex);
                }
            }

            processor.ProcessEventAsync          += ProcessEventHandler;
            processor.ProcessErrorAsync          += ProcessErrorHandler;
            processor.PartitionInitializingAsync += ProcessInitializingHandler;

            try
            {
                Console.WriteLine($"Starting event hub processor at {DateTime.UtcNow}");
                await processor.StartProcessingAsync();

                while (!ct.IsCancellationRequested)
                {
                }

                await processor.StopProcessingAsync();
            }
            finally
            {
                processor.ProcessEventAsync          -= ProcessEventHandler;
                processor.ProcessErrorAsync          -= ProcessErrorHandler;
                processor.PartitionInitializingAsync -= ProcessInitializingHandler;
            }
        }