private Task ProcessErrorHandler(ProcessErrorEventArgs args)
        {
            try
            {
                var eventHubsException = (args.Exception as EventHubsException);
                eventHubsException?.TrackMetrics(Metrics);

                if (eventHubsException == null)
                {
                    Interlocked.Increment(ref Metrics.GeneralExceptions);
                }

                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.ProcessingExceptions);
                ErrorsObserved.Add(args.Exception);
            }
            catch (EventHubsException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                ex.TrackMetrics(Metrics);
                ErrorsObserved.Add(ex);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            return(Task.CompletedTask);
        }
Beispiel #2
0
        private Task ProcessEventHandler(string processorId, ProcessEventArgs args)
        {
            try
            {
                Interlocked.Increment(ref Metrics.TotalServiceOperations);
                Interlocked.Increment(ref Metrics.EventHandlerCalls);

                // If there was no event then there is nothing to do.

                if (!args.HasEvent)
                {
                    return(Task.CompletedTask);
                }

                // We're not expecting any events; capture it as unexpected.

                Interlocked.Increment(ref Metrics.EventsRead);
                ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, false), EventHubsException.FailureReason.GeneralError));
            }
            catch (EventHubsException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.ProcessingExceptions);
                ex.TrackMetrics(Metrics);
                ErrorsObserved.Add(ex);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.ProcessingExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            return(Task.CompletedTask);
        }
Beispiel #3
0
        public async Task Start(CancellationToken cancellationToken)
        {
            IsRunning = true;

            using var process = Process.GetCurrentProcess();
            using var publishCancellationSource = new CancellationTokenSource();

            var publishingTasks = default(IEnumerable <Task>);
            var runDuration     = Stopwatch.StartNew();

            try
            {
                // Begin publishing events in the background.

                publishingTasks = Enumerable
                                  .Range(0, Configuration.ProducerCount)
                                  .Select(_ => Task.Run(() => new Publisher(Configuration, Metrics, ErrorsObserved).Start(publishCancellationSource.Token)))
                                  .ToList();

                // Update metrics.

                while (!cancellationToken.IsCancellationRequested)
                {
                    Metrics.UpdateEnvironmentStatistics(process);
                    Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);

                    await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken).ConfigureAwait(false);
                }
            }
            catch (TaskCanceledException)
            {
                // No action needed.
            }
            catch (Exception ex) when
                (ex is OutOfMemoryException ||
                ex is StackOverflowException ||
                ex is ThreadAbortException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            // The run is ending.  Clean up the outstanding background operations and
            // complete the necessary metrics tracking.

            try
            {
                publishCancellationSource.Cancel();
                await Task.WhenAll(publishingTasks).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
            finally
            {
                runDuration.Stop();
                IsRunning = false;
            }
        }
Beispiel #4
0
        public async Task Start(CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                try
                {
                    var options = new EventHubProducerClientOptions
                    {
                        RetryOptions = new EventHubsRetryOptions
                        {
                            TryTimeout = Configuration.SendTimeout
                        }
                    };

                    var producer = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub, options);

                    await using (producer.ConfigureAwait(false))
                    {
                        while (!cancellationToken.IsCancellationRequested)
                        {
                            // Query partitions each iteration to allow any new partitions to be discovered. Dynamic upscaling of partitions
                            // is a near-term feature being added to the Event Hubs service.

                            var partitions = await producer.GetPartitionIdsAsync(cancellationToken).ConfigureAwait(false);

                            var selectedPartition = partitions[RandomNumberGenerator.Value.Next(0, partitions.Length)];
                            var batchEvents       = new List <EventData>();

                            // Create the batch and generate a set of random events, keeping only those that were able to fit into the batch.

                            using var batch = await producer.CreateBatchAsync(new CreateBatchOptions { PartitionId = selectedPartition }).ConfigureAwait(false);

                            var events = EventGenerator.CreateEvents(
                                Configuration.PublishBatchSize,
                                Configuration.LargeMessageRandomFactorPercent,
                                Configuration.PublishingBodyMinBytes,
                                Configuration.PublishingBodyRegularMaxBytes,
                                currentSequence,
                                selectedPartition);

                            foreach (var currentEvent in events)
                            {
                                if (!batch.TryAdd(currentEvent))
                                {
                                    break;
                                }

                                batchEvents.Add(currentEvent);
                            }

                            // Publish the events and report them, capturing any failures specific to the send operation.

                            try
                            {
                                if (batch.Count > 0)
                                {
                                    await producer.SendAsync(batch, cancellationToken).ConfigureAwait(false);

                                    foreach (var batchEvent in batchEvents)
                                    {
                                        batchEvent.Properties.TryGetValue(EventGenerator.IdPropertyName, out var id);
                                        PublishedEvents.AddOrUpdate(id.ToString(), _ => batchEvent, (k, v) => batchEvent);
                                    }

                                    Interlocked.Add(ref Metrics.EventsPublished, batch.Count);
                                    Interlocked.Increment(ref Metrics.TotalServiceOperations);
                                }
                            }
                            catch (TaskCanceledException)
                            {
                            }
                            catch (EventHubsException ex)
                            {
                                Interlocked.Increment(ref Metrics.TotalExceptions);
                                Interlocked.Increment(ref Metrics.SendExceptions);
                                ex.TrackMetrics(Metrics);
                                ErrorsObserved.Add(ex);
                            }
                            catch (Exception ex)
                            {
                                Interlocked.Increment(ref Metrics.TotalExceptions);
                                Interlocked.Increment(ref Metrics.SendExceptions);
                                Interlocked.Increment(ref Metrics.GeneralExceptions);
                                ErrorsObserved.Add(ex);
                            }

                            // Honor any requested delays to throttle publishing so that it doesn't overwhelm slower consumers.

                            if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
                            {
                                await Task.Delay(Configuration.PublishingDelay.Value).ConfigureAwait(false);
                            }
                        }
                    }
                }
                catch (TaskCanceledException)
                {
                    // No action needed.
                }
                catch (Exception ex) when
                    (ex is OutOfMemoryException ||
                    ex is StackOverflowException ||
                    ex is ThreadAbortException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    Interlocked.Increment(ref Metrics.ProducerRestarted);
                    ErrorsObserved.Add(ex);
                }
            }
        }
        public async Task Start(CancellationToken cancellationToken)
        {
            IsRunning = true;

            using var process = Process.GetCurrentProcess();
            using var publishCancellationSource   = new CancellationTokenSource();
            using var processorCancellationSource = new CancellationTokenSource();

            var publishingTask = default(Task);
            var processorTasks = default(IEnumerable <Task>);
            var runDuration    = Stopwatch.StartNew();

            try
            {
                // Determine the number of partitions in the Event Hub.

                int partitionCount;

                await using (var producerClient = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub))
                {
                    partitionCount = (await producerClient.GetEventHubPropertiesAsync()).PartitionIds.Length;
                }

                // Begin publishing events in the background.

                publishingTask = Task.Run(() => new Publisher(Configuration, Metrics, PublishedEvents, ErrorsObserved).Start(publishCancellationSource.Token));

                // Start processing.

                processorTasks = Enumerable
                                 .Range(0, Configuration.ProcessorCount)
                                 .Select(_ => Task.Run(() => new Processor(Configuration, Metrics, partitionCount, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
                                 .ToList();

                // Test for missing events and update metrics.

                var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);

                while (!cancellationToken.IsCancellationRequested)
                {
                    Metrics.UpdateEnvironmentStatistics(process);
                    Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
                    ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, eventDueInterval, Metrics);

                    await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
                }
            }
            catch (TaskCanceledException)
            {
                // No action needed.
            }
            catch (Exception ex) when
                (ex is OutOfMemoryException ||
                ex is StackOverflowException ||
                ex is ThreadAbortException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            // The run is ending.  Clean up the outstanding background operations and
            // complete the necessary metrics tracking.

            try
            {
                var publishingStopped = DateTimeOffset.UtcNow;

                publishCancellationSource.Cancel();
                await publishingTask.ConfigureAwait(false);

                // Wait a bit after publishing has completed before signaling for
                // processing to be canceled, to allow the recently published
                // events to be read.

                await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false);

                processorCancellationSource.Cancel();
                await Task.WhenAll(processorTasks).ConfigureAwait(false);

                // Wait a bit after processing has completed before and then perform
                // the last bit of bookkeeping, scanning for missing and unexpected events.

                await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);

                ScrubRecentlyPublishedEvents(PublishedEvents, UnexpectedEvents, ReadEvents, publishingStopped.Subtract(TimeSpan.FromMinutes(1)));
                ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes), Metrics);

                foreach (var unexpectedEvent in UnexpectedEvents.Values)
                {
                    Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
                    ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(unexpectedEvent, false), EventHubsException.FailureReason.GeneralError));
                }
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
            finally
            {
                runDuration.Stop();
                IsRunning = false;
            }
        }
        private async Task ProcessEventHandler(string processorId, ProcessEventArgs args)
        {
            try
            {
                Interlocked.Increment(ref Metrics.TotalServiceOperations);

                // If there was no event then there is nothing to do.

                if (!args.HasEvent)
                {
                    return;
                }

                // Determine if the event has an identifier and has been tracked as published.

                var hasId          = args.Data.Properties.TryGetValue(EventGenerator.IdPropertyName, out var id);
                var eventId        = (hasId) ? id?.ToString() : null;
                var isTrackedEvent = PublishedEvents.TryRemove(eventId, out var publishedEvent);

                // If the event has an id that has been seen before, track it as a duplicate processing and
                // take no further action.

                if ((hasId) && (ReadEvents.ContainsKey(eventId)))
                {
                    Interlocked.Increment(ref Metrics.DuplicateEventsDiscarded);
                    return;
                }

                // Since this event wasn't a duplicate, consider it read.

                Interlocked.Increment(ref Metrics.EventsRead);

                // If there the event isn't a known and published event, then track it but take no
                // further action.

                if ((!hasId) || (!isTrackedEvent))
                {
                    // If there was an id, then the event wasn't tracked.  This is likely a race condition in tracking;
                    // allow for a small delay and then try to find the event again.

                    if (hasId)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);

                        isTrackedEvent = PublishedEvents.TryRemove(eventId, out publishedEvent);
                    }
                    else
                    {
                        // If there wasn't an id then consider it an unexpected event failure.

                        Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
                        ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
                        return;
                    }

                    // If there was an id, but the event wasn't tracked as published or processed, cache it as an
                    // unexpected event for later consideration.  If it cannot be cached, consider it a failure.

                    if ((!isTrackedEvent) && (!ReadEvents.ContainsKey(eventId)))
                    {
                        if (!UnexpectedEvents.TryAdd(eventId, args.Data))
                        {
                            Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
                            ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(args.Data, isTrackedEvent), EventHubsException.FailureReason.GeneralError));
                        }

                        return;
                    }
                }

                // It has been proven that the current event is expected; track it as having been read.

                ReadEvents.TryAdd(eventId, 0);

                // Validate the event against expectations.

                if (!args.Data.IsEquivalentTo(publishedEvent))
                {
                    if (!args.Data.Body.ToArray().SequenceEqual(publishedEvent.Body.ToArray()))
                    {
                        Interlocked.Increment(ref Metrics.InvalidBodies);
                    }
                    else
                    {
                        Interlocked.Increment(ref Metrics.InvalidProperties);
                    }
                }

                // Validate that the intended partition that was sent as a property matches the
                // partition that the handler was triggered for.

                if ((!publishedEvent.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var publishedPartition)) ||
                    (args.Partition.PartitionId != publishedPartition.ToString()))
                {
                    Interlocked.Increment(ref Metrics.EventsFromWrongPartition);
                }

                // Validate that the sequence number that was sent as a property is greater than the last read
                // sequence number for the partition; if there hasn't been an event for the partition yet, then
                // there is no validation possible.
                //
                // Track the sequence number for future comparisons.

                var currentSequence = default(object);

                if ((LastReadPartitionSequence.TryGetValue(args.Partition.PartitionId, out var lastReadSequence)) &&
                    ((!publishedEvent.Properties.TryGetValue(EventGenerator.SequencePropertyName, out currentSequence)) || (lastReadSequence >= (long)currentSequence)))
                {
                    Interlocked.Increment(ref Metrics.EventsOutOfOrder);
                }

                var trackSequence = (currentSequence == default) ? -1 : (long)currentSequence;
                LastReadPartitionSequence.AddOrUpdate(args.Partition.PartitionId, _ => trackSequence, (part, seq) => Math.Max(seq, trackSequence));

                // Create a checkpoint every 100 events for the partition, just to follow expected patterns.

                if (trackSequence % 100 == 0)
                {
                    await args.UpdateCheckpointAsync(args.CancellationToken).ConfigureAwait(false);
                }

                // Mark the event as processed.

                Interlocked.Increment(ref Metrics.EventsProcessed);
            }
            catch (EventHubsException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.ProcessingExceptions);
                ex.TrackMetrics(Metrics);
                ErrorsObserved.Add(ex);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.ProcessingExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
        }
        public async Task Start(CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                var options = new EventProcessorClientOptions
                {
                    MaximumWaitTime = Configuration.ReadWaitTime,

                    RetryOptions = new EventHubsRetryOptions
                    {
                        TryTimeout = Configuration.ReadTimeout
                    }
                };

                var storageClient = default(BlobContainerClient);
                var processor     = default(EventProcessorClient);

                try
                {
                    storageClient = new BlobContainerClient(Configuration.StorageConnectionString, Configuration.BlobContainer);
                    processor     = new EventProcessorClient(storageClient, EventHubConsumerClient.DefaultConsumerGroupName, Configuration.EventHubsConnectionString, Configuration.EventHub, options);

                    processor.ProcessEventAsync += ProcessEventHandler;
                    processor.ProcessErrorAsync += ProcessErrorHandler;

                    await processor.StartProcessingAsync(cancellationToken).ConfigureAwait(false);

                    await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
                }
                catch (TaskCanceledException)
                {
                    // No action needed.
                }
                catch (Exception ex) when
                    (ex is OutOfMemoryException ||
                    ex is StackOverflowException ||
                    ex is ThreadAbortException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    Interlocked.Increment(ref Metrics.ProcessorRestarted);
                    ErrorsObserved.Add(ex);
                }
                finally
                {
                    // Wait a short bit to allow for time processing the newly published events.

                    await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);

                    // Constrain stopping the processor, just in case it has issues.  It should not be allowed
                    // to hang, it should be abandoned so that processing can restart.

                    using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));

                    try
                    {
                        if (processor != null)
                        {
                            await processor.StopProcessingAsync(cancellationSource.Token).ConfigureAwait(false);
                        }
                    }
                    catch (Exception ex)
                    {
                        Interlocked.Increment(ref Metrics.ProcessorRestarted);
                        ErrorsObserved.Add(ex);
                    }

                    processor.ProcessEventAsync -= ProcessEventHandler;
                    processor.ProcessErrorAsync -= ProcessErrorHandler;
                }
            }
        }
Beispiel #8
0
        public async Task Start(CancellationToken cancellationToken)
        {
            var sendTasks = new List <Task>();

            while (!cancellationToken.IsCancellationRequested)
            {
                using var backgroundCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);

                try
                {
                    sendTasks.Clear();

                    var options = new EventHubProducerClientOptions
                    {
                        RetryOptions = new EventHubsRetryOptions
                        {
                            TryTimeout = Configuration.SendTimeout
                        }
                    };

                    var producer = new EventHubProducerClient(Configuration.EventHubsConnectionString, Configuration.EventHub, options);

                    await using (producer.ConfigureAwait(false))
                    {
                        // Create a set of background tasks to handle all but one of the concurrent sends.  The final
                        // send will be performed directly.

                        if (Configuration.ConcurrentSends > 1)
                        {
                            for (var index = 0; index < Configuration.ConcurrentSends - 1; ++index)
                            {
                                sendTasks.Add(Task.Run(async() =>
                                {
                                    while (!backgroundCancellationSource.Token.IsCancellationRequested)
                                    {
                                        await PerformSend(producer, backgroundCancellationSource.Token).ConfigureAwait(false);

                                        if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
                                        {
                                            await Task.Delay(Configuration.PublishingDelay.Value, backgroundCancellationSource.Token).ConfigureAwait(false);
                                        }
                                    }
                                }));
                            }
                        }

                        // Perform one of the sends in the foreground, which will allow easier detection of a
                        // processor-level issue.

                        while (!cancellationToken.IsCancellationRequested)
                        {
                            try
                            {
                                await PerformSend(producer, cancellationToken).ConfigureAwait(false);

                                if ((Configuration.PublishingDelay.HasValue) && (Configuration.PublishingDelay.Value > TimeSpan.Zero))
                                {
                                    await Task.Delay(Configuration.PublishingDelay.Value, cancellationToken).ConfigureAwait(false);
                                }
                            }
                            catch (TaskCanceledException)
                            {
                                backgroundCancellationSource.Cancel();
                                await Task.WhenAll(sendTasks).ConfigureAwait(false);
                            }
                        }
                    }
                }
                catch (TaskCanceledException)
                {
                    // No action needed.
                }
                catch (Exception ex) when
                    (ex is OutOfMemoryException ||
                    ex is StackOverflowException ||
                    ex is ThreadAbortException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    // If this catch is hit, there's a problem with the producer itself; cancel the
                    // background sending and wait before allowing the producer to restart.

                    backgroundCancellationSource.Cancel();
                    await Task.WhenAll(sendTasks).ConfigureAwait(false);

                    Interlocked.Increment(ref Metrics.ProducerRestarted);
                    ErrorsObserved.Add(ex);
                }
            }
        }
Beispiel #9
0
        private async Task PerformSend(EventHubProducerClient producer,
                                       CancellationToken cancellationToken)
        {
            // Create the batch and generate a set of random events, keeping only those that were able to fit into the batch.
            // Because there is a side-effect of TryAdd in the statement, ensure that ToList is called to materialize the set
            // or the batch will be empty at send.

            using var batch = await producer.CreateBatchAsync().ConfigureAwait(false);

            var batchEvents = new List <EventData>();

            var events = EventGenerator.CreateEvents(
                Configuration.PublishBatchSize,
                Configuration.LargeMessageRandomFactorPercent,
                Configuration.PublishingBodyMinBytes,
                Configuration.PublishingBodyRegularMaxBytes,
                currentSequence);

            foreach (var currentEvent in events)
            {
                if (!batch.TryAdd(currentEvent))
                {
                    break;
                }

                batchEvents.Add(currentEvent);
            }

            // Publish the events and report them, capturing any failures specific to the send operation.

            try
            {
                if (batch.Count > 0)
                {
                    Interlocked.Increment(ref Metrics.PublishAttempts);
                    await producer.SendAsync(batch, cancellationToken).ConfigureAwait(false);

                    Interlocked.Increment(ref Metrics.BatchesPublished);
                    Interlocked.Add(ref Metrics.EventsPublished, batch.Count);
                    Interlocked.Add(ref Metrics.TotalPublshedSizeBytes, batch.SizeInBytes);
                }
            }
            catch (TaskCanceledException)
            {
                Interlocked.Decrement(ref Metrics.PublishAttempts);
            }
            catch (EventHubsException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.SendExceptions);
                ex.TrackMetrics(Metrics);
                ErrorsObserved.Add(ex);
            }
            catch (OperationCanceledException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.SendExceptions);
                Interlocked.Increment(ref Metrics.CanceledSendExceptions);
                ErrorsObserved.Add(ex);
            }
            catch (TimeoutException ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.SendExceptions);
                Interlocked.Increment(ref Metrics.TimeoutExceptions);
                ErrorsObserved.Add(ex);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.SendExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
        }
Beispiel #10
0
        public async Task Start(CancellationToken cancellationToken)
        {
            IsRunning = true;

            using var process = Process.GetCurrentProcess();
            using var publishCancellationSource   = new CancellationTokenSource();
            using var processorCancellationSource = new CancellationTokenSource();

            var publishingTask = default(Task);
            var processorTasks = default(IEnumerable <Task>);
            var runDuration    = Stopwatch.StartNew();

            try
            {
                // Start processing.

                processorTasks = Enumerable
                                 .Range(0, Configuration.ProcessorCount)
                                 .Select(_ => Task.Run(() => new Processor(Configuration, Metrics, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
                                 .ToList();

                // Test for missing events and update metrics.

                var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);

                while (!cancellationToken.IsCancellationRequested)
                {
                    Metrics.UpdateEnvironmentStatistics(process);
                    Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);

                    await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
                }
            }
            catch (TaskCanceledException)
            {
                // No action needed.
            }
            catch (Exception ex) when
                (ex is OutOfMemoryException ||
                ex is StackOverflowException ||
                ex is ThreadAbortException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            // The run is ending.  Clean up the outstanding background operations and
            // complete the necessary metrics tracking.

            try
            {
                publishCancellationSource.Cancel();
                await publishingTask.ConfigureAwait(false);

                // Wait a bit after publishing has completed before signaling for
                // processing to be canceled, to allow any recently published
                // events to be read.

                await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);

                processorCancellationSource.Cancel();
                await Task.WhenAll(processorTasks).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
            finally
            {
                runDuration.Stop();
                IsRunning = false;
            }
        }