예제 #1
0
        public async Task Start(CancellationToken cancellationToken)
        {
            IsRunning = true;

            using var process = Process.GetCurrentProcess();
            using var publishCancellationSource   = new CancellationTokenSource();
            using var processorCancellationSource = new CancellationTokenSource();

            var publishingTask = default(Task);
            var processorTasks = default(IEnumerable <Task>);
            var runDuration    = Stopwatch.StartNew();

            try
            {
                // Begin publishing events in the background.

                publishingTask = Task.Run(() => new Publisher(Configuration, Metrics, PublishedEvents, ErrorsObserved).Start(publishCancellationSource.Token));

                // Start processing.

                processorTasks = Enumerable
                                 .Range(0, Configuration.ProcessorCount)
                                 .Select(_ => Task.Run(() => new Processor(Configuration, Metrics, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
                                 .ToList();

                // Test for missing events and update metrics.

                var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);

                while (!cancellationToken.IsCancellationRequested)
                {
                    Metrics.UpdateEnvironmentStatistics(process);
                    Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
                    ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, eventDueInterval, Metrics);

                    await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
                }
            }
            catch (TaskCanceledException)
            {
                // No action needed.
            }
            catch (Exception ex) when
                (ex is OutOfMemoryException ||
                ex is StackOverflowException ||
                ex is ThreadAbortException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }

            // The run is ending.  Clean up the outstanding background operations and
            // complete the necessary metrics tracking.

            try
            {
                publishCancellationSource.Cancel();
                await publishingTask.ConfigureAwait(false);

                // Wait a bit after publishing has completed before signaling for
                // processing to be canceled, to allow the recently published
                // events to be read.

                await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false);

                processorCancellationSource.Cancel();
                await Task.WhenAll(processorTasks).ConfigureAwait(false);

                // Wait a bit after processing has completed before and then perform
                // the last bit of bookkeeping, scanning for missing and unexpected events.

                await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);

                ScanForUnreadEvents(PublishedEvents, UnexpectedEvents, ReadEvents, ErrorsObserved, TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes), Metrics);

                foreach (var unexpectedEvent in UnexpectedEvents.Values)
                {
                    Interlocked.Increment(ref Metrics.UnknownEventsProcessed);
                    ErrorsObserved.Add(new EventHubsException(false, Configuration.EventHub, FormatUnexpectedEvent(unexpectedEvent, false), EventHubsException.FailureReason.GeneralError));
                }
            }
            catch (Exception ex)
            {
                Interlocked.Increment(ref Metrics.TotalExceptions);
                Interlocked.Increment(ref Metrics.GeneralExceptions);
                ErrorsObserved.Add(ex);
            }
            finally
            {
                runDuration.Stop();
                IsRunning = false;
            }
        }
예제 #2
0
        private static void ScanForUnreadEvents(ConcurrentDictionary <string, EventData> publishedEvents,
                                                ConcurrentDictionary <string, EventData> unexpectedEvents,
                                                ConcurrentDictionary <string, byte> readEvents,
                                                ConcurrentBag <Exception> errorsObserved,
                                                TimeSpan eventDueInterval,
                                                Metrics metrics)
        {
            // An event is considered missing if it was published longer ago than the due time and
            // still exists in the set of published events.

            object publishDate;

            var now = DateTimeOffset.UtcNow;

            foreach (var publishedEvent in publishedEvents.ToList())
            {
                if ((publishedEvent.Value.Properties.TryGetValue(EventGenerator.PublishTimePropertyName, out publishDate)) &&
                    (((DateTimeOffset)publishDate).Add(eventDueInterval) < now) &&
                    (publishedEvents.TryRemove(publishedEvent.Key, out _)))
                {
                    // Check to see if the event was read and tracked as unexpected.  If so, perform basic validation
                    // and record it.

                    if (unexpectedEvents.TryRemove(publishedEvent.Key, out var readEvent))
                    {
                        Interlocked.Increment(ref metrics.EventsRead);

                        // Validate the event against expectations.

                        if (!readEvent.IsEquivalentTo(publishedEvent.Value))
                        {
                            if (!readEvent.Body.ToArray().SequenceEqual(publishedEvent.Value.Body.ToArray()))
                            {
                                Interlocked.Increment(ref metrics.InvalidBodies);
                            }
                            else
                            {
                                Interlocked.Increment(ref metrics.InvalidProperties);
                            }
                        }

                        if ((!publishedEvent.Value.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var publishedPartition)) ||
                            (!readEvent.Properties.TryGetValue(EventGenerator.PartitionPropertyName, out var readPartition)) ||
                            (readPartition.ToString() != publishedPartition.ToString()))
                        {
                            Interlocked.Increment(ref metrics.EventsFromWrongPartition);
                        }

                        Interlocked.Increment(ref metrics.EventsProcessed);
                        readEvents.TryAdd(publishedEvent.Key, 0);
                    }
                    else if (!readEvents.ContainsKey(publishedEvent.Key))
                    {
                        // The event wasn't read earlier and tracked as unexpected; it has not been seen.  Track it as missing.

                        Interlocked.Increment(ref metrics.EventsNotReceived);
                        errorsObserved.Add(new EventHubsException(false, string.Empty, FormatMissingEvent(publishedEvent.Value, now), EventHubsException.FailureReason.GeneralError));
                    }
                }
            }
        }