/// <summary>
 /// This MetricStatsEngineQueue uses a readwrite lock (allows multiple readers, but only one writer, at a time) to mediate
 /// between metrics that need to merge into one of the engines (readers), and the harvest job (writer), which replaces the
 /// queue and merges the engines in the old queue to create the harvest payload.
 ///
 /// In this method, get the read lock and call a method that will pick an engine and merge metric.
 /// </summary>
 /// <param name="metric"></param>
 /// <returns></returns>
 public bool MergeMetric(IAllMetricStatsCollection metric)
 {
     if (_lock.TryEnterReadLock(50))
     {
         try
         {
             Queue <MetricStatsCollection> statsEngineQueue = this._statsEngineQueue;
             if (statsEngineQueue == null)
             {
                 // We've already been harvested.  Caller should try again, at which point a whole new MetricStatsEngineQueue
                 // will have been created for the next harvest cycle.
                 return(false);
             }
             MergeMetricUnderLock(statsEngineQueue, metric);
             return(true);
         }
         finally
         {
             _lock.ExitReadLock();
         }
     }
     return(false);
 }
            /// <summary>
            /// Take a stats engine (metrics aggregate Dictionary) off the queue and merge metric into it, then put it back
            /// on the queue.  Create one if all the existing ones are "checked out" -- others can use it later.
            ///
            /// We are one of (possibly) several readers of the stats engine queue under the readwrite lock, so harvest will
            /// wait for us to finish before fetching/replacing the entire queue
            /// </summary>
            /// <param name="statsEngineQueue"></param>
            /// <param name="metric"></param>
            private void MergeMetricUnderLock(Queue <MetricStatsCollection> statsEngineQueue, IAllMetricStatsCollection metrics)
            {
                MetricStatsCollection statsEngineToMergeWith = null;

                try
                {
                    lock (_queueReadersLock)
                    {
                        if (statsEngineQueue.Count > 0)
                        {
                            statsEngineToMergeWith = statsEngineQueue.Dequeue();
                        }
                    }

                    // make a new stats engine if there aren't enough to go around right now
                    if (statsEngineToMergeWith == null)
                    {
                        statsEngineToMergeWith = CreateMetricStatsEngine();
                        Interlocked.Increment(ref _statsEngineCount);
                    }

                    metrics.AddMetricsToEngine(statsEngineToMergeWith);
                }
                catch (Exception e)
                {
                    Log.Warn($"Exception dequeueing/creating stats engine: {e}");
                }
                finally
                {
                    if (statsEngineToMergeWith != null)
                    {
                        try
                        {
                            lock (_queueReadersLock)
                            {
                                statsEngineQueue.Enqueue(statsEngineToMergeWith);
                            }
                        }
                        catch (Exception e)
                        {
                            // should never happen
                            Log.Warn($"Exception returning stats engine to queue: {e}");
                        }
                    }
                }
            }