public override Task OnActivateAsync() { try { logger = GetLogger("ClusterMetricsGrain"); Configuration = new MetricsConfiguration(); ClusterSnapshot = new MetricsSnapshot { Source = nameof(ClusterMetricsGrain) }; SiloSnapshots = new Dictionary <string, MetricsSnapshot>(); Subscribers = new ObserverSubscriptionManager <IClusterMetricsGrainObserver>(); return(base.OnActivateAsync()); } catch (Exception ex) { if (logger != null) { logger.TrackException(ex); } throw; } }
// TODO: add unit tests MetricsSnapshot CalculateClusterMetrics(IList <MetricsSnapshot> siloSnapshots) { try { var result = new MetricsSnapshot { Source = nameof(ClusterMetricsGrain) }; foreach (var siloSnapshot in siloSnapshots) { foreach (var siloCounter in siloSnapshot.Counters) { if (!result.Counters.ContainsKey(siloCounter.Key)) { result.Counters.Add(siloCounter.Key, siloCounter.Value); } else { result.Counters[siloCounter.Key] += siloCounter.Value; } } foreach (var siloMetric in siloSnapshot.Metrics) { if (!result.Metrics.ContainsKey(siloMetric.Key)) { result.Metrics.Add(siloMetric.Key, siloMetric.Value); } else { result.Metrics[siloMetric.Key] += siloMetric.Value; } } foreach (var siloMetric in siloSnapshot.TimeSpanMetrics) { if (!result.TimeSpanMetrics.ContainsKey(siloMetric.Key)) { result.TimeSpanMetrics.Add(siloMetric.Key, siloMetric.Value); } else { result.TimeSpanMetrics[siloMetric.Key] += siloMetric.Value; } } } result.SiloCount = siloSnapshots.Count; return(result); } catch (Exception ex) { logger.TrackException(ex); throw; } }
public async Task ReportSiloStatistics(MetricsSnapshot snapshot) { try { // add or replace the snapshot for the silo this came from if (!SiloSnapshots.ContainsKey(snapshot.Source)) { SiloSnapshots.Add(snapshot.Source, snapshot); } else { SiloSnapshots[snapshot.Source] = snapshot; } if (SiloSnapshotStream != null) { await SiloSnapshotStream.OnNextAsync(snapshot); } // recalculate cluster metrics snapshot // any time a silo snapshot is added or updated ClusterSnapshot = CalculateClusterMetrics(SiloSnapshots.Values.ToList()); if (ClusterSnapshotStream != null) { await ClusterSnapshotStream.OnNextAsync(ClusterSnapshot); } logger.IncrementMetric("SiloMetricsReported"); } catch (Exception ex) { logger.TrackException(ex); throw; } }
async Task SampleMetrics() { try { var snapshot = new MetricsSnapshot { Source = Runtime.SiloIdentity }; foreach (var metric in Metrics) { // current value snapshot.Metrics.Add(metric.Key, metric.Value); // history MetricHistory.AddOrUpdate(metric.Key, new ConcurrentQueue <double>(), (key, value) => value); MetricHistory[metric.Key].Enqueue(metric.Value); logger.Verbose("[Metric] " + metric.Key + " = " + metric.Value); } foreach (var counter in Counters) { // current value snapshot.Counters.Add(counter.Key, counter.Value); // history CounterHistory.AddOrUpdate(counter.Key, new ConcurrentQueue <long>(), (key, value) => value); CounterHistory[counter.Key].Enqueue(counter.Value); logger.Verbose("[Counter] " + counter.Key + " = " + counter.Value); } foreach (var request in Requests) { RequestHistory[request.Key].Enqueue(request.Value); logger.Verbose("[Request] " + request.Key + " = " + request.Value); } foreach (var tsmetric in TimeSpanMetrics) { snapshot.TimeSpanMetrics.Add(tsmetric.Key, tsmetric.Value); // history TimeSpanMetricHistory.AddOrUpdate(tsmetric.Key, new ConcurrentQueue <TimeSpan>(), (key, value) => value); TimeSpanMetricHistory[tsmetric.Key].Enqueue(tsmetric.Value); logger.Verbose("[Time Span Metric] " + tsmetric.Key + " = " + tsmetric.Value); } TrimHistories(); // report silo statistics to cluster metrics grain var metricsGrain = Runtime.GrainFactory.GetGrain <IClusterMetricsGrain>(Guid.Empty); await metricsGrain.ReportSiloStatistics(snapshot); } catch (Exception ex) { logger.TrackException(ex); // don't freak out } }