public override Task OnActivateAsync() { try { logger = GetLogger(nameof(ClusterMetricsGrain)); Configuration = new MetricsConfiguration(); ClusterSnapshot = new MetricsSnapshot { Source = nameof(ClusterMetricsGrain) }; SiloSnapshots = new Dictionary <string, MetricsSnapshot>(); Subscribers = new ObserverSubscriptionManager <IClusterMetricsGrainObserver>(); return(base.OnActivateAsync()); } catch (Exception ex) { if (logger != null) { logger.TrackException(ex); } throw; } }
public async Task ReportSiloStatistics(MetricsSnapshot snapshot) { try { // add or replace the snapshot for the silo this came from if (!SiloSnapshots.ContainsKey(snapshot.Source)) { SiloSnapshots.Add(snapshot.Source, snapshot); } else { SiloSnapshots[snapshot.Source] = snapshot; } if (SiloSnapshotStream != null) { await SiloSnapshotStream.OnNextAsync(snapshot); } // recalculate cluster metrics snapshot // any time a silo snapshot is added or updated ClusterSnapshot = CalculateClusterMetrics(SiloSnapshots.Values.ToList()); //if (ClusterSnapshotStream != null) // await ClusterSnapshotStream.OnNextAsync(ClusterSnapshot); logger.IncrementMetric("SiloMetricsReported"); } catch (Exception ex) { logger.TrackException(ex); throw; } }
// TODO: add unit tests MetricsSnapshot CalculateClusterMetrics(IList <MetricsSnapshot> siloSnapshots) { try { var result = new MetricsSnapshot { Source = nameof(ClusterMetricsGrain) }; foreach (var siloSnapshot in siloSnapshots) { foreach (var siloCounter in siloSnapshot.Counters) { if (!result.Counters.ContainsKey(siloCounter.Key)) { result.Counters.Add(siloCounter.Key, siloCounter.Value); } else { result.Counters[siloCounter.Key] += siloCounter.Value; } } foreach (var siloMetric in siloSnapshot.Metrics) { if (!result.Metrics.ContainsKey(siloMetric.Key)) { result.Metrics.Add(siloMetric.Key, siloMetric.Value); } else { result.Metrics[siloMetric.Key] += siloMetric.Value; } } foreach (var siloMetric in siloSnapshot.TimeSpanMetrics) { if (!result.TimeSpanMetrics.ContainsKey(siloMetric.Key)) { result.TimeSpanMetrics.Add(siloMetric.Key, siloMetric.Value); } else { result.TimeSpanMetrics[siloMetric.Key] += siloMetric.Value; } } } result.SiloCount = siloSnapshots.Count; return(result); } catch (Exception ex) { logger.TrackException(ex); throw; } }
async Task ClusterMetricsTimerTriggered(object state) { if (ClusterSnapshotStream == null || SiloSnapshots.Count == 0) { return; } ClusterSnapshot = CalculateClusterMetrics(SiloSnapshots.Values.ToList()); await ClusterSnapshotStream.OnNextAsync(ClusterSnapshot); }
async Task SampleMetrics() { try { var snapshot = new MetricsSnapshot { Source = Runtime.SiloIdentity }; foreach (var metric in Metrics) { // current value snapshot.Metrics.Add(metric.Key, metric.Value); // history //MetricHistory.AddOrUpdate(metric.Key, new ConcurrentQueue<double>(), // (key, value) => value); //MetricHistory[metric.Key].Enqueue(metric.Value); logger.Verbose("[Metric] " + metric.Key + " = " + metric.Value); } foreach (var counter in Counters) { // current value snapshot.Counters.Add(counter.Key, counter.Value); // history //CounterHistory.AddOrUpdate(counter.Key, new ConcurrentQueue<long>(), // (key, value) => value); //CounterHistory[counter.Key].Enqueue(counter.Value); logger.Verbose("[Counter] " + counter.Key + " = " + counter.Value); } foreach (var request in Requests) { //RequestHistory[request.Key].Enqueue(request.Value); logger.Verbose("[Request] " + request.Key + " = " + request.Value); } foreach (var tsmetric in TimeSpanMetrics) { snapshot.TimeSpanMetrics.Add(tsmetric.Key, tsmetric.Value); // history //TimeSpanMetricHistory.AddOrUpdate(tsmetric.Key, new ConcurrentQueue<TimeSpan>(), // (key, value) => value); //TimeSpanMetricHistory[tsmetric.Key].Enqueue(tsmetric.Value); logger.Verbose("[Time Span Metric] " + tsmetric.Key + " = " + tsmetric.Value); } //TrimHistories(); // report silo statistics to cluster metrics grain var metricsGrain = Runtime.GrainFactory.GetGrain <IClusterMetricsGrain>(Guid.Empty); await metricsGrain.ReportSiloStatistics(snapshot); } catch (Exception ex) { logger.TrackException(ex); // don't freak out } }