public override Task OnActivateAsync()
{
try
{
logger = GetLogger("ClusterMetricsGrain");
Configuration = new MetricsConfiguration();
ClusterSnapshot = new MetricsSnapshot {
Source = nameof(ClusterMetricsGrain)
};
SiloSnapshots = new Dictionary <string, MetricsSnapshot>();
Subscribers = new ObserverSubscriptionManager <IClusterMetricsGrainObserver>();
return(base.OnActivateAsync());
}
catch (Exception ex)
{
if (logger != null)
{
logger.TrackException(ex);
}
throw;
}
}
// TODO: add unit tests
MetricsSnapshot CalculateClusterMetrics(IList <MetricsSnapshot> siloSnapshots)
{
try
{
var result = new MetricsSnapshot {
Source = nameof(ClusterMetricsGrain)
};
foreach (var siloSnapshot in siloSnapshots)
{
foreach (var siloCounter in siloSnapshot.Counters)
{
if (!result.Counters.ContainsKey(siloCounter.Key))
{
result.Counters.Add(siloCounter.Key, siloCounter.Value);
}
else
{
result.Counters[siloCounter.Key] += siloCounter.Value;
}
}
foreach (var siloMetric in siloSnapshot.Metrics)
{
if (!result.Metrics.ContainsKey(siloMetric.Key))
{
result.Metrics.Add(siloMetric.Key, siloMetric.Value);
}
else
{
result.Metrics[siloMetric.Key] += siloMetric.Value;
}
}
foreach (var siloMetric in siloSnapshot.TimeSpanMetrics)
{
if (!result.TimeSpanMetrics.ContainsKey(siloMetric.Key))
{
result.TimeSpanMetrics.Add(siloMetric.Key, siloMetric.Value);
}
else
{
result.TimeSpanMetrics[siloMetric.Key] += siloMetric.Value;
}
}
}
result.SiloCount = siloSnapshots.Count;
return(result);
}
catch (Exception ex)
{
logger.TrackException(ex);
throw;
}
}
public async Task ReportSiloStatistics(MetricsSnapshot snapshot)
{
try
{
// add or replace the snapshot for the silo this came from
if (!SiloSnapshots.ContainsKey(snapshot.Source))
{
SiloSnapshots.Add(snapshot.Source, snapshot);
}
else
{
SiloSnapshots[snapshot.Source] = snapshot;
}
if (SiloSnapshotStream != null)
{
await SiloSnapshotStream.OnNextAsync(snapshot);
}
// recalculate cluster metrics snapshot
// any time a silo snapshot is added or updated
ClusterSnapshot = CalculateClusterMetrics(SiloSnapshots.Values.ToList());
if (ClusterSnapshotStream != null)
{
await ClusterSnapshotStream.OnNextAsync(ClusterSnapshot);
}
logger.IncrementMetric("SiloMetricsReported");
}
catch (Exception ex)
{
logger.TrackException(ex);
throw;
}
}
async Task SampleMetrics()
{
try
{
var snapshot = new MetricsSnapshot {
Source = Runtime.SiloIdentity
};
foreach (var metric in Metrics)
{
// current value
snapshot.Metrics.Add(metric.Key, metric.Value);
// history
MetricHistory.AddOrUpdate(metric.Key, new ConcurrentQueue <double>(),
(key, value) => value);
MetricHistory[metric.Key].Enqueue(metric.Value);
logger.Verbose("[Metric] " + metric.Key + " = " + metric.Value);
}
foreach (var counter in Counters)
{
// current value
snapshot.Counters.Add(counter.Key, counter.Value);
// history
CounterHistory.AddOrUpdate(counter.Key, new ConcurrentQueue <long>(),
(key, value) => value);
CounterHistory[counter.Key].Enqueue(counter.Value);
logger.Verbose("[Counter] " + counter.Key + " = " + counter.Value);
}
foreach (var request in Requests)
{
RequestHistory[request.Key].Enqueue(request.Value);
logger.Verbose("[Request] " + request.Key + " = " + request.Value);
}
foreach (var tsmetric in TimeSpanMetrics)
{
snapshot.TimeSpanMetrics.Add(tsmetric.Key, tsmetric.Value);
// history
TimeSpanMetricHistory.AddOrUpdate(tsmetric.Key, new ConcurrentQueue <TimeSpan>(),
(key, value) => value);
TimeSpanMetricHistory[tsmetric.Key].Enqueue(tsmetric.Value);
logger.Verbose("[Time Span Metric] " + tsmetric.Key + " = " + tsmetric.Value);
}
TrimHistories();
// report silo statistics to cluster metrics grain
var metricsGrain = Runtime.GrainFactory.GetGrain <IClusterMetricsGrain>(Guid.Empty);
await metricsGrain.ReportSiloStatistics(snapshot);
}
catch (Exception ex)
{
logger.TrackException(ex);
// don't freak out
}
}