public void Aggregate(ref MetricValue metricValue)
{
// If rates are not stored then there is nothing to aggregate
if (!_storeRates)
{
return;
}
IList <DataSource> dsl = DataSetCollection.Instance.GetDataSource(metricValue.TypeName);
if (dsl == null || metricValue.Values.Length != dsl.Count)
{
return;
}
double now = Util.GetNow();
lock (_cacheLock)
{
CacheEntry cEntry;
string key = metricValue.Key();
if (!(_cache.TryGetValue(key, out cEntry)))
{
cEntry = new CacheEntry(metricValue.DeepCopy());
for (int i = 0; i < metricValue.Values.Length; i++)
{
if (dsl[i].Type == DsType.Derive ||
dsl[i].Type == DsType.Absolute ||
dsl[i].Type == DsType.Counter)
{
metricValue.Values[i] = double.NaN;
cEntry.MetricRate.Values[i] = double.NaN;
}
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
return;
}
for (int i = 0; i < metricValue.Values.Length; i++)
{
double rawValNew = metricValue.Values[i];
double rawValOld = cEntry.RawValues[i];
double rawValDiff = rawValNew - rawValOld;
double timeDiff = cEntry.MetricRate.Epoch - now;
double rateNew = rawValDiff / timeDiff;
switch (dsl[i].Type)
{
case DsType.Gauge:
// no rates calculations are done, values are stored as-is for gauge
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rawValNew;
break;
case DsType.Absolute:
// similar to gauge, except value will be divided by time diff
cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
break;
case DsType.Derive:
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rateNew;
metricValue.Values[i] = rateNew;
break;
case DsType.Counter:
// Counters are very simlar to derive except when counter wraps around
if (rawValNew < rawValOld)
{
// counter has wrapped around
cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
}
else
{
cEntry.MetricRate.Values[i] = rateNew;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = rateNew;
}
break;
}
// range checks
if (metricValue.Values[i] < dsl[i].Min)
{
metricValue.Values[i] = dsl[i].Min;
cEntry.RawValues[i] = metricValue.Values[i];
}
if (metricValue.Values[i] > dsl[i].Max)
{
metricValue.Values[i] = dsl[i].Max;
cEntry.RawValues[i] = metricValue.Values[i];
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
}
}
}
public IList<MetricValue> Read()
{
lock (_lock)
{
var res = new List<MetricValue>();
var removeList = new List<string>();
foreach (var pair in _metrics)
{
StatsdMetric metric = pair.Value;
if (metric.NumUpdates <= 0 &&
((_delCounter && metric.Type == StatsdMetric.StatsdType.StatsdCounter) ||
(_delTimer && metric.Type == StatsdMetric.StatsdType.StatsdTimer) ||
(_delGauge && metric.Type == StatsdMetric.StatsdType.StatsdGauge) ||
(_delSet && metric.Type == StatsdMetric.StatsdType.StatsdSet)))
{
removeList.Add(pair.Key);
continue;
}
var metricVal = new MetricValue
{
HostName = _hostName,
PluginName = "statsd",
PluginInstanceName = "",
TypeInstanceName = metric.Name,
Values = new[] {metric.GetMetric()}
};
switch (metric.Type)
{
case StatsdMetric.StatsdType.StatsdGauge:
metricVal.TypeName = "gauge";
break;
case StatsdMetric.StatsdType.StatsdTimer:
metricVal.TypeName = "latency";
metricVal.TypeInstanceName += "-average";
break;
case StatsdMetric.StatsdType.StatsdSet:
metricVal.TypeName = "objects";
break;
default:
metricVal.TypeName = "derive";
break;
}
TimeSpan t = DateTime.UtcNow - new DateTime(1970, 1, 1);
double epoch = t.TotalMilliseconds/1000;
metricVal.Epoch = Math.Round(epoch, 3);
res.Add(metricVal);
if (metric.Type == StatsdMetric.StatsdType.StatsdTimer)
{
if (_timerLower)
{
MetricValue lowerValue = metricVal.DeepCopy();
lowerValue.TypeInstanceName = metric.Name + "-lower";
lowerValue.Values[0] = metric.Lat.Min;
res.Add(lowerValue);
}
if (_timerUpper)
{
MetricValue upperValue = metricVal.DeepCopy();
upperValue.TypeInstanceName = metric.Name + "-upper";
upperValue.Values[0] = metric.Lat.Max;
res.Add(upperValue);
}
if (_timerSum)
{
MetricValue upperSum = metricVal.DeepCopy();
upperSum.TypeInstanceName = metric.Name + "-Sum";
upperSum.Values[0] = metric.Lat.Sum;
res.Add(upperSum);
}
if (_timerCount)
{
MetricValue upperCount = metricVal.DeepCopy();
upperCount.TypeInstanceName = metric.Name + "-count";
upperCount.Values[0] = metric.Lat.Num;
res.Add(upperCount);
}
Histogram histogram = metric.Lat.Histogram;
if (_percentiles != null && _percentiles.Length > 0 && histogram != null)
{
foreach (float percentile in _percentiles)
{
double val = histogram.GetPercentile(percentile);
MetricValue mv = metricVal.DeepCopy();
mv.TypeInstanceName = metric.Name + "-percentile-" + percentile;
mv.Values[0] = val;
res.Add(mv);
}
}
}
metric.Reset();
}
Logger.Debug("Removing entries that were not updated:{0}", removeList.Count);
foreach (string key in removeList)
{
_metrics.Remove(key);
}
return (res);
}
}
public void Aggregate(ref MetricValue metricValue)
{
// If rates are not stored then there is nothing to aggregate
if (!_storeRates)
{
return;
}
IList<DataSource> dsl = DataSetCollection.Instance.GetDataSource(metricValue.TypeName);
if (dsl == null || metricValue.Values.Length != dsl.Count)
{
return;
}
double now = Util.GetNow();
lock (_cacheLock)
{
CacheEntry cEntry;
string key = metricValue.Key();
if (!(_cache.TryGetValue(key, out cEntry)))
{
cEntry = new CacheEntry(metricValue.DeepCopy());
for (int i = 0; i < metricValue.Values.Length; i++)
{
if (dsl[i].Type == DsType.Derive ||
dsl[i].Type == DsType.Absolute ||
dsl[i].Type == DsType.Counter)
{
metricValue.Values[i] = double.NaN;
cEntry.MetricRate.Values[i] = double.NaN;
}
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
return;
}
for (int i = 0; i < metricValue.Values.Length; i++)
{
double rawValNew = metricValue.Values[i];
double rawValOld = cEntry.RawValues[i];
double rawValDiff = rawValNew - rawValOld;
double timeDiff = cEntry.MetricRate.Epoch - now;
double rateNew = rawValDiff/timeDiff;
switch (dsl[i].Type)
{
case DsType.Gauge:
// no rates calculations are done, values are stored as-is for gauge
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rawValNew;
break;
case DsType.Absolute:
// similar to gauge, except value will be divided by time diff
cEntry.MetricRate.Values[i] = metricValue.Values[i]/timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
break;
case DsType.Derive:
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rateNew;
metricValue.Values[i] = rateNew;
break;
case DsType.Counter:
// Counters are very simlar to derive except when counter wraps around
if (rawValNew < rawValOld)
{
// counter has wrapped around
cEntry.MetricRate.Values[i] = metricValue.Values[i]/timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
}
else
{
cEntry.MetricRate.Values[i] = rateNew;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = rateNew;
}
break;
}
// range checks
if (metricValue.Values[i] < dsl[i].Min)
{
metricValue.Values[i] = dsl[i].Min;
cEntry.RawValues[i] = metricValue.Values[i];
}
if (metricValue.Values[i] > dsl[i].Max)
{
metricValue.Values[i] = dsl[i].Max;
cEntry.RawValues[i] = metricValue.Values[i];
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
}
}
}
public IList <MetricValue> Read()
{
lock (_lock)
{
var res = new List <MetricValue>();
var removeList = new List <string>();
foreach (var pair in _metrics)
{
StatsdMetric metric = pair.Value;
if (metric.NumUpdates <= 0 &&
((_delCounter && metric.Type == StatsdMetric.StatsdType.StatsdCounter) ||
(_delTimer && metric.Type == StatsdMetric.StatsdType.StatsdTimer) ||
(_delGauge && metric.Type == StatsdMetric.StatsdType.StatsdGauge) ||
(_delSet && metric.Type == StatsdMetric.StatsdType.StatsdSet)))
{
removeList.Add(pair.Key);
continue;
}
var metricVal = new MetricValue
{
HostName = _hostName,
PluginName = "statsd",
PluginInstanceName = "",
TypeInstanceName = metric.Name,
Values = new[] { metric.GetMetric() }
};
switch (metric.Type)
{
case StatsdMetric.StatsdType.StatsdGauge:
metricVal.TypeName = "gauge";
break;
case StatsdMetric.StatsdType.StatsdTimer:
metricVal.TypeName = "latency";
metricVal.TypeInstanceName += "-average";
break;
case StatsdMetric.StatsdType.StatsdSet:
metricVal.TypeName = "objects";
break;
default:
metricVal.TypeName = "derive";
break;
}
TimeSpan t = DateTime.UtcNow - new DateTime(1970, 1, 1);
double epoch = t.TotalMilliseconds / 1000;
metricVal.Epoch = Math.Round(epoch, 3);
res.Add(metricVal);
if (metric.Type == StatsdMetric.StatsdType.StatsdTimer)
{
if (_timerLower)
{
MetricValue lowerValue = metricVal.DeepCopy();
lowerValue.TypeInstanceName = metric.Name + "-lower";
lowerValue.Values[0] = metric.Lat.Min;
res.Add(lowerValue);
}
if (_timerUpper)
{
MetricValue upperValue = metricVal.DeepCopy();
upperValue.TypeInstanceName = metric.Name + "-upper";
upperValue.Values[0] = metric.Lat.Max;
res.Add(upperValue);
}
if (_timerSum)
{
MetricValue upperSum = metricVal.DeepCopy();
upperSum.TypeInstanceName = metric.Name + "-Sum";
upperSum.Values[0] = metric.Lat.Sum;
res.Add(upperSum);
}
if (_timerCount)
{
MetricValue upperCount = metricVal.DeepCopy();
upperCount.TypeInstanceName = metric.Name + "-count";
upperCount.Values[0] = metric.Lat.Num;
res.Add(upperCount);
}
Histogram histogram = metric.Lat.Histogram;
if (_percentiles != null && _percentiles.Length > 0 && histogram != null)
{
foreach (float percentile in _percentiles)
{
double val = histogram.GetPercentile(percentile);
MetricValue mv = metricVal.DeepCopy();
mv.TypeInstanceName = metric.Name + "-percentile-" + percentile;
mv.Values[0] = val;
res.Add(mv);
}
}
}
metric.Reset();
}
Logger.Debug("Removing entries that were not updated:{0}", removeList.Count);
foreach (string key in removeList)
{
_metrics.Remove(key);
}
return(res);
}
}
