public void StartConnection()
{
double now = Util.GetNow();
if (now < (_lastConnectTime + ConnectionRetryDelay))
{
return;
}
lock (_connectionLock)
{
try
{
var cf = new ConnectionFactory {
Uri = _url
};
_connection = cf.CreateConnection();
_channel = _connection.CreateModel();
_connected = true;
_lastConnectTime = Util.GetNow();
Logger.Debug("Connection started.");
}
catch (Exception exp)
{
Logger.Error("Got exception when connecting to AMQP broker : ", exp);
}
}
}
public void RemoveExpiredEntries()
{
// If rates are not stored then there is nothing to remove
if (!_storeRates)
{
return;
}
double now = Util.GetNow();
double expirationTime = now - _timeoutSeconds;
var removeList = new List <string>();
lock (_cacheLock)
{
removeList.AddRange(from pair in _cache
let cEntry = pair.Value
where cEntry.MetricRate.Epoch < expirationTime
select pair.Key);
if (removeList.Count > 0)
{
Logger.Debug("Removing expired entries: {0}", removeList.Count);
}
foreach (string key in removeList)
{
_cache.Remove(key);
}
}
}
public void StartConnection()
{
double now = Util.GetNow();
if (now < (_lastConnectTime + ConnectionRetryDelay))
{
return;
}
lock (_connectionLock)
{
try
{
var cf = new ConnectionFactory {
Uri = _url
};
_connection = cf.CreateConnection();
_channel = _connection.CreateModel();
_connected = true;
_lastConnectTime = Util.GetNow();
Logger.Debug("Connection started.");
}
catch (Exception exp)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when connecting to AMQP broker");
logEvent.Exception = exp;
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION);
Logger.Log(logEvent);
}
}
}
public IList <CollectableValue> Read()
{
// Check if it is time to reload the metric config
// We need to periodially reload the config in case new instances or new categories are available (e.g. due to cluster move)
if (Util.GetNow() - _lastUpdated >= _reloadInterval)
{
Configure();
}
var metricValueList = new List <CollectableValue>();
foreach (Metric metric in _metrics)
{
try
{
var vals = new List <double>();
foreach (PerformanceCounter ctr in metric.Counters)
{
double val = ctr.NextValue();
val = val * metric.Multiplier;
if (metric.DecimalPlaces >= 0)
{
val = Math.Round(val, metric.DecimalPlaces);
}
vals.Add(val);
}
var metricValue = new MetricValue
{
HostName = _hostName,
PluginName = metric.CollectdPlugin,
PluginInstanceName = metric.CollectdPluginInstance,
TypeName = metric.CollectdType,
TypeInstanceName = metric.CollectdTypeInstance,
Values = vals.ToArray(),
FriendlyNames = metric.FriendlyNames
};
metricValue.Epoch = Util.toEpoch(DateTime.UtcNow);
metricValueList.Add(metricValue);
}
catch (Exception ex)
{
Logger.Warn(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex);
}
}
return(metricValueList);
}
public void Aggregate(ref MetricValue metricValue)
{
// If rates are not stored then there is nothing to aggregate
if (!_storeRates)
{
return;
}
IList <DataSource> dsl = DataSetCollection.Instance.GetDataSource(metricValue.TypeName);
if (dsl == null || metricValue.Values.Length != dsl.Count)
{
return;
}
double now = Util.GetNow();
lock (_cacheLock)
{
CacheEntry cEntry;
string key = metricValue.Key();
if (!(_cache.TryGetValue(key, out cEntry)))
{
cEntry = new CacheEntry(metricValue.DeepCopy());
for (int i = 0; i < metricValue.Values.Length; i++)
{
if (dsl[i].Type == DsType.Derive ||
dsl[i].Type == DsType.Absolute ||
dsl[i].Type == DsType.Counter)
{
metricValue.Values[i] = double.NaN;
cEntry.MetricRate.Values[i] = double.NaN;
}
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
return;
}
for (int i = 0; i < metricValue.Values.Length; i++)
{
double rawValNew = metricValue.Values[i];
double rawValOld = cEntry.RawValues[i];
double rawValDiff = rawValNew - rawValOld;
double timeDiff = cEntry.MetricRate.Epoch - now;
double rateNew = rawValDiff / timeDiff;
switch (dsl[i].Type)
{
case DsType.Gauge:
// no rates calculations are done, values are stored as-is for gauge
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rawValNew;
break;
case DsType.Absolute:
// similar to gauge, except value will be divided by time diff
cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
break;
case DsType.Derive:
cEntry.RawValues[i] = rawValNew;
cEntry.MetricRate.Values[i] = rateNew;
metricValue.Values[i] = rateNew;
break;
case DsType.Counter:
// Counters are very simlar to derive except when counter wraps around
if (rawValNew < rawValOld)
{
// counter has wrapped around
cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = cEntry.MetricRate.Values[i];
}
else
{
cEntry.MetricRate.Values[i] = rateNew;
cEntry.RawValues[i] = rawValNew;
metricValue.Values[i] = rateNew;
}
break;
}
// range checks
if (metricValue.Values[i] < dsl[i].Min)
{
metricValue.Values[i] = dsl[i].Min;
cEntry.RawValues[i] = metricValue.Values[i];
}
if (metricValue.Values[i] > dsl[i].Max)
{
metricValue.Values[i] = dsl[i].Max;
cEntry.RawValues[i] = metricValue.Values[i];
}
cEntry.MetricRate.Epoch = now;
_cache[key] = cEntry;
}
}
}
private void WriteThreadProc()
{
Logger.Trace("WriteThreadProc() begin");
// Wait a few seconds to give read thread a chance to get metrics.
// Otherwise write thread is always pushing metrics _interval seconds after they were read
Thread.Sleep(15000);
while (_runWriteThread)
{
try
{
double writeStart = Util.GetNow();
int numValues = _collectedValueQueue.Count;
if (numValues > 0)
{
// Transfer current queue contents to working list
// Individual write plugins can choose how to handle the list of collectable values.
Queue <CollectableValue> collectedValues = new Queue <CollectableValue>();
lock (_queueLock)
{
while (_collectedValueQueue.Count > 0)
{
collectedValues.Enqueue(_collectedValueQueue.Dequeue());
}
}
if (collectedValues.Count > 0)
{
foreach (CollectableValue collectedValue in collectedValues)
{
if (collectedValue is MetricValue)
{
MetricValue metricValue = (MetricValue)collectedValue;
_aggregator.Aggregate(ref metricValue);
}
}
foreach (ICollectdPlugin plugin in _plugins)
{
var writePlugin = plugin as ICollectdWritePlugin;
if (writePlugin == null)
{
// skip if plugin is not a writeplugin
continue;
}
writePlugin.Write(collectedValues);
}
}
}
double writeEnd = Util.GetNow();
double elapsed = writeEnd - writeStart;
double revisedInterval = (_interval - elapsed) * 1000;
if (revisedInterval / _interval < 0.1)
{
Logger.Warn("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval);
}
if (revisedInterval >= 0)
{
Thread.Sleep((int)revisedInterval);
}
else
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Write thread exceeded cycle time");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_WRITE_EXCEEDED_CYCLE_TIME);
Logger.Log(logEvent);
}
}
catch (ThreadInterruptedException)
{
Logger.Info("Write thread interrupted");
}
catch (Exception exp)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in WriteThreadProc()");
logEvent.Exception = exp;
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION);
Logger.Log(logEvent);
Thread.Sleep(_interval * 1000);
}
}
Logger.Trace("WriteThreadProc() return");
}
private void ReadThreadProc()
{
Logger.Trace("ReadThreadProc() begin");
int numMetricsDropped = 0;
while (_runReadThread)
{
try
{
double collectionStart = Util.GetNow();
foreach (ICollectdPlugin plugin in _plugins)
{
var readPlugin = plugin as ICollectdReadPlugin;
if (readPlugin == null)
{
// skip if plugin is not a readplugin, it might be a writeplugin
continue;
}
double start = Util.GetNow();
IList <CollectableValue> collectedValues = readPlugin.Read();
double end = Util.GetNow();
Logger.Info("Read {1} items in {2:0.00}s [{0}]", readPlugin.GetType().Name, collectedValues.Count, end - start);
if (collectedValues == null || !collectedValues.Any())
{
continue;
}
lock (_queueLock)
{
foreach (CollectableValue metric in collectedValues)
{
_collectedValueQueue.Enqueue(metric);
while (_collectedValueQueue.Count >= MaxQueueSize)
{
// When queue size grows above the Max limit,
// old entries are removed
_collectedValueQueue.Dequeue();
if ((++numMetricsDropped % 1000) == 0)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exceeded max queue length");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_EXCEEDED_MAX_QUEUE_LENGTH);
Logger.Log(logEvent);
Logger.Warn("Number of metrics dropped : {0}", numMetricsDropped);
}
}
}
}
}
double collectionEnd = Util.GetNow();
double elapsed = collectionEnd - collectionStart;
double revisedInterval = (_interval - elapsed) * 1000;
if (revisedInterval / _interval < 0.1)
{
Logger.Warn("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval);
}
if (revisedInterval > 0)
{
Thread.Sleep((int)revisedInterval);
}
else
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Read thread exceeded cycle time");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_READ_EXCEEDED_CYCLE_TIME);
Logger.Log(logEvent);
}
}
catch (ThreadInterruptedException)
{
Logger.Info("Read thread interrupted");
}
catch (Exception exp)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in ReadThreadProc()");
logEvent.Exception = exp;
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION);
Logger.Log(logEvent);
Thread.Sleep(_interval * 1000);
}
}
Logger.Trace("ReadThreadProc() return");
}
public void Configure()
{
var config = ConfigurationManager.GetSection("ReadWindowsPerfCounters") as ReadWindowsPerfCountersPluginConfig;
if (config == null)
{
throw new Exception("Cannot get configuration section : ReadWindowsPerfCounters");
}
_hostName = Util.GetHostName();
// Set reload time
_reloadInterval = config.ReloadInterval;
Logger.Info("Loading metric configuration. Reload interval: {0} sec", _reloadInterval);
_lastUpdated = Util.GetNow();
// Load the metrics - this checks for existence
_metrics.Clear();
int metricCounter = 0;
foreach (CounterConfig counter in config.Counters)
{
if (counter.Instance == "")
{
// Instance not specified
if (AddPerformanceCounter(counter.Category, counter.Name,
counter.Instance, counter.Multiplier,
counter.DecimalPlaces, counter.CollectdPlugin,
counter.CollectdPluginInstance, counter.CollectdType,
counter.CollectdTypeInstance))
{
metricCounter++;
}
}
else
{
// Match instance with regex
string[] instances = new string[0];
try
{
Regex regex = new Regex(counter.Instance, RegexOptions.None);
var cat = new PerformanceCounterCategory(counter.Category);
instances = cat.GetInstanceNames();
List <string> instanceList = new List <string>();
foreach (string instance in instances)
{
if (regex.IsMatch(instance))
{
instanceList.Add(instance);
}
}
instances = instanceList.ToArray();
}
catch (ArgumentException ex)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION);
Logger.Log(logEvent);
Logger.Warn(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex);
}
catch (InvalidOperationException ex)
{
if (ex.Message.ToLower().Contains("category does not exist"))
{
Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", counter.Category));
}
else
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION);
Logger.Log(logEvent);
Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex);
}
}
catch (Exception ex)
{
LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter");
logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION);
Logger.Log(logEvent);
Logger.Warn(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex);
}
if (instances.Length == 0)
{
Logger.Warn("No instances matching category: {0}, instance: {1}", counter.Category, counter.Instance);
}
foreach (string instance in instances)
{
string instanceAlias = instance;
if (instances.Length == 1)
{
// There is just one instance
// If this is because the regex was hardcoded then replace the instance name with the CollectdInstanceName - i.e., alias the instance
// But if the regex contains wildcards then it is a fluke that there was a single match
if (counter.Instance.IndexOf("?") < 0 && counter.Instance.IndexOf("*") < 0)
{
// No wildcards => this was hardcoded value.
instanceAlias = counter.CollectdPluginInstance;
}
}
// Replace collectd_plugin_instance with the Instance got from counter
if (AddPerformanceCounter(counter.Category, counter.Name,
instance, counter.Multiplier,
counter.DecimalPlaces, counter.CollectdPlugin,
instanceAlias, counter.CollectdType,
counter.CollectdTypeInstance))
{
metricCounter++;
}
}
}
}
// Wait 1 second for the two-valued counters to be ready for next incremental read - see https://msdn.microsoft.com/en-us/library/system.diagnostics.performancecounter.nextvalue(v=vs.110).aspx
Thread.Sleep(1000);
Logger.Info("ReadWindowsPerfeCounters plugin configured {0} metrics", metricCounter);
}