public void StartConnection() { double now = Util.GetNow(); if (now < (_lastConnectTime + ConnectionRetryDelay)) { return; } lock (_connectionLock) { try { var cf = new ConnectionFactory { Uri = _url }; _connection = cf.CreateConnection(); _channel = _connection.CreateModel(); _connected = true; _lastConnectTime = Util.GetNow(); Logger.Debug("Connection started."); } catch (Exception exp) { Logger.Error("Got exception when connecting to AMQP broker : ", exp); } } }
public void RemoveExpiredEntries() { // If rates are not stored then there is nothing to remove if (!_storeRates) { return; } double now = Util.GetNow(); double expirationTime = now - _timeoutSeconds; var removeList = new List <string>(); lock (_cacheLock) { removeList.AddRange(from pair in _cache let cEntry = pair.Value where cEntry.MetricRate.Epoch < expirationTime select pair.Key); if (removeList.Count > 0) { Logger.Debug("Removing expired entries: {0}", removeList.Count); } foreach (string key in removeList) { _cache.Remove(key); } } }
public void StartConnection() { double now = Util.GetNow(); if (now < (_lastConnectTime + ConnectionRetryDelay)) { return; } lock (_connectionLock) { try { var cf = new ConnectionFactory { Uri = _url }; _connection = cf.CreateConnection(); _channel = _connection.CreateModel(); _connected = true; _lastConnectTime = Util.GetNow(); Logger.Debug("Connection started."); } catch (Exception exp) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when connecting to AMQP broker"); logEvent.Exception = exp; logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); Logger.Log(logEvent); } } }
public IList <CollectableValue> Read() { // Check if it is time to reload the metric config // We need to periodially reload the config in case new instances or new categories are available (e.g. due to cluster move) if (Util.GetNow() - _lastUpdated >= _reloadInterval) { Configure(); } var metricValueList = new List <CollectableValue>(); foreach (Metric metric in _metrics) { try { var vals = new List <double>(); foreach (PerformanceCounter ctr in metric.Counters) { double val = ctr.NextValue(); val = val * metric.Multiplier; if (metric.DecimalPlaces >= 0) { val = Math.Round(val, metric.DecimalPlaces); } vals.Add(val); } var metricValue = new MetricValue { HostName = _hostName, PluginName = metric.CollectdPlugin, PluginInstanceName = metric.CollectdPluginInstance, TypeName = metric.CollectdType, TypeInstanceName = metric.CollectdTypeInstance, Values = vals.ToArray(), FriendlyNames = metric.FriendlyNames }; metricValue.Epoch = Util.toEpoch(DateTime.UtcNow); metricValueList.Add(metricValue); } catch (Exception ex) { Logger.Warn(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex); } } return(metricValueList); }
public void Aggregate(ref MetricValue metricValue) { // If rates are not stored then there is nothing to aggregate if (!_storeRates) { return; } IList <DataSource> dsl = DataSetCollection.Instance.GetDataSource(metricValue.TypeName); if (dsl == null || metricValue.Values.Length != dsl.Count) { return; } double now = Util.GetNow(); lock (_cacheLock) { CacheEntry cEntry; string key = metricValue.Key(); if (!(_cache.TryGetValue(key, out cEntry))) { cEntry = new CacheEntry(metricValue.DeepCopy()); for (int i = 0; i < metricValue.Values.Length; i++) { if (dsl[i].Type == DsType.Derive || dsl[i].Type == DsType.Absolute || dsl[i].Type == DsType.Counter) { metricValue.Values[i] = double.NaN; cEntry.MetricRate.Values[i] = double.NaN; } } cEntry.MetricRate.Epoch = now; _cache[key] = cEntry; return; } for (int i = 0; i < metricValue.Values.Length; i++) { double rawValNew = metricValue.Values[i]; double rawValOld = cEntry.RawValues[i]; double rawValDiff = rawValNew - rawValOld; double timeDiff = cEntry.MetricRate.Epoch - now; double rateNew = rawValDiff / timeDiff; switch (dsl[i].Type) { case DsType.Gauge: // no rates calculations are done, values are stored as-is for gauge cEntry.RawValues[i] = rawValNew; cEntry.MetricRate.Values[i] = rawValNew; break; case DsType.Absolute: // similar to gauge, except value will be divided by time diff cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff; cEntry.RawValues[i] = rawValNew; metricValue.Values[i] = cEntry.MetricRate.Values[i]; break; case DsType.Derive: cEntry.RawValues[i] = rawValNew; cEntry.MetricRate.Values[i] = rateNew; metricValue.Values[i] = rateNew; break; case DsType.Counter: // Counters are very simlar to derive except when counter wraps around if (rawValNew < rawValOld) { // counter has wrapped around cEntry.MetricRate.Values[i] = metricValue.Values[i] / timeDiff; cEntry.RawValues[i] = rawValNew; metricValue.Values[i] = cEntry.MetricRate.Values[i]; } else { cEntry.MetricRate.Values[i] = rateNew; cEntry.RawValues[i] = rawValNew; metricValue.Values[i] = rateNew; } break; } // range checks if (metricValue.Values[i] < dsl[i].Min) { metricValue.Values[i] = dsl[i].Min; cEntry.RawValues[i] = metricValue.Values[i]; } if (metricValue.Values[i] > dsl[i].Max) { metricValue.Values[i] = dsl[i].Max; cEntry.RawValues[i] = metricValue.Values[i]; } cEntry.MetricRate.Epoch = now; _cache[key] = cEntry; } } }
private void WriteThreadProc() { Logger.Trace("WriteThreadProc() begin"); // Wait a few seconds to give read thread a chance to get metrics. // Otherwise write thread is always pushing metrics _interval seconds after they were read Thread.Sleep(15000); while (_runWriteThread) { try { double writeStart = Util.GetNow(); int numValues = _collectedValueQueue.Count; if (numValues > 0) { // Transfer current queue contents to working list // Individual write plugins can choose how to handle the list of collectable values. Queue <CollectableValue> collectedValues = new Queue <CollectableValue>(); lock (_queueLock) { while (_collectedValueQueue.Count > 0) { collectedValues.Enqueue(_collectedValueQueue.Dequeue()); } } if (collectedValues.Count > 0) { foreach (CollectableValue collectedValue in collectedValues) { if (collectedValue is MetricValue) { MetricValue metricValue = (MetricValue)collectedValue; _aggregator.Aggregate(ref metricValue); } } foreach (ICollectdPlugin plugin in _plugins) { var writePlugin = plugin as ICollectdWritePlugin; if (writePlugin == null) { // skip if plugin is not a writeplugin continue; } writePlugin.Write(collectedValues); } } } double writeEnd = Util.GetNow(); double elapsed = writeEnd - writeStart; double revisedInterval = (_interval - elapsed) * 1000; if (revisedInterval / _interval < 0.1) { Logger.Warn("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval); } if (revisedInterval >= 0) { Thread.Sleep((int)revisedInterval); } else { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Write thread exceeded cycle time"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_WRITE_EXCEEDED_CYCLE_TIME); Logger.Log(logEvent); } } catch (ThreadInterruptedException) { Logger.Info("Write thread interrupted"); } catch (Exception exp) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in WriteThreadProc()"); logEvent.Exception = exp; logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } Logger.Trace("WriteThreadProc() return"); }
private void ReadThreadProc() { Logger.Trace("ReadThreadProc() begin"); int numMetricsDropped = 0; while (_runReadThread) { try { double collectionStart = Util.GetNow(); foreach (ICollectdPlugin plugin in _plugins) { var readPlugin = plugin as ICollectdReadPlugin; if (readPlugin == null) { // skip if plugin is not a readplugin, it might be a writeplugin continue; } double start = Util.GetNow(); IList <CollectableValue> collectedValues = readPlugin.Read(); double end = Util.GetNow(); Logger.Info("Read {1} items in {2:0.00}s [{0}]", readPlugin.GetType().Name, collectedValues.Count, end - start); if (collectedValues == null || !collectedValues.Any()) { continue; } lock (_queueLock) { foreach (CollectableValue metric in collectedValues) { _collectedValueQueue.Enqueue(metric); while (_collectedValueQueue.Count >= MaxQueueSize) { // When queue size grows above the Max limit, // old entries are removed _collectedValueQueue.Dequeue(); if ((++numMetricsDropped % 1000) == 0) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exceeded max queue length"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_EXCEEDED_MAX_QUEUE_LENGTH); Logger.Log(logEvent); Logger.Warn("Number of metrics dropped : {0}", numMetricsDropped); } } } } } double collectionEnd = Util.GetNow(); double elapsed = collectionEnd - collectionStart; double revisedInterval = (_interval - elapsed) * 1000; if (revisedInterval / _interval < 0.1) { Logger.Warn("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval); } if (revisedInterval > 0) { Thread.Sleep((int)revisedInterval); } else { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Read thread exceeded cycle time"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_READ_EXCEEDED_CYCLE_TIME); Logger.Log(logEvent); } } catch (ThreadInterruptedException) { Logger.Info("Read thread interrupted"); } catch (Exception exp) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in ReadThreadProc()"); logEvent.Exception = exp; logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } Logger.Trace("ReadThreadProc() return"); }
public void Configure() { var config = ConfigurationManager.GetSection("ReadWindowsPerfCounters") as ReadWindowsPerfCountersPluginConfig; if (config == null) { throw new Exception("Cannot get configuration section : ReadWindowsPerfCounters"); } _hostName = Util.GetHostName(); // Set reload time _reloadInterval = config.ReloadInterval; Logger.Info("Loading metric configuration. Reload interval: {0} sec", _reloadInterval); _lastUpdated = Util.GetNow(); // Load the metrics - this checks for existence _metrics.Clear(); int metricCounter = 0; foreach (CounterConfig counter in config.Counters) { if (counter.Instance == "") { // Instance not specified if (AddPerformanceCounter(counter.Category, counter.Name, counter.Instance, counter.Multiplier, counter.DecimalPlaces, counter.CollectdPlugin, counter.CollectdPluginInstance, counter.CollectdType, counter.CollectdTypeInstance)) { metricCounter++; } } else { // Match instance with regex string[] instances = new string[0]; try { Regex regex = new Regex(counter.Instance, RegexOptions.None); var cat = new PerformanceCounterCategory(counter.Category); instances = cat.GetInstanceNames(); List <string> instanceList = new List <string>(); foreach (string instance in instances) { if (regex.IsMatch(instance)) { instanceList.Add(instance); } } instances = instanceList.ToArray(); } catch (ArgumentException ex) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); Logger.Log(logEvent); Logger.Warn(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex); } catch (InvalidOperationException ex) { if (ex.Message.ToLower().Contains("category does not exist")) { Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", counter.Category)); } else { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); Logger.Log(logEvent); Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex); } } catch (Exception ex) { LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); Logger.Log(logEvent); Logger.Warn(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex); } if (instances.Length == 0) { Logger.Warn("No instances matching category: {0}, instance: {1}", counter.Category, counter.Instance); } foreach (string instance in instances) { string instanceAlias = instance; if (instances.Length == 1) { // There is just one instance // If this is because the regex was hardcoded then replace the instance name with the CollectdInstanceName - i.e., alias the instance // But if the regex contains wildcards then it is a fluke that there was a single match if (counter.Instance.IndexOf("?") < 0 && counter.Instance.IndexOf("*") < 0) { // No wildcards => this was hardcoded value. instanceAlias = counter.CollectdPluginInstance; } } // Replace collectd_plugin_instance with the Instance got from counter if (AddPerformanceCounter(counter.Category, counter.Name, instance, counter.Multiplier, counter.DecimalPlaces, counter.CollectdPlugin, instanceAlias, counter.CollectdType, counter.CollectdTypeInstance)) { metricCounter++; } } } } // Wait 1 second for the two-valued counters to be ready for next incremental read - see https://msdn.microsoft.com/en-us/library/system.diagnostics.performancecounter.nextvalue(v=vs.110).aspx Thread.Sleep(1000); Logger.Info("ReadWindowsPerfeCounters plugin configured {0} metrics", metricCounter); }