public void QueueMetricCollectionResult(AnalyzerResult result, bool forceBackup = false) { lock (_metricAnalyzerResultQueue) { // check if we've reached the queue max and if so remove the eldest if (_metricAnalyzerResultQueue.Count == MetricResultsQueueMax) { _metricAnalyzerResultQueue.Dequeue(); } // backup to disk if the queue count is more than MetricResultsQueueBackupThreshold if (forceBackup || _metricAnalyzerResultQueue.Count >= MetricResultsQueueBackupThreshold) { try { File.WriteAllText(Path.Combine(AgentContext.ResultsBackupDirectory, string.Format("{0}.json", result.Id)), JsonConvert.SerializeObject(result)); result.IsBackedUp = true; } catch (Exception ex) { Log.Error(ex); } } // add to queue _metricAnalyzerResultQueue.Enqueue(result); } }
public AnalyzerResult Evaluate(DateTime collectionTime, List<CollectedMetric> collectedMetrics) { var result = new AnalyzerResult(); // analyze each metric foreach (var metric in collectedMetrics) { var shouldReportMetric = false; var metricSchedules = AgentContext.Current.PluginResourceSchedules.Where(s => s.Id == metric.ScheduleId && (!s.CollectMetricNextAt.HasValue || s.CollectMetricNextAt.Value <= collectionTime)).ToArray(); foreach (var schedule in metricSchedules) { if (schedule.IsAlertable && schedule.State.IsAnomaly(metric)) { shouldReportMetric = true; result.Anomalies.Add(new Anomaly(schedule.Id, schedule.AlertThresholdId.Value, metric.TimeStamp, schedule.State.CurrentState == PluginResourceScheduleState.StateType.Alert)); } else if (!schedule.ReportMetricNextAt.HasValue || schedule.ReportMetricNextAt.Value <= collectionTime) { shouldReportMetric = true; schedule.ReportMetricNextAt = collectionTime.AddSeconds(schedule.Frequency); } // update the next check time switch (schedule.State.CurrentState) { case PluginResourceScheduleState.StateType.Triggered: schedule.CollectMetricNextAt = collectionTime.AddMinutes(1); if (Log.IsDebugEnabled) { Log.DebugFormat("[TRIGGERED] Scheduled <{0}:Option='{1}', Id={2}, ThresholdId={3}> next metric collection for 1 min from {4} at {5}", schedule.PluginResourceTextKey, schedule.Option, schedule.Id, schedule.AlertThresholdId, collectionTime, schedule.CollectMetricNextAt); } break; case PluginResourceScheduleState.StateType.Alert: var delayedInterval = 1; if (AgentContext.Current.EnableAlertCheckDelay) { delayedInterval = (Math.Min((int)(collectionTime - schedule.State.StateChangedAt.Value).TotalHours, 4) * 15) + 1; } schedule.CollectMetricNextAt = collectionTime.AddMinutes(delayedInterval); if (Log.IsDebugEnabled) { Log.DebugFormat("[ALERT] Scheduled <{0}:Option='{1}', Id={2}, ThresholdId={3}> next metric collection for {4} min from {5} at {6}", schedule.PluginResourceTextKey, schedule.Option, schedule.Id, schedule.AlertThresholdId, delayedInterval, collectionTime, schedule.CollectMetricNextAt); } break; default: schedule.CollectMetricNextAt = collectionTime.AddSeconds(schedule.Frequency); if (Log.IsDebugEnabled) { Log.DebugFormat("[NORMAL] Scheduled <{0}:Option='{1}', Id={2}, ThresholdId={3}> next metric collection for {4} seconds frequency from {5} at {6}", schedule.PluginResourceTextKey, schedule.Option, schedule.Id, schedule.AlertThresholdId, schedule.Frequency, collectionTime, schedule.CollectMetricNextAt); } break; } } if (shouldReportMetric) { result.Metrics.Add(metric); } } return result; }