public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>(); StatusReportLoops.Add( new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) => { RegularStatusReport ThisLoop = (RegularStatusReport)InLoop; StringBuilder StatusReportMessage = new StringBuilder(); // #WRH Hack to disable sending status reports when the queue size is low. A more efficient solution would avoid writing the string values as well. This is just a surgical strike. bool bSendStatusReport = false; // clamp the min queue waiting time for alert to be 15 sec. But still allow for 0 to mean "never do it" TimeSpan QueueWaitTimeThreshold = (Config.Default.QueueWaitingTimeAlertThreshold > TimeSpan.Zero && Config.Default.QueueWaitingTimeAlertThreshold.TotalSeconds < 15.0) ? TimeSpan.FromSeconds(15.0) : Config.Default.QueueWaitingTimeAlertThreshold; lock (DataLock) { Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters); if (SetQueueSizesCallCount >= QueueSizes.Count) { int ProcessingStartedInPeriodReceiver = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver); int ProcessingStartedInPeriodDataRouter = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter); int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter; if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod)); ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum; int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes); string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(1)) { WaitTimeString = "< 1 minute"; } else if (WaitMinutes == 1) { WaitTimeString = "1 minute"; } else { WaitTimeString = string.Format("{0} minutes", WaitMinutes); } if (MeanWaitTime >= QueueWaitTimeThreshold && QueueWaitTimeThreshold > TimeSpan.Zero) { bSendStatusReport = true; } StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); StatusReportMessage.Append("> "); foreach (var Queue in QueueSizes) { StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " "); } StatusReportMessage.AppendLine(); } if (CountsInPeriod.Values.Sum() > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString)); foreach (var CountInPeriod in CountsInPeriod) { if (CountInPeriod.Value > 0) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } } return(bSendStatusReport ? StatusReportMessage.ToString() : ""); })); if (Config.Default.DiskSpaceAvailableAlertInterval > TimeSpan.Zero) { // clamp to min 15 sec interval. TimeSpan DiskStatusReportInterval = Config.Default.DiskSpaceAvailableAlertInterval.TotalSeconds < 15.0 ? TimeSpan.FromSeconds(15.0) : Config.Default.DiskSpaceAvailableAlertInterval; StatusReportLoops.Add( new DiskStatusReport(DiskStatusReportInterval, (InLoop, InPeriod) => { StringBuilder DailyReportMessage = new StringBuilder(); lock (DataLock) { DailyReportMessage.AppendLine("Disk space available..."); foreach (var FolderMonitor in FolderMonitors) { string FreeSpaceText = "#Error"; string Drive = FolderMonitor.Key; Int64 FreeSpace; float FreePercent; if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent); } DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText); } } return(DailyReportMessage.ToString()); })); } if (Config.Default.MonitorPerformance) { StatusReportLoops.Add( new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) => { PerfStatusReport ThisLoop = (PerfStatusReport)InLoop; StringBuilder PerfReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters); if (MeanCountsInPeriod.Count > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString)); foreach (var MeanCountInPeriod in MeanCountsInPeriod) { if (MeanCountInPeriod.Value.SampleCount == 0) { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil"); } else { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " + PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) + " / Count " + MeanCountInPeriod.Value.SampleCount + " = Mean " + PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount)); } } } } return(PerfReportMessage.ToString()); })); } ReporterTasks = StatusReportLoops; }
public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>(); StatusReportLoops.Add( new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) => { RegularStatusReport ThisLoop = (RegularStatusReport)InLoop; StringBuilder StatusReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters); if (SetQueueSizesCallCount >= QueueSizes.Count) { int ProcessingStartedInPeriodReceiver = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver); int ProcessingStartedInPeriodDataRouter = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter); int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter; if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod)); ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum; int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes); string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(1)) { WaitTimeString = "< 1 minute"; } else if (WaitMinutes == 1) { WaitTimeString = "1 minute"; } else { WaitTimeString = string.Format("{0} minutes", WaitMinutes); } StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); StatusReportMessage.Append("> "); foreach (var Queue in QueueSizes) { StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " "); } StatusReportMessage.AppendLine(); } if (CountsInPeriod.Values.Sum() > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString)); foreach (var CountInPeriod in CountsInPeriod) { if (CountInPeriod.Value > 0) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } } return(StatusReportMessage.ToString()); })); StatusReportLoops.Add( new DiskStatusReport(TimeSpan.FromDays(1.0), (InLoop, InPeriod) => { StringBuilder DailyReportMessage = new StringBuilder(); lock (DataLock) { DailyReportMessage.AppendLine("Disk space available..."); foreach (var FolderMonitor in FolderMonitors) { string FreeSpaceText = "#Error"; string Drive = FolderMonitor.Key; Int64 FreeSpace; float FreePercent; if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent); } DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText); } } return(DailyReportMessage.ToString()); })); if (Config.Default.MonitorPerformance) { StatusReportLoops.Add( new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) => { PerfStatusReport ThisLoop = (PerfStatusReport)InLoop; StringBuilder PerfReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters); if (MeanCountsInPeriod.Count > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString)); foreach (var MeanCountInPeriod in MeanCountsInPeriod) { if (MeanCountInPeriod.Value.SampleCount == 0) { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil"); } else { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " + PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) + " / Count " + MeanCountInPeriod.Value.SampleCount + " = Mean " + PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount)); } } } } return(PerfReportMessage.ToString()); })); } ReporterTasks = StatusReportLoops; }