public StatusReportLoop(TimeSpan InLoopPeriod, Func <TimeSpan, string> LoopFunction) { InitialPause = TimeSpan.FromSeconds(30); LoopPeriod = InLoopPeriod; LoopTask = Task.Factory.StartNew(() => { var Cancel = CancelSource.Token; // Small pause to allow the app to get up and running before we run the first report Thread.Sleep(InitialPause); DateTime PeriodStartTime = CreationTimeUtc; lock (TaskMonitor) { while (!Cancel.IsCancellationRequested) { DateTime PeriodEndTime = DateTime.UtcNow; string ReportMessage = LoopFunction.Invoke(PeriodEndTime - PeriodStartTime); PeriodStartTime = PeriodEndTime; if (!string.IsNullOrWhiteSpace(ReportMessage)) { CrashReporterProcessServicer.WriteSlack(ReportMessage); } Monitor.Wait(TaskMonitor, LoopPeriod); } } }); }
public void Alert(string AlertKey, string AlertText, int RepeatMinimumMinutes) { // Do "no repeat" check here of recent alerts and block duplicates based on timer (use AlertKey) if (CheckRecentAlerts(AlertKey, RepeatMinimumMinutes)) { // Report an alert condition to Slack immediately CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText); } }
public void Dispose() { if (ReporterTasks != null) { foreach (var ReporterTask in ReporterTasks) { ReporterTask.Dispose(); } } CrashReporterProcessServicer.WriteSlack("CRP stopped"); }
public void Dispose() { if (StatusReportingTask != null) { CancelSource.Cancel(); lock (TaskMonitor) { Monitor.PulseAll(TaskMonitor); } StatusReportingTask.Wait(); CancelSource.Dispose(); } CrashReporterProcessServicer.WriteSlack("CRP stopped"); }
public void OnPreStopping() { CrashReporterProcessServicer.WriteSlack("CRP stopping..."); }
public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>(); StatusReportLoops.Add( new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) => { RegularStatusReport ThisLoop = (RegularStatusReport)InLoop; StringBuilder StatusReportMessage = new StringBuilder(); // #WRH Hack to disable sending status reports when the queue size is low. A more efficient solution would avoid writing the string values as well. This is just a surgical strike. bool bSendStatusReport = false; // clamp the min queue waiting time for alert to be 15 sec. But still allow for 0 to mean "never do it" TimeSpan QueueWaitTimeThreshold = (Config.Default.QueueWaitingTimeAlertThreshold > TimeSpan.Zero && Config.Default.QueueWaitingTimeAlertThreshold.TotalSeconds < 15.0) ? TimeSpan.FromSeconds(15.0) : Config.Default.QueueWaitingTimeAlertThreshold; lock (DataLock) { Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters); if (SetQueueSizesCallCount >= QueueSizes.Count) { int ProcessingStartedInPeriodReceiver = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver); int ProcessingStartedInPeriodDataRouter = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter); int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter; if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod)); ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum; int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes); string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(1)) { WaitTimeString = "< 1 minute"; } else if (WaitMinutes == 1) { WaitTimeString = "1 minute"; } else { WaitTimeString = string.Format("{0} minutes", WaitMinutes); } if (MeanWaitTime >= QueueWaitTimeThreshold && QueueWaitTimeThreshold > TimeSpan.Zero) { bSendStatusReport = true; } StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); StatusReportMessage.Append("> "); foreach (var Queue in QueueSizes) { StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " "); } StatusReportMessage.AppendLine(); } if (CountsInPeriod.Values.Sum() > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString)); foreach (var CountInPeriod in CountsInPeriod) { if (CountInPeriod.Value > 0) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } } return(bSendStatusReport ? StatusReportMessage.ToString() : ""); })); if (Config.Default.DiskSpaceAvailableAlertInterval > TimeSpan.Zero) { // clamp to min 15 sec interval. TimeSpan DiskStatusReportInterval = Config.Default.DiskSpaceAvailableAlertInterval.TotalSeconds < 15.0 ? TimeSpan.FromSeconds(15.0) : Config.Default.DiskSpaceAvailableAlertInterval; StatusReportLoops.Add( new DiskStatusReport(DiskStatusReportInterval, (InLoop, InPeriod) => { StringBuilder DailyReportMessage = new StringBuilder(); lock (DataLock) { DailyReportMessage.AppendLine("Disk space available..."); foreach (var FolderMonitor in FolderMonitors) { string FreeSpaceText = "#Error"; string Drive = FolderMonitor.Key; Int64 FreeSpace; float FreePercent; if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent); } DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText); } } return(DailyReportMessage.ToString()); })); } if (Config.Default.MonitorPerformance) { StatusReportLoops.Add( new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) => { PerfStatusReport ThisLoop = (PerfStatusReport)InLoop; StringBuilder PerfReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters); if (MeanCountsInPeriod.Count > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString)); foreach (var MeanCountInPeriod in MeanCountsInPeriod) { if (MeanCountInPeriod.Value.SampleCount == 0) { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil"); } else { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " + PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) + " / Count " + MeanCountInPeriod.Value.SampleCount + " = Mean " + PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount)); } } } } return(PerfReportMessage.ToString()); })); } ReporterTasks = StatusReportLoops; }
public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>(); StatusReportLoops.Add( new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) => { RegularStatusReport ThisLoop = (RegularStatusReport)InLoop; StringBuilder StatusReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters); if (SetQueueSizesCallCount >= QueueSizes.Count) { int ProcessingStartedInPeriodReceiver = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver); int ProcessingStartedInPeriodDataRouter = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter); int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter; if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod)); ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum; int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes); string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(1)) { WaitTimeString = "< 1 minute"; } else if (WaitMinutes == 1) { WaitTimeString = "1 minute"; } else { WaitTimeString = string.Format("{0} minutes", WaitMinutes); } StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); StatusReportMessage.Append("> "); foreach (var Queue in QueueSizes) { StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " "); } StatusReportMessage.AppendLine(); } if (CountsInPeriod.Values.Sum() > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString)); foreach (var CountInPeriod in CountsInPeriod) { if (CountInPeriod.Value > 0) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } } return(StatusReportMessage.ToString()); })); StatusReportLoops.Add( new DiskStatusReport(TimeSpan.FromDays(1.0), (InLoop, InPeriod) => { StringBuilder DailyReportMessage = new StringBuilder(); lock (DataLock) { DailyReportMessage.AppendLine("Disk space available..."); foreach (var FolderMonitor in FolderMonitors) { string FreeSpaceText = "#Error"; string Drive = FolderMonitor.Key; Int64 FreeSpace; float FreePercent; if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent); } DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText); } } return(DailyReportMessage.ToString()); })); if (Config.Default.MonitorPerformance) { StatusReportLoops.Add( new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) => { PerfStatusReport ThisLoop = (PerfStatusReport)InLoop; StringBuilder PerfReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters); if (MeanCountsInPeriod.Count > 0) { string PeriodTimeString = GetTimePeriodString(InPeriod); PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString)); foreach (var MeanCountInPeriod in MeanCountsInPeriod) { if (MeanCountInPeriod.Value.SampleCount == 0) { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil"); } else { PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " + PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) + " / Count " + MeanCountInPeriod.Value.SampleCount + " = Mean " + PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount)); } } } } return(PerfReportMessage.ToString()); })); } ReporterTasks = StatusReportLoops; }
public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); StatusReportingTask = Task.Factory.StartNew(() => { var Cancel = CancelSource.Token; lock (TaskMonitor) { while (!Cancel.IsCancellationRequested) { StringBuilder StatusReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, int> CountsInPeriod = GetCountsInPeriod(Counters, CountersAtLastReport); if (bQueueSizesAvail) { int ProcessingStartedInPeriod = 0; CountsInPeriod.TryGetValue(StatusReportingConstants.ProcessingStartedEvent, out ProcessingStartedInPeriod); if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromMinutes(0.5 * Config.Default.MinutesBetweenQueueSizeReports * (QueueSizeSum + QueueSizeSumAtLastReport) / ProcessingStartedInPeriod); QueueSizeSumAtLastReport = QueueSizeSum; string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(2)) { WaitTimeString = MeanWaitTime.TotalSeconds + " seconds"; } else { WaitTimeString = Convert.ToInt32(MeanWaitTime.TotalMinutes) + " minutes"; } StartupMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); foreach (var Queue in QueueSizes) { StatusReportMessage.AppendLine("> " + Queue.Key + " " + Queue.Value); } } if (CountsInPeriod.Count > 0) { StatusReportMessage.AppendLine(string.Format("Events in the last {0} minutes...", Config.Default.MinutesBetweenQueueSizeReports)); foreach (var CountInPeriod in CountsInPeriod) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } CrashReporterProcessServicer.WriteSlack(StatusReportMessage.ToString()); Monitor.Wait(TaskMonitor, TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports)); } } }); }
public void Alert(string AlertText) { // Report an alert condition to Slack immediately CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText); }
public void Start() { CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString)); StringBuilder StartupMessage = new StringBuilder(); StartupMessage.AppendLine("Queues..."); foreach (var Queue in QueueLocations) { StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value); } CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString()); ReporterTasks = new[] { new StatusReportLoop(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), InPeriod => { StringBuilder StatusReportMessage = new StringBuilder(); lock (DataLock) { Dictionary <string, int> CountsInPeriod = GetCountsInPeriod(Counters, CountersAtLastReport); if (SetQueueSizesCallCount >= QueueSizes.Count) { int ProcessingStartedInPeriodReceiver = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver); int ProcessingStartedInPeriodDataRouter = 0; CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter); int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter; if (ProcessingStartedInPeriod > 0) { int QueueSizeSum = QueueSizes.Values.Sum(); TimeSpan MeanWaitTime = TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + QueueSizeSumAtLastReport) / ProcessingStartedInPeriod)); QueueSizeSumAtLastReport = QueueSizeSum; int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes); string WaitTimeString; if (MeanWaitTime == TimeSpan.Zero) { WaitTimeString = "nil"; } else if (MeanWaitTime < TimeSpan.FromMinutes(1)) { WaitTimeString = "< 1 minute"; } else if (WaitMinutes == 1) { WaitTimeString = "1 minute"; } else { WaitTimeString = string.Format("{0} minutes", WaitMinutes); } StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString); } StatusReportMessage.AppendLine("Queue sizes..."); StatusReportMessage.Append("> "); foreach (var Queue in QueueSizes) { StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " "); } StatusReportMessage.AppendLine(); } if (CountsInPeriod.Count > 0) { string PeriodTimeString; if (InPeriod.TotalMinutes < 1.0) { PeriodTimeString = string.Format("{0:N0} seconds", InPeriod.TotalSeconds); } else { PeriodTimeString = string.Format("{0:N0} minutes", InPeriod.TotalMinutes); } StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString)); foreach (var CountInPeriod in CountsInPeriod) { if (CountInPeriod.Value > 0) { StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value); } } } } return(StatusReportMessage.ToString()); }), new StatusReportLoop(TimeSpan.FromDays(1.0), InPeriod => { StringBuilder DailyReportMessage = new StringBuilder(); lock (DataLock) { DailyReportMessage.AppendLine("Disk space available..."); foreach (var FolderMonitor in FolderMonitors) { string FreeSpaceText = "#Error"; string Drive = FolderMonitor.Key; Int64 FreeSpace; float FreePercent; if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent); } DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText); } } return(DailyReportMessage.ToString()); }), }; }