public StatusReportLoop(TimeSpan InLoopPeriod, Func <TimeSpan, string> LoopFunction)
{
InitialPause = TimeSpan.FromSeconds(30);
LoopPeriod = InLoopPeriod;
LoopTask = Task.Factory.StartNew(() =>
{
var Cancel = CancelSource.Token;
// Small pause to allow the app to get up and running before we run the first report
Thread.Sleep(InitialPause);
DateTime PeriodStartTime = CreationTimeUtc;
lock (TaskMonitor)
{
while (!Cancel.IsCancellationRequested)
{
DateTime PeriodEndTime = DateTime.UtcNow;
string ReportMessage = LoopFunction.Invoke(PeriodEndTime - PeriodStartTime);
PeriodStartTime = PeriodEndTime;
if (!string.IsNullOrWhiteSpace(ReportMessage))
{
CrashReporterProcessServicer.WriteSlack(ReportMessage);
}
Monitor.Wait(TaskMonitor, LoopPeriod);
}
}
});
}
public void Alert(string AlertKey, string AlertText, int RepeatMinimumMinutes)
{
// Do "no repeat" check here of recent alerts and block duplicates based on timer (use AlertKey)
if (CheckRecentAlerts(AlertKey, RepeatMinimumMinutes))
{
// Report an alert condition to Slack immediately
CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText);
}
}
public void Dispose()
{
if (ReporterTasks != null)
{
foreach (var ReporterTask in ReporterTasks)
{
ReporterTask.Dispose();
}
}
CrashReporterProcessServicer.WriteSlack("CRP stopped");
}
public void Dispose()
{
if (StatusReportingTask != null)
{
CancelSource.Cancel();
lock (TaskMonitor)
{
Monitor.PulseAll(TaskMonitor);
}
StatusReportingTask.Wait();
CancelSource.Dispose();
}
CrashReporterProcessServicer.WriteSlack("CRP stopped");
}
public void OnPreStopping()
{
CrashReporterProcessServicer.WriteSlack("CRP stopping...");
}
public void Start()
{
CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString));
StringBuilder StartupMessage = new StringBuilder();
StartupMessage.AppendLine("Queues...");
foreach (var Queue in QueueLocations)
{
StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value);
}
CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString());
List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>();
StatusReportLoops.Add(
new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) =>
{
RegularStatusReport ThisLoop = (RegularStatusReport)InLoop;
StringBuilder StatusReportMessage = new StringBuilder();
// #WRH Hack to disable sending status reports when the queue size is low. A more efficient solution would avoid writing the string values as well. This is just a surgical strike.
bool bSendStatusReport = false;
// clamp the min queue waiting time for alert to be 15 sec. But still allow for 0 to mean "never do it"
TimeSpan QueueWaitTimeThreshold = (Config.Default.QueueWaitingTimeAlertThreshold > TimeSpan.Zero && Config.Default.QueueWaitingTimeAlertThreshold.TotalSeconds < 15.0) ? TimeSpan.FromSeconds(15.0) : Config.Default.QueueWaitingTimeAlertThreshold;
lock (DataLock)
{
Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters);
if (SetQueueSizesCallCount >= QueueSizes.Count)
{
int ProcessingStartedInPeriodReceiver = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver);
int ProcessingStartedInPeriodDataRouter = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter);
int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter;
if (ProcessingStartedInPeriod > 0)
{
int QueueSizeSum = QueueSizes.Values.Sum();
TimeSpan MeanWaitTime =
TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod));
ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum;
int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes);
string WaitTimeString;
if (MeanWaitTime == TimeSpan.Zero)
{
WaitTimeString = "nil";
}
else if (MeanWaitTime < TimeSpan.FromMinutes(1))
{
WaitTimeString = "< 1 minute";
}
else if (WaitMinutes == 1)
{
WaitTimeString = "1 minute";
}
else
{
WaitTimeString = string.Format("{0} minutes", WaitMinutes);
}
if (MeanWaitTime >= QueueWaitTimeThreshold && QueueWaitTimeThreshold > TimeSpan.Zero)
{
bSendStatusReport = true;
}
StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString);
}
StatusReportMessage.AppendLine("Queue sizes...");
StatusReportMessage.Append("> ");
foreach (var Queue in QueueSizes)
{
StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " ");
}
StatusReportMessage.AppendLine();
}
if (CountsInPeriod.Values.Sum() > 0)
{
string PeriodTimeString = GetTimePeriodString(InPeriod);
StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString));
foreach (var CountInPeriod in CountsInPeriod)
{
if (CountInPeriod.Value > 0)
{
StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value);
}
}
}
}
return(bSendStatusReport ? StatusReportMessage.ToString() : "");
}));
if (Config.Default.DiskSpaceAvailableAlertInterval > TimeSpan.Zero)
{
// clamp to min 15 sec interval.
TimeSpan DiskStatusReportInterval = Config.Default.DiskSpaceAvailableAlertInterval.TotalSeconds < 15.0 ? TimeSpan.FromSeconds(15.0) : Config.Default.DiskSpaceAvailableAlertInterval;
StatusReportLoops.Add(
new DiskStatusReport(DiskStatusReportInterval, (InLoop, InPeriod) =>
{
StringBuilder DailyReportMessage = new StringBuilder();
lock (DataLock)
{
DailyReportMessage.AppendLine("Disk space available...");
foreach (var FolderMonitor in FolderMonitors)
{
string FreeSpaceText = "#Error";
string Drive = FolderMonitor.Key;
Int64 FreeSpace;
float FreePercent;
if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent);
}
DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText);
}
}
return(DailyReportMessage.ToString());
}));
}
if (Config.Default.MonitorPerformance)
{
StatusReportLoops.Add(
new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) =>
{
PerfStatusReport ThisLoop = (PerfStatusReport)InLoop;
StringBuilder PerfReportMessage = new StringBuilder();
lock (DataLock)
{
Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters);
if (MeanCountsInPeriod.Count > 0)
{
string PeriodTimeString = GetTimePeriodString(InPeriod);
PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString));
foreach (var MeanCountInPeriod in MeanCountsInPeriod)
{
if (MeanCountInPeriod.Value.SampleCount == 0)
{
PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil");
}
else
{
PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " +
PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) +
" / Count " + MeanCountInPeriod.Value.SampleCount +
" = Mean " +
PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount));
}
}
}
}
return(PerfReportMessage.ToString());
}));
}
ReporterTasks = StatusReportLoops;
}
public void Start()
{
CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString));
StringBuilder StartupMessage = new StringBuilder();
StartupMessage.AppendLine("Queues...");
foreach (var Queue in QueueLocations)
{
StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value);
}
CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString());
List <StatusReportLoop> StatusReportLoops = new List <StatusReportLoop>();
StatusReportLoops.Add(
new RegularStatusReport(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), (InLoop, InPeriod) =>
{
RegularStatusReport ThisLoop = (RegularStatusReport)InLoop;
StringBuilder StatusReportMessage = new StringBuilder();
lock (DataLock)
{
Dictionary <string, int> CountsInPeriod = ThisLoop.GetCountsInPeriod(Counters);
if (SetQueueSizesCallCount >= QueueSizes.Count)
{
int ProcessingStartedInPeriodReceiver = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver);
int ProcessingStartedInPeriodDataRouter = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter);
int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter;
if (ProcessingStartedInPeriod > 0)
{
int QueueSizeSum = QueueSizes.Values.Sum();
TimeSpan MeanWaitTime =
TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + ThisLoop.QueueSizeSumAtLastReport) / ProcessingStartedInPeriod));
ThisLoop.QueueSizeSumAtLastReport = QueueSizeSum;
int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes);
string WaitTimeString;
if (MeanWaitTime == TimeSpan.Zero)
{
WaitTimeString = "nil";
}
else if (MeanWaitTime < TimeSpan.FromMinutes(1))
{
WaitTimeString = "< 1 minute";
}
else if (WaitMinutes == 1)
{
WaitTimeString = "1 minute";
}
else
{
WaitTimeString = string.Format("{0} minutes", WaitMinutes);
}
StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString);
}
StatusReportMessage.AppendLine("Queue sizes...");
StatusReportMessage.Append("> ");
foreach (var Queue in QueueSizes)
{
StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " ");
}
StatusReportMessage.AppendLine();
}
if (CountsInPeriod.Values.Sum() > 0)
{
string PeriodTimeString = GetTimePeriodString(InPeriod);
StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString));
foreach (var CountInPeriod in CountsInPeriod)
{
if (CountInPeriod.Value > 0)
{
StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value);
}
}
}
}
return(StatusReportMessage.ToString());
}));
StatusReportLoops.Add(
new DiskStatusReport(TimeSpan.FromDays(1.0), (InLoop, InPeriod) =>
{
StringBuilder DailyReportMessage = new StringBuilder();
lock (DataLock)
{
DailyReportMessage.AppendLine("Disk space available...");
foreach (var FolderMonitor in FolderMonitors)
{
string FreeSpaceText = "#Error";
string Drive = FolderMonitor.Key;
Int64 FreeSpace;
float FreePercent;
if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent);
}
DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText);
}
}
return(DailyReportMessage.ToString());
}));
if (Config.Default.MonitorPerformance)
{
StatusReportLoops.Add(
new PerfStatusReport(TimeSpan.FromMinutes(10), (InLoop, InPeriod) =>
{
PerfStatusReport ThisLoop = (PerfStatusReport)InLoop;
StringBuilder PerfReportMessage = new StringBuilder();
lock (DataLock)
{
Dictionary <string, StatusReportingMeanCounter> MeanCountsInPeriod = ThisLoop.GetMeanCountsInPeriod(MeanCounters);
if (MeanCountsInPeriod.Count > 0)
{
string PeriodTimeString = GetTimePeriodString(InPeriod);
PerfReportMessage.AppendLine(string.Format("Performance in the last {0}...", PeriodTimeString));
foreach (var MeanCountInPeriod in MeanCountsInPeriod)
{
if (MeanCountInPeriod.Value.SampleCount == 0)
{
PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": nil");
}
else
{
PerfReportMessage.AppendLine("> " + MeanCountInPeriod.Key + ": Total " +
PerfStatusReport.GetPerfTimeString((float)MeanCountInPeriod.Value.TotalMillisec) +
" / Count " + MeanCountInPeriod.Value.SampleCount +
" = Mean " +
PerfStatusReport.GetPerfTimeString(MeanCountInPeriod.Value.TotalMillisec / (float)MeanCountInPeriod.Value.SampleCount));
}
}
}
}
return(PerfReportMessage.ToString());
}));
}
ReporterTasks = StatusReportLoops;
}
public void Start()
{
CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString));
StringBuilder StartupMessage = new StringBuilder();
StartupMessage.AppendLine("Queues...");
foreach (var Queue in QueueLocations)
{
StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value);
}
CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString());
StatusReportingTask = Task.Factory.StartNew(() =>
{
var Cancel = CancelSource.Token;
lock (TaskMonitor)
{
while (!Cancel.IsCancellationRequested)
{
StringBuilder StatusReportMessage = new StringBuilder();
lock (DataLock)
{
Dictionary <string, int> CountsInPeriod = GetCountsInPeriod(Counters, CountersAtLastReport);
if (bQueueSizesAvail)
{
int ProcessingStartedInPeriod = 0;
CountsInPeriod.TryGetValue(StatusReportingConstants.ProcessingStartedEvent, out ProcessingStartedInPeriod);
if (ProcessingStartedInPeriod > 0)
{
int QueueSizeSum = QueueSizes.Values.Sum();
TimeSpan MeanWaitTime =
TimeSpan.FromMinutes(0.5 * Config.Default.MinutesBetweenQueueSizeReports *
(QueueSizeSum + QueueSizeSumAtLastReport) / ProcessingStartedInPeriod);
QueueSizeSumAtLastReport = QueueSizeSum;
string WaitTimeString;
if (MeanWaitTime == TimeSpan.Zero)
{
WaitTimeString = "nil";
}
else if (MeanWaitTime < TimeSpan.FromMinutes(2))
{
WaitTimeString = MeanWaitTime.TotalSeconds + " seconds";
}
else
{
WaitTimeString = Convert.ToInt32(MeanWaitTime.TotalMinutes) + " minutes";
}
StartupMessage.AppendLine("Queue waiting time " + WaitTimeString);
}
StatusReportMessage.AppendLine("Queue sizes...");
foreach (var Queue in QueueSizes)
{
StatusReportMessage.AppendLine("> " + Queue.Key + " " + Queue.Value);
}
}
if (CountsInPeriod.Count > 0)
{
StatusReportMessage.AppendLine(string.Format("Events in the last {0} minutes...",
Config.Default.MinutesBetweenQueueSizeReports));
foreach (var CountInPeriod in CountsInPeriod)
{
StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value);
}
}
}
CrashReporterProcessServicer.WriteSlack(StatusReportMessage.ToString());
Monitor.Wait(TaskMonitor, TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports));
}
}
});
}
public void Alert(string AlertText)
{
// Report an alert condition to Slack immediately
CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText);
}
public void Start()
{
CrashReporterProcessServicer.WriteSlack(string.Format("CRP started (version {0})", Config.Default.VersionString));
StringBuilder StartupMessage = new StringBuilder();
StartupMessage.AppendLine("Queues...");
foreach (var Queue in QueueLocations)
{
StartupMessage.AppendLine("> " + Queue.Key + " @ " + Queue.Value);
}
CrashReporterProcessServicer.WriteSlack(StartupMessage.ToString());
ReporterTasks = new[]
{
new StatusReportLoop(TimeSpan.FromMinutes(Config.Default.MinutesBetweenQueueSizeReports), InPeriod =>
{
StringBuilder StatusReportMessage = new StringBuilder();
lock (DataLock)
{
Dictionary <string, int> CountsInPeriod = GetCountsInPeriod(Counters, CountersAtLastReport);
if (SetQueueSizesCallCount >= QueueSizes.Count)
{
int ProcessingStartedInPeriodReceiver = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedReceiverEvent, out ProcessingStartedInPeriodReceiver);
int ProcessingStartedInPeriodDataRouter = 0;
CountsInPeriod.TryGetValue(StatusReportingEventNames.ProcessingStartedDataRouterEvent, out ProcessingStartedInPeriodDataRouter);
int ProcessingStartedInPeriod = ProcessingStartedInPeriodReceiver + ProcessingStartedInPeriodDataRouter;
if (ProcessingStartedInPeriod > 0)
{
int QueueSizeSum = QueueSizes.Values.Sum();
TimeSpan MeanWaitTime =
TimeSpan.FromTicks((long)(0.5 * InPeriod.Ticks * (QueueSizeSum + QueueSizeSumAtLastReport) / ProcessingStartedInPeriod));
QueueSizeSumAtLastReport = QueueSizeSum;
int WaitMinutes = Convert.ToInt32(MeanWaitTime.TotalMinutes);
string WaitTimeString;
if (MeanWaitTime == TimeSpan.Zero)
{
WaitTimeString = "nil";
}
else if (MeanWaitTime < TimeSpan.FromMinutes(1))
{
WaitTimeString = "< 1 minute";
}
else if (WaitMinutes == 1)
{
WaitTimeString = "1 minute";
}
else
{
WaitTimeString = string.Format("{0} minutes", WaitMinutes);
}
StatusReportMessage.AppendLine("Queue waiting time " + WaitTimeString);
}
StatusReportMessage.AppendLine("Queue sizes...");
StatusReportMessage.Append("> ");
foreach (var Queue in QueueSizes)
{
StatusReportMessage.Append(Queue.Key + " " + Queue.Value + " ");
}
StatusReportMessage.AppendLine();
}
if (CountsInPeriod.Count > 0)
{
string PeriodTimeString;
if (InPeriod.TotalMinutes < 1.0)
{
PeriodTimeString = string.Format("{0:N0} seconds", InPeriod.TotalSeconds);
}
else
{
PeriodTimeString = string.Format("{0:N0} minutes", InPeriod.TotalMinutes);
}
StatusReportMessage.AppendLine(string.Format("Events in the last {0}...", PeriodTimeString));
foreach (var CountInPeriod in CountsInPeriod)
{
if (CountInPeriod.Value > 0)
{
StatusReportMessage.AppendLine("> " + CountInPeriod.Key + " " + CountInPeriod.Value);
}
}
}
}
return(StatusReportMessage.ToString());
}),
new StatusReportLoop(TimeSpan.FromDays(1.0), InPeriod =>
{
StringBuilder DailyReportMessage = new StringBuilder();
lock (DataLock)
{
DailyReportMessage.AppendLine("Disk space available...");
foreach (var FolderMonitor in FolderMonitors)
{
string FreeSpaceText = "#Error";
string Drive = FolderMonitor.Key;
Int64 FreeSpace;
float FreePercent;
if (CrashReportCommon.StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
FreeSpaceText = GetDiskSpaceString(FreeSpace, FreePercent);
}
DailyReportMessage.AppendLine("> " + FolderMonitor.Value + " =>> " + FreeSpaceText);
}
}
return(DailyReportMessage.ToString());
}),
};
}