private void ProcessResourceDataList <T>(
IReadOnlyCollection <FabricResourceUsageData <T> > data,
T thresholdError,
T thresholdWarning)
where T : struct
{
foreach (var dataItem in data)
{
Token.ThrowIfCancellationRequested();
if (dataItem.Data.Count == 0 || dataItem.AverageDataValue <= 0)
{
continue;
}
if (CsvFileLogger != null && CsvFileLogger.EnableCsvLogging)
{
var fileName = "FabricSystemServices_" + NodeName;
var propertyName = data.First().Property;
// Log average data value to long-running store (CSV).
string dataLogMonitorType = propertyName;
switch (propertyName)
{
case ErrorWarningProperty.TotalMemoryConsumptionMb:
dataLogMonitorType = "Working Set %";
break;
case ErrorWarningProperty.TotalCpuTime:
dataLogMonitorType = "% CPU Time";
break;
}
CsvFileLogger.LogData(fileName, dataItem.Id, dataLogMonitorType, "Average", Math.Round(dataItem.AverageDataValue, 2));
CsvFileLogger.LogData(fileName, dataItem.Id, dataLogMonitorType, "Peak", Math.Round(Convert.ToDouble(dataItem.MaxDataValue)));
}
// This function will clear Data items in list (will call Clear() on the supplied FabricResourceUsageData instance's Data field..)
ProcessResourceDataReportHealth(
dataItem,
thresholdError,
thresholdWarning,
SetHealthReportTimeToLive(),
HealthReportType.Application);
}
}
public override Task ReportAsync(CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
if (CsvFileLogger != null && CsvFileLogger.EnableCsvLogging)
{
var fileName = "CpuMemFirewallsPorts" + NodeName;
// Log (csv) system-wide CPU/Mem data.
CsvFileLogger.LogData(
fileName,
NodeName,
"CPU Time",
"Average",
Math.Round(AllCpuTimeData.AverageDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"CPU Time",
"Peak",
Math.Round(AllCpuTimeData.MaxDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"Committed Memory (MB)",
"Average",
Math.Round(this.allMemDataCommittedBytes.AverageDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"Committed Memory (MB)",
"Peak",
Math.Round(this.allMemDataCommittedBytes.MaxDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"All Active Ports",
"Total",
this.activePortsData.Data[0]);
CsvFileLogger.LogData(
fileName,
NodeName,
"Ephemeral Active Ports",
"Total",
this.ephemeralPortsData.Data[0]);
CsvFileLogger.LogData(
fileName,
NodeName,
"Firewall Rules",
"Total",
this.firewallData.Data[0]);
DataTableFileLogger.Flush();
}
// Report on the global health state (system-wide (node) metrics).
// User-configurable in NodeObserver.config.json
var timeToLiveWarning = SetHealthReportTimeToLive();
// CPU
if (AllCpuTimeData.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
AllCpuTimeData,
CpuErrorUsageThresholdPct,
CpuWarningUsageThresholdPct,
timeToLiveWarning);
}
// Memory
if (this.allMemDataCommittedBytes.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
this.allMemDataCommittedBytes,
MemErrorUsageThresholdMb,
MemWarningUsageThresholdMb,
timeToLiveWarning);
}
if (this.allMemDataPercentUsed.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
this.allMemDataPercentUsed,
MemoryErrorLimitPercent,
MemoryWarningLimitPercent,
timeToLiveWarning);
}
// Firewall rules
ProcessResourceDataReportHealth(
this.firewallData,
FirewallRulesErrorThreshold,
FirewallRulesWarningThreshold,
timeToLiveWarning);
// Ports - Active TCP
ProcessResourceDataReportHealth(
this.activePortsData,
ActivePortsErrorThreshold,
ActivePortsWarningThreshold,
timeToLiveWarning);
// Ports - Active Ephemeral TCP
ProcessResourceDataReportHealth(
this.ephemeralPortsData,
EphemeralPortsErrorThreshold,
EphemeralPortsWarningThreshold,
timeToLiveWarning);
return(Task.CompletedTask);
}
catch (AggregateException e) when(e.InnerException is OperationCanceledException || e.InnerException is TaskCanceledException || e.InnerException is TimeoutException)
{
return(Task.CompletedTask);
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
$"Unhandled exception re-thrown:{Environment.NewLine}{e}");
throw;
}
}
