protected virtual void Dispose(bool disposing)
{
if (!this.hasDisposed)
{
if (disposing)
{
if (this.IsObserverRunning)
{
this.StopObservers();
}
if (this.globalShutdownEventHandle != null)
{
this.globalShutdownEventHandle.Dispose();
}
if (this.observers?.Count > 0)
{
foreach (var obs in this.observers)
{
obs?.Dispose();
}
this.observers.Clear();
}
if (FabricClientInstance != null)
{
FabricClientInstance.Dispose();
FabricClientInstance = null;
}
if (this.cts != null)
{
this.cts.Dispose();
this.cts = null;
}
// Flush and Dispose all NLog targets. No more logging.
Logger.Flush();
DataTableFileLogger.Flush();
Logger.ShutDown();
DataTableFileLogger.ShutDown();
this.hasDisposed = true;
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ObserverManager"/> class.
/// </summary>
public ObserverManager(
StatelessServiceContext context,
CancellationToken token)
{
this.token = token;
this.cts = new CancellationTokenSource();
this.token.Register(() => { this.ShutdownHandler(this, null); });
FabricClientInstance = new FabricClient();
FabricServiceContext = context;
this.nodeName = FabricServiceContext?.NodeContext.NodeName;
// Observer Logger setup...
string logFolderBasePath = null;
string observerLogPath = GetConfigSettingValue(
ObserverConstants.ObserverLogPath);
if (!string.IsNullOrEmpty(observerLogPath))
{
logFolderBasePath = observerLogPath;
}
else
{
string logFolderBase = $@"{Environment.CurrentDirectory}\observer_logs";
logFolderBasePath = logFolderBase;
}
// this logs metrics from observers, if enabled, and/or sends
// telemetry data to your implemented provider...
this.DataLogger = new DataTableFileLogger();
// this logs error/warning/info messages for ObserverManager...
this.Logger = new Logger("ObserverManager", logFolderBasePath);
this.HealthReporter = new ObserverHealthReporter(this.Logger);
this.SetPropertiesFromConfigurationParameters();
// Populate the Observer list for the sequential run loop...
this.observers = GetObservers();
}
private void LogAllAppResourceDataToCsv(string appName)
{
if (!this.CsvFileLogger.EnableCsvLogging && !this.IsTelemetryEnabled)
{
return;
}
var fileName = appName.Replace(":", string.Empty) +
this.NodeName;
// CPU Time
this.CsvFileLogger.LogData(
fileName,
appName,
"% CPU Time",
"Average",
Math.Round(this.allAppCpuData.Where(x => x.Id == appName)
.FirstOrDefault().AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
appName,
"% CPU Time",
"Peak",
Math.Round(Convert.ToDouble(this.allAppCpuData.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
// Memory
this.CsvFileLogger.LogData(
fileName,
appName,
"Memory (Working set) MB",
"Average",
Math.Round(this.allAppMemDataMB.Where(x => x.Id == appName)
.FirstOrDefault().AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
appName,
"Memory (Working set) MB",
"Peak",
Math.Round(Convert.ToDouble(this.allAppMemDataMB.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
this.CsvFileLogger.LogData(
fileName,
appName,
"Memory (Percent in use)",
"Average",
Math.Round(this.allAppMemDataPercent.Where(x => x.Id == appName)
.FirstOrDefault().AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
appName,
"Memory (Percent in use)",
"Peak",
Math.Round(Convert.ToDouble(this.allAppMemDataPercent.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
// IO Read Bytes/s
this.CsvFileLogger.LogData(
fileName,
appName,
"IO Read Bytes/sec",
"Average",
Math.Round(this.allAppDiskReadsData.Where(x => x.Id == appName)
.FirstOrDefault().AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
appName,
"IO Read Bytes/sec",
"Peak",
Math.Round(Convert.ToDouble(this.allAppDiskReadsData.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
// IO Write Bytes/s
this.CsvFileLogger.LogData(
fileName,
appName,
"IO Write Bytes/sec",
"Average",
Math.Round(this.allAppDiskWritesData.Where(x => x.Id == appName)
.FirstOrDefault().AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
appName,
"IO Write Bytes/sec",
"Peak",
Math.Round(Convert.ToDouble(this.allAppDiskWritesData.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
// Network
this.CsvFileLogger.LogData(
fileName,
appName,
"Active Ports",
"Total",
Math.Round(Convert.ToDouble(this.allAppTotalActivePortsData.Where(x => x.Id == appName)
.FirstOrDefault().MaxDataValue)));
DataTableFileLogger.Flush();
}
/// <inheritdoc/>
public override Task ReportAsync(CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
if (this.CsvFileLogger.EnableCsvLogging || this.IsTelemetryEnabled)
{
var fileName = "CpuMemFirewallsPorts" + this.NodeName;
// Log (csv) system-wide CPU/Mem data...
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"CPU Time",
"Average",
Math.Round(this.allCpuDataPrivTime.AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"CPU Time",
"Peak",
Math.Round(this.allCpuDataPrivTime.MaxDataValue));
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"Committed Memory (MB)",
"Average",
Math.Round(this.allMemDataCommittedBytes.AverageDataValue));
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"Committed Memory (MB)",
"Peak",
Math.Round(this.allMemDataCommittedBytes.MaxDataValue));
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"All Active Ports",
"Total",
this.activePortsData.Data[0]);
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"Ephemeral Active Ports",
"Total",
this.ephemeralPortsData.Data[0]);
this.CsvFileLogger.LogData(
fileName,
this.NodeName,
"Firewall Rules",
"Total",
this.firewallData.Data[0]);
DataTableFileLogger.Flush();
}
// Report on the global health state (system-wide (node) metrics).
// User-configurable in NodeObserver.config.json
var timeToLiveWarning = this.SetTimeToLiveWarning();
// CPU
if (this.allCpuDataPrivTime.AverageDataValue > 0)
{
this.ProcessResourceDataReportHealth(
this.allCpuDataPrivTime,
this.CpuErrorUsageThresholdPct,
this.CpuWarningUsageThresholdPct,
timeToLiveWarning);
}
// Memory
if (this.allMemDataCommittedBytes.AverageDataValue > 0)
{
this.ProcessResourceDataReportHealth(
this.allMemDataCommittedBytes,
this.MemErrorUsageThresholdMB,
this.MemWarningUsageThresholdMB,
timeToLiveWarning);
}
if (this.allMemDataPercentUsed.AverageDataValue > 0)
{
this.ProcessResourceDataReportHealth(
this.allMemDataPercentUsed,
this.MemoryErrorLimitPercent,
this.MemoryWarningLimitPercent,
timeToLiveWarning);
}
// Firewall rules
this.ProcessResourceDataReportHealth(
this.firewallData,
this.FirewallRulesErrorThreshold,
this.FirewallRulesWarningThreshold,
timeToLiveWarning);
// Ports
this.ProcessResourceDataReportHealth(
this.activePortsData,
this.ActivePortsErrorThreshold,
this.ActivePortsWarningThreshold,
timeToLiveWarning);
this.ProcessResourceDataReportHealth(
this.ephemeralPortsData,
this.EphemeralPortsErrorThreshold,
this.EphemeralPortsWarningThreshold,
timeToLiveWarning);
return(Task.FromResult(1));
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
e.ToString());
}
throw;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ObserverManager"/> class.
/// </summary>
public ObserverManager(
StatelessServiceContext context,
CancellationToken token)
{
this.token = token;
this.cts = new CancellationTokenSource();
this.token.Register(() => { this.ShutdownHandler(this, null); });
FabricClientInstance = new FabricClient();
FabricServiceContext = context;
this.nodeName = FabricServiceContext?.NodeContext.NodeName;
// Observer Logger setup.
string logFolderBasePath = null;
string observerLogPath = GetConfigSettingValue(
ObserverConstants.ObserverLogPath);
if (!string.IsNullOrEmpty(observerLogPath))
{
logFolderBasePath = observerLogPath;
}
else
{
string logFolderBase = $@"{Environment.CurrentDirectory}\observer_logs";
logFolderBasePath = logFolderBase;
}
// this logs metrics from observers, if enabled, and/or sends
// telemetry data to your implemented provider.
this.DataLogger = new DataTableFileLogger();
// this logs error/warning/info messages for ObserverManager.
this.Logger = new Logger("ObserverManager", logFolderBasePath);
this.HealthReporter = new ObserverHealthReporter(this.Logger);
this.SetPropertiesFromConfigurationParameters();
// Populate the Observer list for the sequential run loop.
this.observers = GetObservers();
// FabricObserver Internal Diagnostic Telemetry (Non-PII).
// Internally, TelemetryEvents determines current Cluster Id as a unique identifier for transmitted events.
if (FabricObserverInternalTelemetryEnabled)
{
string codePkgVersion = FabricServiceContext.CodePackageActivationContext.CodePackageVersion;
string serviceManifestVersion = FabricServiceContext.CodePackageActivationContext.GetConfigurationPackageObject("Config").Description.ServiceManifestVersion;
string filepath = Path.Combine(logFolderBasePath, $"fo_telemetry_sent_{codePkgVersion.Replace(".", string.Empty)}_{serviceManifestVersion.Replace(".", string.Empty)}_{FabricServiceContext.NodeContext.NodeType}.txt");
#if !DEBUG
// If this has already been sent for this activated version (code/config) of nodetype x
if (File.Exists(filepath))
{
return;
}
#endif
this.telemetryEvents = new TelemetryEvents(
FabricClientInstance,
FabricServiceContext,
ServiceEventSource.Current,
this.token);
if (this.telemetryEvents.FabricObserverRuntimeNodeEvent(
codePkgVersion,
this.GetFabricObserverInternalConfiguration(),
"HealthState.Initialized"))
{
// Log a file to prevent re-sending this in case of process restart(s).
// This non-PII FO/Cluster info is versioned and should only be sent once per deployment (config or code updates.).
_ = this.Logger.TryWriteLogFile(filepath, "_");
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ObserverBase"/> class.
/// </summary>
protected ObserverBase()
{
ObserverName = GetType().Name;
ConfigurationSectionName = ObserverName + "Configuration";
FabricClientInstance = ObserverManager.FabricClientInstance;
if (IsTelemetryProviderEnabled)
{
TelemetryClient = ObserverManager.TelemetryClient;
}
FabricServiceContext = ObserverManager.FabricServiceContext;
NodeName = FabricServiceContext.NodeContext.NodeName;
NodeType = FabricServiceContext.NodeContext.NodeType;
ConfigurationSectionName = ObserverName + "Configuration";
// Observer Logger setup.
string logFolderBasePath;
string observerLogPath = GetSettingParameterValue(
ObserverConstants.ObserverManagerConfigurationSectionName,
ObserverConstants.ObserverLogPathParameter);
if (!string.IsNullOrEmpty(observerLogPath))
{
logFolderBasePath = observerLogPath;
}
else
{
string logFolderBase = Path.Combine(Environment.CurrentDirectory, "observer_logs");
logFolderBasePath = logFolderBase;
}
ObserverLogger = new Logger(ObserverName, logFolderBasePath);
if (string.IsNullOrEmpty(this.dumpsPath))
{
SetDefaultSfDumpPath();
}
// DataLogger setup
if (bool.TryParse(
GetSettingParameterValue(
ConfigurationSectionName,
ObserverConstants.EnableLongRunningCsvLogging),
out bool enableDataLogging))
{
if (enableDataLogging)
{
CsvFileLogger = new DataTableFileLogger
{
EnableCsvLogging = enableDataLogging,
};
string dataLogPath = GetSettingParameterValue(
ObserverConstants.ObserverManagerConfigurationSectionName,
ObserverConstants.DataLogPathParameter);
if (!string.IsNullOrEmpty(dataLogPath))
{
CsvFileLogger.DataLogFolderPath = dataLogPath;
}
else
{
CsvFileLogger.DataLogFolderPath = Path.Combine(Environment.CurrentDirectory, "observer_data_logs");
}
}
}
if (string.IsNullOrEmpty(this.dumpsPath))
{
SetDefaultSfDumpPath();
}
if (!IsTestRun)
{
ConfigurationSettings = new Utilities.ConfigSettings(
FabricServiceContext.CodePackageActivationContext.GetConfigurationPackageObject("Config").Settings,
ConfigurationSectionName);
EnableVerboseLogging = ConfigurationSettings.EnableVerboseLogging;
IsEnabled = ConfigurationSettings.IsEnabled;
IsObserverTelemetryEnabled = ConfigurationSettings.IsObserverTelemetryEnabled;
MonitorDuration = ConfigurationSettings.MonitorDuration;
RunInterval = ConfigurationSettings.RunInterval;
ObserverLogger.EnableVerboseLogging = EnableVerboseLogging;
DataCapacity = ConfigurationSettings.DataCapacity;
UseCircularBuffer = ConfigurationSettings.UseCircularBuffer;
}
HealthReporter = new ObserverHealthReporter(ObserverLogger);
}
public override Task ReportAsync(CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
if (CsvFileLogger != null && CsvFileLogger.EnableCsvLogging)
{
var fileName = "CpuMemFirewallsPorts" + NodeName;
// Log (csv) system-wide CPU/Mem data.
CsvFileLogger.LogData(
fileName,
NodeName,
"CPU Time",
"Average",
Math.Round(AllCpuTimeData.AverageDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"CPU Time",
"Peak",
Math.Round(AllCpuTimeData.MaxDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"Committed Memory (MB)",
"Average",
Math.Round(this.allMemDataCommittedBytes.AverageDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"Committed Memory (MB)",
"Peak",
Math.Round(this.allMemDataCommittedBytes.MaxDataValue));
CsvFileLogger.LogData(
fileName,
NodeName,
"All Active Ports",
"Total",
this.activePortsData.Data[0]);
CsvFileLogger.LogData(
fileName,
NodeName,
"Ephemeral Active Ports",
"Total",
this.ephemeralPortsData.Data[0]);
CsvFileLogger.LogData(
fileName,
NodeName,
"Firewall Rules",
"Total",
this.firewallData.Data[0]);
DataTableFileLogger.Flush();
}
// Report on the global health state (system-wide (node) metrics).
// User-configurable in NodeObserver.config.json
var timeToLiveWarning = SetHealthReportTimeToLive();
// CPU
if (AllCpuTimeData.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
AllCpuTimeData,
CpuErrorUsageThresholdPct,
CpuWarningUsageThresholdPct,
timeToLiveWarning);
}
// Memory
if (this.allMemDataCommittedBytes.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
this.allMemDataCommittedBytes,
MemErrorUsageThresholdMb,
MemWarningUsageThresholdMb,
timeToLiveWarning);
}
if (this.allMemDataPercentUsed.AverageDataValue > 0)
{
ProcessResourceDataReportHealth(
this.allMemDataPercentUsed,
MemoryErrorLimitPercent,
MemoryWarningLimitPercent,
timeToLiveWarning);
}
// Firewall rules
ProcessResourceDataReportHealth(
this.firewallData,
FirewallRulesErrorThreshold,
FirewallRulesWarningThreshold,
timeToLiveWarning);
// Ports - Active TCP
ProcessResourceDataReportHealth(
this.activePortsData,
ActivePortsErrorThreshold,
ActivePortsWarningThreshold,
timeToLiveWarning);
// Ports - Active Ephemeral TCP
ProcessResourceDataReportHealth(
this.ephemeralPortsData,
EphemeralPortsErrorThreshold,
EphemeralPortsWarningThreshold,
timeToLiveWarning);
return(Task.CompletedTask);
}
catch (AggregateException e) when(e.InnerException is OperationCanceledException || e.InnerException is TaskCanceledException || e.InnerException is TimeoutException)
{
return(Task.CompletedTask);
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
$"Unhandled exception re-thrown:{Environment.NewLine}{e}");
throw;
}
}
