protected override void Dispose(bool disposing)
{
if (!disposing)
{
return;
}
var errWarnHealthStates = this.connectionStatus.Where(
conn => conn.Health == HealthState.Error || conn.Health == HealthState.Warning);
foreach (var state in errWarnHealthStates)
{
// Clear existing Health Warning.
var report = new HealthReport
{
AppName = new Uri(state.TargetApp),
Code = FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable,
EmitLogEvent = true,
HealthMessage = $"Clearing NetworkObserver's Health Error/Warning for {state.TargetApp}/{state.HostName} connection state since FO is stopping.",
HealthReportTimeToLive = default(TimeSpan),
State = HealthState.Ok,
NodeName = NodeName,
Observer = ObserverName,
Property = $"EndpointUnreachable({state.HostName})",
ReportType = HealthReportType.Application,
};
HealthReporter.ReportHealthToServiceFabric(report);
}
}
public override Task ReportAsync(CancellationToken token)
{
Token.ThrowIfCancellationRequested();
// Informational report. For now, Linux is where we pay close attention to memory use by Fabric system services as there are still a few issues in that realm..
var timeToLiveWarning = SetHealthReportTimeToLive();
var portInformationReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = $"Number of ports in use by Fabric services: {TotalActivePortCountAllSystemServices}{Environment.NewLine}" +
$"Number of ephemeral ports in use by Fabric services: {TotalActiveEphemeralPortCountAllSystemServices}{Environment.NewLine}" +
$"Fabric memory use MB: {this.allMemData.Where(x => x.Id == "Fabric")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" +
(RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ?
$"FabricGateway memory use MB: {this.allMemData.Where(x => x.Id == "FabricGateway.exe")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" +
$"FabricHost memory use MB: {this.allMemData.Where(x => x.Id == "FabricHost")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" : string.Empty),
State = HealthState.Ok,
HealthReportTimeToLive = timeToLiveWarning,
};
HealthReporter.ReportHealthToServiceFabric(portInformationReport);
// Reset ports counters.
TotalActivePortCountAllSystemServices = 0;
TotalActiveEphemeralPortCountAllSystemServices = 0;
// CPU
ProcessResourceDataList(
this.allCpuData,
CpuErrorUsageThresholdPct,
CpuWarnUsageThresholdPct);
// Memory
ProcessResourceDataList(
this.allMemData,
MemErrorUsageThresholdMb,
MemWarnUsageThresholdMb);
// Ports - Active TCP
ProcessResourceDataList(
this.allActiveTcpPortData,
ActiveTcpPortCountError,
ActiveTcpPortCountWarning);
// Ports - Ephemeral
ProcessResourceDataList(
this.allEphemeralTcpPortData,
ActiveEphemeralPortCountError,
ActiveEphemeralPortCountWarning);
// Windows Event Log
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && ObserverManager.ObserverWebAppDeployed &&
this.monitorWinEventLog)
{
// SF Eventlog Errors?
// Write this out to a new file, for use by the web front end log viewer.
// Format = HTML.
int count = this.evtRecordList.Count();
var logPath = Path.Combine(ObserverLogger.LogFolderBasePath, "EventVwrErrors.txt");
// Remove existing file.
if (File.Exists(logPath))
{
try
{
File.Delete(logPath);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
if (count >= 10)
{
var sb = new StringBuilder();
_ = sb.AppendLine("<br/><div><strong>" +
"<a href='javascript:toggle(\"evtContainer\")'>" +
"<div id=\"plus\" style=\"display: inline; font-size: 25px;\">+</div> " + count +
" Error Events in ServiceFabric and System</a> " +
"Event logs</strong>.<br/></div>");
_ = sb.AppendLine("<div id='evtContainer' style=\"display: none;\">");
foreach (var evt in this.evtRecordList.Distinct())
{
token.ThrowIfCancellationRequested();
try
{
// Access event properties:
_ = sb.AppendLine("<div>" + evt.LogName + "</div>");
_ = sb.AppendLine("<div>" + evt.LevelDisplayName + "</div>");
if (evt.TimeCreated.HasValue)
{
_ = sb.AppendLine("<div>" + evt.TimeCreated.Value.ToShortDateString() + "</div>");
}
foreach (var prop in evt.Properties)
{
if (prop.Value != null && Convert.ToString(prop.Value).Length > 0)
{
_ = sb.AppendLine("<div>" + prop.Value + "</div>");
}
}
}
catch (EventLogException)
{
}
}
_ = sb.AppendLine("</div>");
_ = ObserverLogger.TryWriteLogFile(logPath, sb.ToString());
_ = sb.Clear();
}
// Clean up.
if (count > 0)
{
this.evtRecordList.Clear();
}
}
ClearDataContainers();
return(Task.CompletedTask);
}
public override Task ReportAsync(CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
// OS Health.
if (this.osStatus != null && !string.Equals(this.osStatus, "OK", StringComparison.OrdinalIgnoreCase))
{
string healthMessage = $"OS reporting unhealthy: {this.osStatus}";
var healthReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = healthMessage,
State = HealthState.Error,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
// This means this observer created a Warning or Error SF Health Report
HasActiveFabricErrorOrWarning = true;
// Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example.).
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
_ = TelemetryClient?.ReportHealthAsync(
HealthScope.Application,
FabricRuntime.GetActivationContext().ApplicationName,
HealthState.Error,
$"{NodeName} - OS reporting unhealthy: {this.osStatus}",
ObserverName,
Token);
}
}
else if (HasActiveFabricErrorOrWarning && string.Equals(this.osStatus, "OK", StringComparison.OrdinalIgnoreCase))
{
// Clear Error or Warning with an OK Health Report.
string healthMessage = $"OS reporting healthy: {this.osStatus}";
var healthReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = healthMessage,
State = HealthState.Ok,
HealthReportTimeToLive = default(TimeSpan),
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
// Reset internal health state.
HasActiveFabricErrorOrWarning = false;
}
if (ObserverManager.ObserverWebAppDeployed)
{
var logPath = Path.Combine(ObserverLogger.LogFolderBasePath, "SysInfo.txt");
// This file is used by the web application (log reader.).
if (!ObserverLogger.TryWriteLogFile(logPath, $"Last updated on {DateTime.UtcNow.ToString("M/d/yyyy HH:mm:ss")} UTC<br/>{this.osReport}"))
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
"Unable to create SysInfo.txt file.");
}
}
var report = new HealthReport
{
Observer = ObserverName,
HealthMessage = this.osReport,
State = HealthState.Ok,
NodeName = NodeName,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(report);
// Windows Update automatic download enabled?
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
this.isWindowsUpdateAutoDownloadEnabled)
{
string linkText =
$"{Environment.NewLine}For clusters of Silver durability or above, " +
$"please consider <a href=\"https://docs.microsoft.com/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-automatic-upgrade\" target=\"blank\">" +
$"enabling VMSS automatic OS image upgrades</a> to prevent unexpected VM reboots. " +
$"For Bronze durability clusters, please consider deploying the " +
$"<a href=\"https://docs.microsoft.com/azure/service-fabric/service-fabric-patch-orchestration-application\" target=\"blank\">Patch Orchestration Service</a>.";
string auServiceEnabledMessage = $"Windows Update Automatic Download is enabled.{linkText}";
report = new HealthReport
{
Observer = ObserverName,
Property = "OSConfiguration",
HealthMessage = auServiceEnabledMessage,
State = HealthState.Warning,
NodeName = NodeName,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(report);
if (IsTelemetryProviderEnabled &&
IsObserverTelemetryEnabled &&
RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example.).
var telemetryData = new TelemetryData(FabricClientInstance, token)
{
HealthEventDescription = auServiceEnabledMessage,
HealthState = "Warning",
Metric = "WUAutoDownloadEnabled",
Value = this.isWindowsUpdateAutoDownloadEnabled,
NodeName = NodeName,
ObserverName = ObserverName,
Source = ObserverConstants.FabricObserverName,
};
_ = TelemetryClient?.ReportMetricAsync(
telemetryData,
Token);
}
// ETW.
if (IsEtwEnabled && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
HealthState = "Warning",
HealthEventDescription = auServiceEnabledMessage,
ObserverName,
Metric = "WUAutoDownloadEnabled",
Value = this.isWindowsUpdateAutoDownloadEnabled,
NodeName,
});
}
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// reset au globals for fresh detection during next observer run.
this.isWindowsUpdateAutoDownloadEnabled = false;
this.auStateUnknown = false;
this.isWUADSettingEnabled = false;
}
return(Task.CompletedTask);
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Error,
$"Unhandled exception processing OS information:{Environment.NewLine}{e}");
throw;
}
}
public override Task ReportAsync(CancellationToken token)
{
var timeToLiveWarning = SetHealthReportTimeToLive();
// Report on connection state.
foreach (var config in this.userConfig)
{
token.ThrowIfCancellationRequested();
foreach (var conn in this.connectionStatus.Where(cs => cs.TargetApp == config.TargetApp))
{
token.ThrowIfCancellationRequested();
var connState = conn;
if (!connState.Connected)
{
this.healthState = HealthState.Warning;
var healthMessage = $"Outbound Internet connection failure detected for endpoint {connState.HostName}{Environment.NewLine}";
// Send Health Telemetry (perhaps it signals an Alert in AppInsights or LogAnalytics).
// This will also be serialied into the health event (Desf.
var telemetryData = new TelemetryData(FabricClientInstance, token)
{
ApplicationName = conn.TargetApp,
Code = FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable,
HealthState = "Warning",
HealthEventDescription = healthMessage,
ObserverName = ObserverName,
Metric = ErrorWarningProperty.InternetConnectionFailure,
NodeName = NodeName,
};
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
_ = TelemetryClient?.ReportMetricAsync(
telemetryData,
Token);
}
var report = new HealthReport
{
AppName = new Uri(conn.TargetApp),
EmitLogEvent = true,
HealthData = telemetryData,
HealthMessage = healthMessage,
HealthReportTimeToLive = timeToLiveWarning,
State = this.healthState,
NodeName = NodeName,
Observer = ObserverName,
Property = $"EndpointUnreachable({conn.HostName})",
ReportType = HealthReportType.Application,
ResourceUsageDataProperty = $"{ErrorWarningProperty.InternetConnectionFailure}: {connState.HostName}",
};
// Send health report Warning and log event locally.
HealthReporter.ReportHealthToServiceFabric(report);
// This means this observer created a Warning or Error SF Health Report
HasActiveFabricErrorOrWarning = true;
// ETW.
if (IsEtwEnabled)
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
ApplicationName = conn.TargetApp,
Code = FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable,
HealthState = "Warning",
HealthEventDescription = healthMessage,
ObserverName,
Metric = ErrorWarningProperty.InternetConnectionFailure,
NodeName,
});
}
}
else
{
if (connState.Health != HealthState.Warning ||
connState.Health != HealthState.Error)
{
continue;
}
this.healthState = HealthState.Ok;
var healthMessage = $"Outbound Internet connection successful for {connState?.HostName} from node {NodeName}.";
// Clear existing Health Warning.
var report = new HealthReport
{
AppName = new Uri(conn.TargetApp),
Code = FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable,
EmitLogEvent = true,
HealthMessage = healthMessage,
HealthReportTimeToLive = default(TimeSpan),
State = HealthState.Ok,
NodeName = NodeName,
Observer = ObserverName,
Property = $"EndpointUnreachable({conn.HostName})",
ReportType = HealthReportType.Application,
};
HealthReporter.ReportHealthToServiceFabric(report);
// Telemetry.
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
var telemetryData = new TelemetryData(FabricClientInstance, token)
{
ApplicationName = conn.TargetApp,
Code = FOErrorWarningCodes.Ok,
HealthState = "Ok",
HealthEventDescription = healthMessage,
ObserverName = ObserverName,
Metric = "Internet Connection State",
NodeName = NodeName,
};
_ = TelemetryClient?.ReportMetricAsync(
telemetryData,
Token);
}
// ETW.
if (IsEtwEnabled)
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
ApplicationName = conn.TargetApp,
Code = FOErrorWarningCodes.Ok,
HealthState = "Ok",
HealthEventDescription = healthMessage,
ObserverName,
Metric = "Internet Connection State",
NodeName,
});
}
// Reset health state.
HasActiveFabricErrorOrWarning = false;
}
}
}
// Clear
_ = this.connectionStatus.RemoveAll(conn => conn.Connected);
this.connectionStatus.TrimExcess();
this.connEndpointTestResults.Clear();
return(Task.CompletedTask);
}
private async Task MonitorDeployedAppsAsync(CancellationToken token)
{
Process currentProcess = null;
foreach (var repOrInst in ReplicaOrInstanceList)
{
token.ThrowIfCancellationRequested();
var timer = new Stopwatch();
int processId = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
bool checkCpu = false, checkMemMb = false, checkMemPct = false, checkAllPorts = false, checkEphemeralPorts = false;
var application = this.deployedTargetList?.FirstOrDefault(
app => app?.TargetApp?.ToLower() == repOrInst.ApplicationName?.OriginalString?.ToLower() ||
app?.TargetAppType?.ToLower() == repOrInst.ApplicationTypeName?.ToLower());
if (application?.TargetApp == null && application?.TargetAppType == null)
{
continue;
}
try
{
// App level.
currentProcess = Process.GetProcessById(processId);
token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process where the metric is specified in configuration for related observation.
if (this.AllAppCpuData.All(list => list.Id != id) && (application.CpuErrorLimitPercent > 0 || application.CpuWarningLimitPercent > 0))
{
this.AllAppCpuData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalCpuTime, id, DataCapacity, UseCircularBuffer));
}
if (this.AllAppCpuData.Any(list => list.Id == id))
{
checkCpu = true;
}
if (this.AllAppMemDataMb.All(list => list.Id != id) && (application.MemoryErrorLimitMb > 0 || application.MemoryWarningLimitMb > 0))
{
this.AllAppMemDataMb.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.TotalMemoryConsumptionMb, id, DataCapacity, UseCircularBuffer));
}
if (this.AllAppMemDataMb.Any(list => list.Id == id))
{
checkMemMb = true;
}
if (this.AllAppMemDataPercent.All(list => list.Id != id) && (application.MemoryErrorLimitPercent > 0 || application.MemoryWarningLimitPercent > 0))
{
this.AllAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id, DataCapacity, UseCircularBuffer));
}
if (this.AllAppMemDataPercent.Any(list => list.Id == id))
{
checkMemPct = true;
}
if (this.AllAppTotalActivePortsData.All(list => list.Id != id) && (application.NetworkErrorActivePorts > 0 || application.NetworkWarningActivePorts > 0))
{
this.AllAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id, 1));
}
if (this.AllAppTotalActivePortsData.Any(list => list.Id == id))
{
checkAllPorts = true;
}
if (this.AllAppEphemeralPortsData.All(list => list.Id != id) && (application.NetworkErrorEphemeralPorts > 0 || application.NetworkWarningEphemeralPorts > 0))
{
this.AllAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id, 1));
}
if (this.AllAppEphemeralPortsData.Any(list => list.Id == id))
{
checkEphemeralPorts = true;
}
// Measure Total and Ephemeral ports.
if (checkAllPorts)
{
this.AllAppTotalActivePortsData.FirstOrDefault(x => x.Id == id).Data.Add(OperatingSystemInfoProvider.Instance.GetActivePortCount(currentProcess.Id, FabricServiceContext));
}
if (checkEphemeralPorts)
{
this.AllAppEphemeralPortsData.FirstOrDefault(x => x.Id == id).Data.Add(OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount(currentProcess.Id, FabricServiceContext));
}
// No need to proceed further if no cpu and mem thresholds are specified in configuration.
if (!checkCpu && !checkMemMb && !checkMemPct)
{
continue;
}
/* CPU and Memory Usage */
TimeSpan duration = TimeSpan.FromSeconds(15);
if (MonitorDuration > TimeSpan.MinValue)
{
duration = MonitorDuration;
}
// Warm up the counters.
if (checkCpu)
{
_ = cpuUsage.GetCpuUsagePercentageProcess(currentProcess);
}
if (checkMemMb || checkMemPct)
{
_ = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(currentProcess.Id);
}
timer.Start();
while (!currentProcess.HasExited && timer.Elapsed.Seconds <= duration.Seconds)
{
token.ThrowIfCancellationRequested();
if (checkCpu)
{
// CPU (all cores).
double cpu = cpuUsage.GetCpuUsagePercentageProcess(currentProcess);
if (cpu >= 0)
{
if (cpu > 100)
{
cpu = 100;
}
this.AllAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
}
float processMem = 0;
if (checkMemMb || checkMemPct)
{
processMem = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(currentProcess.Id);
}
if (checkMemMb)
{
// Memory (private working set (process)).
this.AllAppMemDataMb.FirstOrDefault(x => x.Id == id).Data.Add(processMem);
}
if (checkMemPct)
{
// Memory (percent in use (total)).
var(TotalMemory, PercentInUse) = OperatingSystemInfoProvider.Instance.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = TotalMemory;
if (totalMem > 0)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
this.AllAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(Math.Round(usedPct, 1));
}
}
await Task.Delay(250, Token);
}
timer.Stop();
timer.Reset();
}
catch (Exception e)
{
#if DEBUG
// DEBUG INFO
var healthReport = new Utilities.HealthReport
{
AppName = repOrInst.ApplicationName,
HealthMessage = $"Error:{Environment.NewLine}{e}{Environment.NewLine}",
State = HealthState.Ok,
Code = FOErrorWarningCodes.Ok,
NodeName = NodeName,
Observer = ObserverName,
Property = $"{e.Source}",
ReportType = HealthReportType.Application,
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
#endif
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
WriteToLogWithLevel(
ObserverName,
$"MonitorAsync failed to find current service process for {repOrInst.ApplicationName?.OriginalString ?? repOrInst.ApplicationTypeName}{Environment.NewLine}{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException || e is TaskCanceledException))
{
WriteToLogWithLevel(
ObserverName,
$"Unhandled exception in MonitorAsync:{Environment.NewLine}{e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
private async Task MonitorDeployedAppsAsync(CancellationToken token)
{
Process currentProcess = null;
foreach (var repOrInst in ReplicaOrInstanceList)
{
token.ThrowIfCancellationRequested();
var timer = new Stopwatch();
int processId = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level.
currentProcess = Process.GetProcessById(processId);
token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process.
if (this.allAppCpuData.All(list => list.Id != id))
{
this.allAppCpuData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalCpuTime, id, DataCapacity, UseCircularBuffer));
this.allAppMemDataMb.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.TotalMemoryConsumptionMb, id, DataCapacity, UseCircularBuffer));
this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id, DataCapacity, UseCircularBuffer));
this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id, 1));
this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id, 1));
}
TimeSpan duration = TimeSpan.FromSeconds(15);
if (MonitorDuration > TimeSpan.MinValue)
{
duration = MonitorDuration;
}
// Warm up the counters.
_ = cpuUsage.GetCpuUsagePercentageProcess(currentProcess);
_ = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(currentProcess.Id);
timer.Start();
while (!currentProcess.HasExited && timer.Elapsed.Seconds <= duration.Seconds)
{
token.ThrowIfCancellationRequested();
// CPU (all cores).
double cpu = cpuUsage.GetCpuUsagePercentageProcess(currentProcess);
if (cpu >= 0)
{
if (cpu > 100)
{
cpu = 100;
}
this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process)).
var processMem = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(currentProcess.Id);
this.allAppMemDataMb.FirstOrDefault(x => x.Id == id).Data.Add(processMem);
// Memory (percent in use (total)).
var(TotalMemory, PercentInUse) = OperatingSystemInfoProvider.Instance.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = TotalMemory;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(Math.Round(usedPct, 1));
}
await Task.Delay(250, Token);
}
timer.Stop();
timer.Reset();
// Total and Ephemeral ports..
this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(OperatingSystemInfoProvider.Instance.GetActivePortCount(currentProcess.Id, FabricServiceContext));
this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount(currentProcess.Id, FabricServiceContext));
}
catch (Exception e)
{
#if DEBUG
// DEBUG INFO
var healthReport = new Utilities.HealthReport
{
AppName = repOrInst.ApplicationName,
HealthMessage = $"Error: {e}\n\n",
State = HealthState.Ok,
Code = FOErrorWarningCodes.Ok,
NodeName = NodeName,
Observer = ObserverName,
Property = $"{e.Source}",
ReportType = HealthReportType.Application,
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
#endif
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
WriteToLogWithLevel(
ObserverName,
$"MonitorAsync failed to find current service process for {repOrInst.ApplicationName?.OriginalString ?? repOrInst.ApplicationTypeName}/n{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException || e is TaskCanceledException))
{
WriteToLogWithLevel(
ObserverName,
$"Unhandled exception in MonitorAsync: \n {e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
public override Task ReportAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
// Someone calling without observing first, must be run after a new run of ObserveAsync
if (ExpiringWarnings == null ||
ExpiredWarnings == null ||
NotFoundWarnings == null)
{
return(Task.CompletedTask);
}
HealthReport healthReport;
if (ExpiringWarnings.Count == 0 &&
ExpiredWarnings.Count == 0 &&
NotFoundWarnings.Count == 0)
{
healthReport = new HealthReport
{
Observer = ObserverName,
ReportType = HealthReportType.Node,
EmitLogEvent = true,
NodeName = NodeName,
HealthMessage = $"All cluster and monitored app certificates are healthy.",
State = HealthState.Ok,
HealthReportTimeToLive = RunInterval > TimeSpan.MinValue ? RunInterval : HealthReportTimeToLive,
};
HasActiveFabricErrorOrWarning = false;
}
else
{
string healthMessage = (ExpiredWarnings.Count == 0 ? string.Empty : (ExpiredWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j) + "\n")) +
(NotFoundWarnings.Count == 0 ? string.Empty : (NotFoundWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j) + "\n")) +
(ExpiringWarnings.Count == 0 ? string.Empty : ExpiringWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j));
healthReport = new HealthReport
{
Code = FOErrorWarningCodes.WarningCertificateExpiration,
Observer = ObserverName,
ReportType = HealthReportType.Node,
EmitLogEvent = true,
NodeName = NodeName,
HealthMessage = healthMessage,
State = HealthState.Warning,
HealthReportTimeToLive = RunInterval > TimeSpan.MinValue ? RunInterval : HealthReportTimeToLive,
};
HasActiveFabricErrorOrWarning = true;
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
TelemetryData telemetryData = new TelemetryData(FabricClientInstance, token)
{
Code = FOErrorWarningCodes.WarningCertificateExpiration,
HealthState = "Warning",
NodeName = NodeName,
Metric = ErrorWarningProperty.CertificateExpiration,
HealthEventDescription = healthMessage,
ObserverName = ObserverName,
OS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Windows" : "Linux",
Source = ObserverConstants.FabricObserverName,
Value = FOErrorWarningCodes.GetErrorWarningNameFromFOCode(FOErrorWarningCodes.WarningCertificateExpiration),
};
_ = TelemetryClient?.ReportMetricAsync(
telemetryData,
Token);
}
if (IsEtwEnabled)
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
Code = FOErrorWarningCodes.WarningCertificateExpiration,
HealthState = "Warning",
NodeName,
Metric = ErrorWarningProperty.CertificateExpiration,
HealthEventDescription = healthMessage,
ObserverName,
OS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Windows" : "Linux",
Source = ObserverConstants.FabricObserverName,
Value = FOErrorWarningCodes.GetErrorWarningNameFromFOCode(FOErrorWarningCodes.WarningCertificateExpiration),
});
}
}
HealthReporter.ReportHealthToServiceFabric(healthReport);
return(Task.CompletedTask);
}