private async Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
await Task.Run(
() =>
{
token.ThrowIfCancellationRequested();
try
{
// Ports.
int activePortCountTotal = NetworkUsage.GetActivePortCount();
int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
for (int i = 0; i < 30; i++)
{
token.ThrowIfCancellationRequested();
if (this.CpuWarningUsageThresholdPct > 0 &&
this.CpuWarningUsageThresholdPct <= 100)
{
this.allCpuDataPrivTime.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
}
if (this.MemWarningUsageThresholdMb > 0)
{
this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
}
if (this.MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(
ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
}
Thread.Sleep(250);
}
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync: {e.Message}: \n {e.StackTrace}");
}
throw;
}
}, token).ConfigureAwait(true);
}
private string PrintNetworkUsage(NetworkUsage networkUsage, DateTimeOffset startTime)
{
string result = "Usage from " + startTime.ToString() + " to " + (startTime + networkUsage.ConnectionDuration).ToString() +
"\n\tBytes sent: " + networkUsage.BytesSent +
"\n\tBytes received: " + networkUsage.BytesReceived + "\n";
return(result);
}
private NetworkInterfaceUsageInfo CreateInfo(NetworkUsage usage)
{
var result = usage is TeamingNetworkUsage teamingNetworkUsage
? new TeamingInterfaceUsageInfo {
ChildInterfaces = teamingNetworkUsage.ChildInterfaces
}
: new NetworkInterfaceUsageInfo();
result.InterfaceName = usage.InterfaceName;
return(result);
}
public object GetResource()
{
var currentNetworkUsage = GetTotalUsage();
var usageOfPastMinute = new NetworkUsage(
currentNetworkUsage.DownloadInBytes - _lastNetworkUsage.DownloadInBytes,
currentNetworkUsage.UploadInBytes - _lastNetworkUsage.UploadInBytes
);
this._lastNetworkUsage = currentNetworkUsage;
return(usageOfPastMinute);
}
private void FillInfo(NetworkInterfaceUsageInfo toFill, NetworkUsage previousUsage, NetworkUsage usage, double deltaSeconds)
{
var deltaReceivedBytes = usage.ReceivedBytes - previousUsage.ReceivedBytes;
var deltaSentBytes = usage.SentBytes - previousUsage.SentBytes;
if (deltaSeconds > 0d)
{
toFill.ReceivedBytesPerSecond = (long)(deltaReceivedBytes / deltaSeconds);
toFill.SentBytesPerSecond = (long)(deltaSentBytes / deltaSeconds);
}
toFill.BandwidthBytesPerSecond = usage.NetworkMaxMBitsPerSecond * 1000L * 1000L / 8L;
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
var repOrInstList = await this.GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.Target)).ConfigureAwait(true);
if (repOrInstList.Count == 0)
{
return;
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
this.Token.ThrowIfCancellationRequested();
int processid = (int)repOrInst.ReplicaHostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level...
currentProcess = Process.GetProcessById(processid);
this.Token.ThrowIfCancellationRequested();
if (currentProcess == null)
{
continue;
}
var procName = currentProcess.ProcessName;
// Add new resource data structures for each app service process...
var id = $"{application.Target.Replace("fabric:/", string.Empty)}:{procName}";
if (!this.allAppCpuData.Any(list => list.Id == id))
{
this.allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id));
this.allAppMemDataMB.Add(new FabricResourceUsageData <long>(ErrorWarningProperty.TotalMemoryConsumptionMB, id));
this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id));
this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id));
this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id));
}
// CPU (all cores)...
int i = Environment.ProcessorCount + 10;
while (!currentProcess.HasExited && i > 0)
{
this.Token.ThrowIfCancellationRequested();
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process))...
var mem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMB(currentProcess.ProcessName);
this.allAppMemDataMB.FirstOrDefault(x => x.Id == id).Data.Add((long)mem);
// Memory (percent in use (total))...
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.Item1;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(mem * 100)) / (totalMem * 1024), 2);
this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(usedPct);
}
--i;
Thread.Sleep(250);
}
// Total and Ephemeral ports....
this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"MonitorAsync failed to find current service process for {application.Target}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Unhandled exception in MonitorAsync: \n {e.ToString()}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
private void GetProcessInfo(string procName)
{
var processes = Process.GetProcessesByName(procName);
if (processes?.Length == 0)
{
return;
}
foreach (var process in processes)
{
try
{
this.Token.ThrowIfCancellationRequested();
// ports in use by Fabric services...
this.TotalActivePortCount += NetworkUsage.GetActivePortCount(process.Id);
this.TotalActiveEphemeralPortCount += NetworkUsage.GetActiveEphemeralPortCount(process.Id);
int procCount = Environment.ProcessorCount;
while (!process.HasExited && procCount > 0)
{
this.Token.ThrowIfCancellationRequested();
try
{
int cpu = (int)this.perfCounters.PerfCounterGetProcessorInfo("% Processor Time", "Process", process.ProcessName);
this.allCpuData.FirstOrDefault(x => x.Id == procName).Data.Add(cpu);
// Disk IO (per-process disk reads/writes per sec)
this.allAppDiskReadsData.FirstOrDefault(x => x.Id == procName)
.Data.Add(this.diskUsage.PerfCounterGetDiskIOInfo(
process.ProcessName,
"Process",
"IO Read Operations/sec"));
this.allAppDiskWritesData.FirstOrDefault(x => x.Id == procName)
.Data.Add(this.diskUsage.PerfCounterGetDiskIOInfo(
process.ProcessName,
"Process",
"IO Write Operations/sec"));
// Memory - Private WS for proc...
var workingset = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMB(process.ProcessName);
this.allMemData.FirstOrDefault(x => x.Id == procName).Data.Add((long)workingset);
--procCount;
Thread.Sleep(250);
}
catch (Exception e)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Can't observe {process} details due to {e.Message} - {e.StackTrace}",
LogLevel.Warning);
throw;
}
}
}
catch (Win32Exception)
{
// This will always be the case if FabricObserver.exe is not running as Admin or LocalSystem...
// It's OK. Just means that the elevated process (like FabricHost.exe) won't be observed.
this.WriteToLogWithLevel(
this.ObserverName,
$"Can't observe {process} due to it's privilege level - " + "FabricObserver must be running as System or Admin for this specific task.",
LogLevel.Information);
break;
}
finally
{
process?.Dispose();
}
}
}
private void GetComputerInfo(CancellationToken token)
{
ManagementObjectSearcher win32OSInfo = null;
ManagementObjectCollection results = null;
var sb = new StringBuilder();
var diskUsage = new DiskUsage();
string osName = string.Empty;
string osVersion = string.Empty;
string numProcs = "-1";
string lastBootTime = string.Empty;
string installDate = string.Empty;
int driveCount = 0;
try
{
win32OSInfo = new ManagementObjectSearcher("SELECT Caption,Version,Status,OSLanguage,NumberOfProcesses,FreePhysicalMemory,FreeVirtualMemory,TotalVirtualMemorySize,TotalVisibleMemorySize,InstallDate,LastBootUpTime FROM Win32_OperatingSystem");
results = win32OSInfo.Get();
sb.AppendLine($"\nOS Info:\n");
foreach (var prop in results)
{
token.ThrowIfCancellationRequested();
foreach (var p in prop.Properties)
{
token.ThrowIfCancellationRequested();
string n = p.Name;
string v = p.Value.ToString();
if (string.IsNullOrEmpty(n) || string.IsNullOrEmpty(v))
{
continue;
}
if (n.ToLower() == "caption")
{
n = "OS";
osName = v;
}
if (n.ToLower() == "numberofprocesses")
{
// Number of running processes
numProcs = v;
// Also show number of processors on machine (logical cores)...
sb.AppendLine($"LogicalProcessorCount: {Environment.ProcessorCount}");
}
if (n.ToLower() == "status")
{
this.osStatus = v;
}
if (n.ToLower() == "version")
{
osVersion = v;
}
if (n.ToLower().Contains("bootuptime"))
{
v = ManagementDateTimeConverter.ToDateTime(v).ToString();
lastBootTime = v;
}
if (n.ToLower().Contains("date"))
{
v = ManagementDateTimeConverter.ToDateTime(v).ToString();
installDate = v;
}
if (n.ToLower().Contains("memory"))
{
if (n.ToLower().Contains("freephysical") ||
n.ToLower().Contains("freevirtual"))
{
continue;
}
// For output...
int i = int.Parse(v) / 1024 / 1024;
v = i.ToString() + " GB";
// TotalVisible only needs to be set once...
if (n.ToLower().Contains("totalvisible"))
{
this.totalVisibleMemoryGB = i;
}
}
sb.AppendLine($"{n}: {v}");
}
}
try
{
// We only care about ready drives (so, things like an empty DVD drive are not interesting...)
driveCount = DriveInfo.GetDrives().Where(d => d.IsReady).Count();
sb.AppendLine($"LogicalDriveCount: {driveCount}");
}
catch (ArgumentException)
{
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
this.osReport = sb.ToString();
// Active, bound ports...
int activePorts = NetworkUsage.GetActivePortCount();
// Active, ephemeral ports...
int activeEphemeralPorts = NetworkUsage.GetActiveEphemeralPortCount();
var dynamicPortRange = NetworkUsage.TupleGetDynamicPortRange();
string clusterManifestXml = null;
string osEphemeralPortRange = string.Empty;
string fabricAppPortRange = string.Empty;
if (this.IsTestRun)
{
clusterManifestXml = File.ReadAllText(this.TestManifestPath);
}
else
{
clusterManifestXml = this.FabricClientInstance.ClusterManager.GetClusterManifestAsync().GetAwaiter().GetResult();
}
var appPortRange = NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(this.FabricServiceContext.NodeContext.NodeType, clusterManifestXml);
// Enabled Firewall rules...
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
if (firewalls > -1)
{
this.osReport += $"EnabledFireWallRules: {firewalls}\r\n";
}
if (activePorts > -1)
{
this.osReport += $"TotalActiveTCPPorts: {activePorts}\r\n";
}
if (dynamicPortRange.Item1 > -1)
{
osEphemeralPortRange = $"{dynamicPortRange.Item1} - {dynamicPortRange.Item2}";
this.osReport += $"WindowsEphemeralTCPPortRange: {osEphemeralPortRange}\r\n";
}
if (appPortRange.Item1 > -1)
{
fabricAppPortRange = $"{appPortRange.Item1} - {appPortRange.Item2}";
this.osReport += $"FabricApplicationTCPPortRange: {fabricAppPortRange}\r\n";
}
if (activeEphemeralPorts > -1)
{
this.osReport += $"ActiveEphemeralTCPPorts: {activeEphemeralPorts}\r\n";
}
string osHotFixes = GetWindowsHotFixes(token);
if (!string.IsNullOrEmpty(osHotFixes))
{
this.osReport += $"\nOS Patches/Hot Fixes:\n\n{osHotFixes}\r\n";
}
// ETW...
if (this.IsEtwEnabled)
{
Logger.EtwLogger?.Write(
$"FabricObserverDataEvent",
new
{
Level = 0, // Info
Node = this.NodeName,
Observer = this.ObserverName,
OS = osName,
OSVersion = osVersion,
OSInstallDate = installDate,
LastBootUpTime = lastBootTime,
TotalMemorySizeGB = this.totalVisibleMemoryGB,
LogicalProcessorCount = Environment.ProcessorCount,
LogicalDriveCount = driveCount,
NumberOfRunningProcesses = int.Parse(numProcs),
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = osHotFixes.Replace("\r\n", ", ").TrimEnd(','),
});
}
}
catch (ManagementException)
{
}
catch (Exception e)
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Error,
$"Unhandled exception processing OS information: {e.Message}: \n {e.StackTrace}");
throw;
}
finally
{
results?.Dispose();
win32OSInfo?.Dispose();
diskUsage?.Dispose();
sb.Clear();
}
}
private Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
try
{
// Ports.
int activePortCountTotal = NetworkUsage.GetActivePortCount();
int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
// Note: Please make sure you understand the normal state of your nodes
// with respect to the machine resource use and/or abuse by your service(s).
// For example, if it is normal for your services to consume 90% of available CPU and memory
// as part of the work they perform under normal traffic flow, then it doesn't make sense to warn or
// error on these conditions.
// TODO: Look into making this a long running background task with signaling.
TimeSpan duration = TimeSpan.FromSeconds(30);
if (this.MonitorDuration > TimeSpan.MinValue)
{
duration = this.MonitorDuration;
}
// Warn up the counters.
_ = this.perfCounters.PerfCounterGetProcessorInfo();
_ = this.perfCounters.PerfCounterGetMemoryInfoMb();
while (this.stopwatch.Elapsed <= duration)
{
token.ThrowIfCancellationRequested();
if (this.CpuWarningUsageThresholdPct > 0 &&
this.CpuWarningUsageThresholdPct <= 100)
{
this.AllCpuTimeData.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
}
if (this.MemWarningUsageThresholdMb > 0)
{
this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
}
if (this.MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(
ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
}
Thread.Sleep(250);
}
}
catch (OperationCanceledException)
{
}
catch (Exception e)
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync:{Environment.NewLine}{e}");
throw;
}
return(Task.CompletedTask);
}
private void GetComputerInfo(CancellationToken token)
{
ManagementObjectSearcher win32OSInfo = null;
ManagementObjectCollection results = null;
var sb = new StringBuilder();
var diskUsage = new DiskUsage();
string osName = string.Empty;
string osVersion = string.Empty;
int numProcs = 0;
string lastBootTime = string.Empty;
string installDate = string.Empty;
int logicalProcessorCount = Environment.ProcessorCount;
int logicalDriveCount = 0;
int activePorts = 0;
int activeEphemeralPorts = 0;
int totalVirtMem = 0;
string fabricAppPortRange = string.Empty;
string osLang = string.Empty;
double freePhysicalMem = 0;
double freeVirtualMem = 0;
try
{
win32OSInfo = new ManagementObjectSearcher("SELECT Caption,Version,Status,OSLanguage,NumberOfProcesses,FreePhysicalMemory,FreeVirtualMemory,TotalVirtualMemorySize,TotalVisibleMemorySize,InstallDate,LastBootUpTime FROM Win32_OperatingSystem");
results = win32OSInfo.Get();
foreach (var prop in results)
{
token.ThrowIfCancellationRequested();
foreach (var p in prop.Properties)
{
token.ThrowIfCancellationRequested();
string name = p.Name;
string value = p.Value.ToString();
if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(value))
{
continue;
}
if (name.ToLower() == "caption")
{
osName = value;
}
else if (name.ToLower() == "numberofprocesses")
{
// Number of running processes
_ = int.TryParse(value, out numProcs);
}
else if (name.ToLower() == "status")
{
this.osStatus = value;
}
else if (name.ToLower() == "oslanguage")
{
osLang = value;
}
else if (name.ToLower() == "version")
{
osVersion = value;
}
else if (name.ToLower().Contains("bootuptime"))
{
value = ManagementDateTimeConverter.ToDateTime(value).ToUniversalTime().ToString("o");
lastBootTime = value;
}
else if (name.ToLower().Contains("date"))
{
value = ManagementDateTimeConverter.ToDateTime(value).ToUniversalTime().ToString("o");
installDate = value;
}
else if (name.ToLower().Contains("memory"))
{
// For output.
int i = int.Parse(value) / 1024 / 1024;
// TotalVisible only needs to be set once.
if (name.ToLower().Contains("totalvisible"))
{
this.totalVisibleMemoryGB = i;
}
else if (name.ToLower().Contains("totalvirtual"))
{
totalVirtMem = i;
}
else if (name.ToLower().Contains("freephysical"))
{
_ = double.TryParse(value, out freePhysicalMem);
}
else if (name.ToLower().Contains("freevirtual"))
{
_ = double.TryParse(value, out freeVirtualMem);
}
}
}
}
// Active, bound ports.
activePorts = NetworkUsage.GetActivePortCount();
// Active, ephemeral ports.
activeEphemeralPorts = NetworkUsage.GetActiveEphemeralPortCount();
var dynamicPortRange = NetworkUsage.TupleGetDynamicPortRange();
string clusterManifestXml = null;
string osEphemeralPortRange = string.Empty;
fabricAppPortRange = string.Empty;
if (this.IsTestRun)
{
clusterManifestXml = File.ReadAllText(this.TestManifestPath);
}
else
{
clusterManifestXml = this.FabricClientInstance.ClusterManager.GetClusterManifestAsync(this.AsyncClusterOperationTimeoutSeconds, this.Token).GetAwaiter().GetResult();
}
var appPortRange = NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(this.FabricServiceContext.NodeContext.NodeType, clusterManifestXml);
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
// OS info.
sb.AppendLine("OS Information:\r\n");
sb.AppendLine($"Name: {osName}");
sb.AppendLine($"Version: {osVersion}");
sb.AppendLine($"InstallDate: {installDate}");
sb.AppendLine($"LastBootUpTime*: {lastBootTime}");
sb.AppendLine($"OSLanguage: {osLang}");
sb.AppendLine($"OSHealthStatus*: {this.osStatus}");
sb.AppendLine($"NumberOfProcesses*: {numProcs}");
if (dynamicPortRange.Item1 > -1)
{
osEphemeralPortRange = $"{dynamicPortRange.Item1} - {dynamicPortRange.Item2}";
sb.AppendLine($"WindowsEphemeralTCPPortRange: {osEphemeralPortRange} (Active*: {activeEphemeralPorts})");
}
if (appPortRange.Item1 > -1)
{
fabricAppPortRange = $"{appPortRange.Item1} - {appPortRange.Item2}";
sb.AppendLine($"FabricApplicationTCPPortRange: {fabricAppPortRange}");
}
if (firewalls > -1)
{
sb.AppendLine($"ActiveFirewallRules*: {firewalls}");
}
if (activePorts > -1)
{
sb.AppendLine($"TotalActiveTCPPorts*: {activePorts}");
}
// Hardware info.
// Proc/Mem
sb.AppendLine("\r\nHardware Information:\r\n");
sb.AppendLine($"LogicalProcessorCount: {logicalProcessorCount}");
sb.AppendLine($"TotalVirtualMemorySize: {totalVirtMem} GB");
sb.AppendLine($"TotalVisibleMemorySize: {this.totalVisibleMemoryGB} GB");
sb.AppendLine($"FreePhysicalMemory*: {Math.Round(freePhysicalMem / 1024 / 1024, 2)} GB");
sb.AppendLine($"FreeVirtualMemory*: {Math.Round(freeVirtualMem / 1024 / 1024, 2)} GB");
// Disk
var drivesInformation = diskUsage.GetCurrentDiskSpaceTotalAndUsedPercentAllDrives(SizeUnit.Gigabytes);
logicalDriveCount = drivesInformation.Count;
sb.AppendLine($"LogicalDriveCount: {logicalDriveCount}");
foreach (var tuple in drivesInformation)
{
string systemDrv = "Data";
if (Environment.SystemDirectory.Substring(0, 1) == tuple.Item1)
{
systemDrv = "System";
}
sb.AppendLine($"Drive {tuple.Item1} ({systemDrv}) Size: {tuple.Item2} GB");
sb.AppendLine($"Drive {tuple.Item1} ({systemDrv}) Consumed*: {tuple.Item3}%");
}
string osHotFixes = GetWindowsHotFixes(token);
if (!string.IsNullOrEmpty(osHotFixes))
{
sb.AppendLine($"\nWindows Patches/Hot Fixes*:\n\n{osHotFixes}");
}
// Dynamic info qualifier (*)
sb.AppendLine($"\n* Dynamic data.");
this.osReport = sb.ToString();
// ETW.
if (this.IsEtwEnabled)
{
Logger.EtwLogger?.Write(
$"FabricObserverDataEvent",
new
{
Level = 0, // Info
Node = this.NodeName,
Observer = this.ObserverName,
OS = osName,
OSVersion = osVersion,
OSInstallDate = installDate,
LastBootUpTime = lastBootTime,
TotalMemorySizeGB = this.totalVisibleMemoryGB,
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
NumberOfRunningProcesses = numProcs,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = GetWindowsHotFixes(token, false).Replace("\r\n", ", ").TrimEnd(','),
});
}
}
catch (ManagementException)
{
}
catch (Exception e)
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Error,
$"Unhandled exception processing OS information: {e.Message}: \n {e.StackTrace}");
throw;
}
finally
{
results?.Dispose();
win32OSInfo?.Dispose();
diskUsage?.Dispose();
sb.Clear();
}
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
List <ReplicaOrInstanceMonitoringInfo> repOrInstList;
if (!string.IsNullOrEmpty(application.TargetType))
{
repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetType).ConfigureAwait(true);
}
else
{
repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.Target)).ConfigureAwait(true);
}
if (repOrInstList.Count == 0)
{
ObserverLogger.LogInfo("No target or targetType specified.");
return;
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
Token.ThrowIfCancellationRequested();
int processid = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level.
currentProcess = Process.GetProcessById(processid);
Token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process.
if (!allAppCpuData.Any(list => list.Id == id))
{
allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id));
allAppMemDataMB.Add(new FabricResourceUsageData <long>(ErrorWarningProperty.TotalMemoryConsumptionMb, id));
allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id));
allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id));
allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id));
}
// CPU (all cores).
int i = Environment.ProcessorCount + 10;
while (!currentProcess.HasExited && i > 0)
{
Token.ThrowIfCancellationRequested();
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process)).
var processMem = perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
allAppMemDataMB.FirstOrDefault(x => x.Id == id).Data.Add((long)processMem);
// Memory (percent in use (total)).
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.TotalMemory;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(usedPct);
}
--i;
Thread.Sleep(250);
}
// Total and Ephemeral ports..
allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
WriteToLogWithLevel(
ObserverName,
$"MonitorAsync failed to find current service process for {application.Target}/n{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
WriteToLogWithLevel(
ObserverName,
$"Unhandled exception in MonitorAsync: \n {e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
List <ReplicaOrInstanceMonitoringInfo> repOrInstList;
if (this.IsTestRun)
{
repOrInstList = this.ReplicaOrInstanceList;
}
else
{
if (!string.IsNullOrEmpty(application.TargetAppType))
{
repOrInstList = await this
.GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetAppType)
.ConfigureAwait(true);
}
else
{
repOrInstList = await this
.GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.TargetApp))
.ConfigureAwait(true);
}
if (repOrInstList.Count == 0)
{
this.ObserverLogger.LogInfo("No targetApp or targetAppType specified.");
return;
}
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
this.Token.ThrowIfCancellationRequested();
var timer = new Stopwatch();
int processId = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level.
currentProcess = Process.GetProcessById(processId);
this.Token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process.
if (this.allAppCpuData.All(list => list.Id != id))
{
this.allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppMemDataMb.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.TotalMemoryConsumptionMb, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id, 1));
this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id, 1));
}
TimeSpan duration = TimeSpan.FromSeconds(15);
if (this.MonitorDuration > TimeSpan.MinValue)
{
duration = this.MonitorDuration;
}
// Warm up the counters.
_ = cpuUsage.GetCpuUsageProcess(currentProcess);
_ = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
timer.Start();
while (!currentProcess.HasExited && timer.Elapsed <= duration)
{
this.Token.ThrowIfCancellationRequested();
// CPU (all cores).
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process)).
var processMem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
this.allAppMemDataMb.FirstOrDefault(x => x.Id == id).Data.Add(processMem);
// Memory (percent in use (total)).
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.TotalMemory;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(Math.Round(usedPct, 1));
}
Thread.Sleep(250);
}
timer.Stop();
timer.Reset();
// Total and Ephemeral ports..
this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"MonitorAsync failed to find current service process for {application.TargetApp}/n{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Unhandled exception in MonitorAsync: \n {e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
private async Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
CpuUtilizationProvider cpuUtilizationProvider = null;
try
{
// Ports.
int activePortCountTotal = OperatingSystemInfoProvider.Instance.GetActivePortCount();
int ephemeralPortCountTotal = OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
// Note: Please make sure you understand the normal state of your nodes
// with respect to the machine resource use and/or abuse by your service(s).
// For example, if it is normal for your services to consume 90% of available CPU and memory
// as part of the work they perform under normal traffic flow, then it doesn't make sense to warn or
// error on these conditions.
// TODO: Look into making this a long running background task with signaling.
TimeSpan duration = TimeSpan.FromSeconds(10);
if (MonitorDuration > TimeSpan.MinValue)
{
duration = MonitorDuration;
}
cpuUtilizationProvider = CpuUtilizationProvider.Create();
// Warm up the counters.
_ = await cpuUtilizationProvider.NextValueAsync();
while (this.stopwatch.Elapsed <= duration)
{
token.ThrowIfCancellationRequested();
if (CpuWarningUsageThresholdPct > 0 &&
CpuWarningUsageThresholdPct <= 100)
{
AllCpuTimeData.Data.Add(await cpuUtilizationProvider.NextValueAsync());
}
if (MemWarningUsageThresholdMb > 0)
{
float committedMegaBytes = MemoryUsageProvider.Instance.GetCommittedBytes() / 1048576.0f;
this.allMemDataCommittedBytes.Data.Add(committedMegaBytes);
}
if (MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(
OperatingSystemInfoProvider.Instance.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
}
await Task.Delay(250).ConfigureAwait(false);
}
}
catch (AggregateException e) when(e.InnerException is OperationCanceledException || e.InnerException is TaskCanceledException || e.InnerException is TimeoutException)
{
return;
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync:{Environment.NewLine}{e}");
throw;
}
finally
{
cpuUtilizationProvider?.Dispose();
}
}
private void GetProcessInfo(string procName)
{
var processes = Process.GetProcessesByName(procName);
if (processes.Length == 0)
{
return;
}
Stopwatch timer = new Stopwatch();
foreach (var process in processes)
{
try
{
this.Token.ThrowIfCancellationRequested();
// ports in use by Fabric services.
this.TotalActivePortCount += NetworkUsage.GetActivePortCount(process.Id);
this.TotalActiveEphemeralPortCount += NetworkUsage.GetActiveEphemeralPortCount(process.Id);
TimeSpan duration = TimeSpan.FromSeconds(15);
if (this.MonitorDuration > TimeSpan.MinValue)
{
duration = this.MonitorDuration;
}
// Warm up the counters.
_ = this.perfCounters.PerfCounterGetProcessorInfo("% Processor Time", "Process", process.ProcessName);
_ = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(process.ProcessName);
timer.Start();
while (!process.HasExited && timer.Elapsed <= duration)
{
this.Token.ThrowIfCancellationRequested();
try
{
// CPU Time for service process.
int cpu = (int)this.perfCounters.PerfCounterGetProcessorInfo("% Processor Time", "Process", process.ProcessName);
this.allCpuData.FirstOrDefault(x => x.Id == procName)?.Data.Add(cpu);
// Private Working Set for service process.
float mem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(process.ProcessName);
this.allMemData.FirstOrDefault(x => x.Id == procName)?.Data.Add(mem);
Thread.Sleep(250);
}
catch (Exception e)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Can't observe {process} details:{Environment.NewLine}{e}",
LogLevel.Warning);
throw;
}
}
}
catch (Win32Exception)
{
// This will always be the case if FabricObserver.exe is not running as Admin or LocalSystem.
// It's OK. Just means that the elevated process (like FabricHost.exe) won't be observed.
this.WriteToLogWithLevel(
this.ObserverName,
$"Can't observe {process.ProcessName} due to it's privilege level. FabricObserver must be running as System or Admin for this specific task.",
LogLevel.Information);
break;
}
finally
{
process?.Dispose();
}
timer.Stop();
timer.Reset();
}
}
public NetworkUsageResource()
{
_lastNetworkUsage = GetTotalUsage();
}
private async Task GetComputerInfoAsync(CancellationToken token)
{
var sb = new StringBuilder();
int logicalProcessorCount = Environment.ProcessorCount;
try
{
OSInfo osInfo = await OperatingSystemInfoProvider.Instance.GetOSInfoAsync(token);
this.osStatus = osInfo.Status;
// Active, bound ports.
int activePorts = OperatingSystemInfoProvider.Instance.GetActivePortCount();
// Active, ephemeral ports.
int activeEphemeralPorts = OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount();
(int lowPortOS, int highPortOS) = OperatingSystemInfoProvider.Instance.TupleGetDynamicPortRange();
string osEphemeralPortRange = string.Empty;
string fabricAppPortRange = string.Empty;
string clusterManifestXml = IsTestRun ? File.ReadAllText(
TestManifestPath) : await FabricClientInstance.ClusterManager.GetClusterManifestAsync(
AsyncClusterOperationTimeoutSeconds, Token).ConfigureAwait(false);
(int lowPortApp, int highPortApp) =
NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(
FabricServiceContext.NodeContext.NodeType,
clusterManifestXml);
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
// OS info.
_ = sb.AppendLine("OS Information:\r\n");
_ = sb.AppendLine($"Name: {osInfo.Name}");
_ = sb.AppendLine($"Version: {osInfo.Version}");
if (string.IsNullOrEmpty(osInfo.InstallDate))
{
_ = sb.AppendLine($"InstallDate: {osInfo.InstallDate}");
}
_ = sb.AppendLine($"LastBootUpTime*: {osInfo.LastBootUpTime}");
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// WU AutoUpdate - Download enabled.
// If the config setting EnableWindowsAutoUpdateCheck is set to false, then don't add this info to sb.
if (this.isWUADSettingEnabled)
{
string auMessage = "WindowsUpdateAutoDownloadEnabled: ";
if (this.auStateUnknown)
{
auMessage += "Unknown";
}
else
{
auMessage += this.isWindowsUpdateAutoDownloadEnabled;
}
_ = sb.AppendLine(auMessage);
}
// Not supported for Linux.
_ = sb.AppendLine($"OSLanguage: {osInfo.Language}");
_ = sb.AppendLine($"OSHealthStatus*: {osInfo.Status}");
}
_ = sb.AppendLine($"NumberOfProcesses*: {osInfo.NumberOfProcesses}");
if (lowPortOS > -1)
{
osEphemeralPortRange = $"{lowPortOS} - {highPortOS}";
_ = sb.AppendLine($"OSEphemeralTCPPortRange: {osEphemeralPortRange} (Active*: {activeEphemeralPorts})");
}
if (lowPortApp > -1)
{
fabricAppPortRange = $"{lowPortApp} - {highPortApp}";
_ = sb.AppendLine($"FabricApplicationTCPPortRange: {fabricAppPortRange}");
}
if (firewalls > -1)
{
_ = sb.AppendLine($"ActiveFirewallRules*: {firewalls}");
}
if (activePorts > -1)
{
_ = sb.AppendLine($"TotalActiveTCPPorts*: {activePorts}");
}
// Hardware info.
// Proc/Mem
_ = sb.AppendLine($"{Environment.NewLine}Hardware Information:{Environment.NewLine}");
_ = sb.AppendLine($"LogicalProcessorCount: {logicalProcessorCount}");
if (osInfo.TotalVirtualMemorySizeKB > 0)
{
_ = sb.AppendLine($"TotalVirtualMemorySize: {osInfo.TotalVirtualMemorySizeKB / 1048576} GB");
}
if (osInfo.TotalVisibleMemorySizeKB > 0)
{
_ = sb.AppendLine($"TotalVisibleMemorySize: {osInfo.TotalVisibleMemorySizeKB / 1048576} GB");
}
_ = sb.AppendLine($"FreePhysicalMemory*: {Math.Round(osInfo.AvailableMemoryKB / 1048576.0, 2)} GB");
_ = sb.AppendLine($"FreeVirtualMemory*: {Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2)} GB");
// Disk
var drivesInformationTuple = DiskUsage.GetCurrentDiskSpaceTotalAndUsedPercentAllDrives(SizeUnit.Gigabytes);
var logicalDriveCount = drivesInformationTuple.Count;
string driveInfo = string.Empty;
_ = sb.AppendLine($"LogicalDriveCount: {logicalDriveCount}");
foreach (var(driveName, diskSize, percentConsumed) in drivesInformationTuple)
{
string drvSize;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
string systemDrv = "Data";
if (string.Equals(Environment.SystemDirectory.Substring(0, 1), driveName.Substring(0, 1), StringComparison.OrdinalIgnoreCase))
{
systemDrv = "System";
}
drvSize = $"Drive {driveName} ({systemDrv}) Size: {diskSize} GB, Consumed*: {percentConsumed}%";
}
else
{
drvSize = $"Mount point: {driveName}, Size: {diskSize} GB, Consumed*: {percentConsumed}%";
}
_ = sb.AppendLine(drvSize);
driveInfo += $"{drvSize}{Environment.NewLine}";
}
string osHotFixes = string.Empty;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
osHotFixes = GetWindowsHotFixes(token);
}
if (!string.IsNullOrEmpty(osHotFixes))
{
_ = sb.AppendLine($"\nWindows Patches/Hot Fixes*:\n\n{osHotFixes}");
}
// Dynamic info qualifier (*)
_ = sb.AppendLine($"\n* Dynamic data.");
this.osReport = sb.ToString();
string hotFixes = string.Empty;
// ETW.
if (IsEtwEnabled)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
hotFixes = GetWindowsHotFixes(token, generateUrl: false).Replace("\r\n", ", ").TrimEnd(',');
}
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
HealthState = "Ok",
Node = NodeName,
Observer = ObserverName,
OS = osInfo.Name,
OSVersion = osInfo.Version,
OSInstallDate = osInfo.InstallDate,
AutoUpdateEnabled = this.auStateUnknown ? "Unknown" : this.isWindowsUpdateAutoDownloadEnabled.ToString(),
osInfo.LastBootUpTime,
WindowsAutoUpdateEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = (int)(osInfo.TotalVisibleMemorySizeKB / 1048576),
AvailablePhysicalMemoryGB = Math.Round(osInfo.FreePhysicalMemoryKB / 1048576.0, 2),
AvailableVirtualMemoryGB = Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = osInfo.NumberOfProcesses,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = hotFixes,
});
}
// Telemetry
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
if (string.IsNullOrEmpty(hotFixes) && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
hotFixes = GetWindowsHotFixes(token, generateUrl: false).Replace("\r\n", ", ").TrimEnd(',');
}
TelemetryClient?.ReportMetricAsync(
new MachineTelemetryData
{
HealthState = "Ok",
Node = NodeName,
Observer = ObserverName,
OS = osInfo.Name,
OSVersion = osInfo.Version,
OSInstallDate = osInfo.InstallDate,
LastBootUpTime = osInfo.LastBootUpTime,
WindowsUpdateAutoDownloadEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = (int)osInfo.TotalVisibleMemorySizeKB / 1048576,
AvailablePhysicalMemoryGB = Math.Round(osInfo.FreePhysicalMemoryKB / 1048576.0, 2),
AvailableVirtualMemoryGB = Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = osInfo.NumberOfProcesses,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = hotFixes,
}, Token);
}
}
catch (Exception e) when(e is FabricException || e is OperationCanceledException || e is TaskCanceledException || e is InvalidComObjectException)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
$"Handled Exception processing OS information:{Environment.NewLine}{e}");
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Error,
$"Unhandled Exception processing OS information:{Environment.NewLine}{e}");
throw;
}
}
private async Task <string> GetDeployedAppsInfoAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
ApplicationList appList = null;
var sb = new StringBuilder();
string clusterManifestXml;
if (this.IsTestRun)
{
clusterManifestXml = File.ReadAllText(Path.Combine(Environment.CurrentDirectory, "clusterManifest.xml"));
}
else
{
appList = await this.FabricClientInstance.QueryManager.GetApplicationListAsync().ConfigureAwait(true);
clusterManifestXml = await this.FabricClientInstance.ClusterManager.GetClusterManifestAsync().ConfigureAwait(true);
}
token.ThrowIfCancellationRequested();
XmlReader xreader = null;
StringReader sreader = null;
string ret = null;
try
{
// Safe XML pattern - *Do not use LoadXml*...
var xdoc = new XmlDocument {
XmlResolver = null
};
sreader = new StringReader(clusterManifestXml);
xreader = XmlReader.Create(sreader, new XmlReaderSettings()
{
XmlResolver = null
});
xdoc.Load(xreader);
// Cluster Information...
var nsmgr = new XmlNamespaceManager(xdoc.NameTable);
nsmgr.AddNamespace("sf", "http://schemas.microsoft.com/2011/01/fabric");
// Failover Manager...
var fMparameterNodes = xdoc.SelectNodes("//sf:Section[@Name='FailoverManager']//sf:Parameter", nsmgr);
sb.AppendLine("\nCluster Information:\n");
foreach (XmlNode node in fMparameterNodes)
{
token.ThrowIfCancellationRequested();
sb.AppendLine(node.Attributes.Item(0).Value + ": " + node.Attributes.Item(1).Value);
}
token.ThrowIfCancellationRequested();
// Node Information...
sb.AppendLine($"\nNode Info:\n");
sb.AppendLine($"Node Name: {this.NodeName}");
sb.AppendLine($"Node Id: {this.FabricServiceContext.NodeContext.NodeId}");
sb.AppendLine($"Node Instance Id: {this.FabricServiceContext.NodeContext.NodeInstanceId}");
sb.AppendLine($"Node Type: {this.FabricServiceContext.NodeContext.NodeType}");
Tuple <int, int> portRange = NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(this.FabricServiceContext.NodeContext.NodeType, clusterManifestXml);
if (portRange.Item1 > -1)
{
sb.AppendLine($"Application Port Range: {portRange.Item1} - {portRange.Item2}");
}
var infraNode = xdoc.SelectSingleNode("//sf:Node", nsmgr);
if (infraNode != null)
{
sb.AppendLine("Is Seed Node: " + infraNode.Attributes["IsSeedNode"]?.Value);
sb.AppendLine("Fault Domain: " + infraNode.Attributes["FaultDomain"]?.Value);
sb.AppendLine("Upgrade Domain: " + infraNode.Attributes["UpgradeDomain"]?.Value);
}
token.ThrowIfCancellationRequested();
if (!string.IsNullOrEmpty(this.sFNodeLastBootTime))
{
sb.AppendLine("Last Rebooted: " + this.sFNodeLastBootTime);
}
// Stop here for unit testing...
if (this.IsTestRun)
{
ret = sb.ToString();
sb.Clear();
return(ret);
}
// Application Info...
// TODO: Emit ETW event for App and Service info...
if (appList != null)
{
sb.AppendLine("\nDeployed Apps:\n");
foreach (var app in appList)
{
token.ThrowIfCancellationRequested();
var appName = app.ApplicationName.OriginalString;
var appType = app.ApplicationTypeName;
var appVersion = app.ApplicationTypeVersion;
var healthState = app.HealthState.ToString();
var status = app.ApplicationStatus.ToString();
sb.AppendLine("Application Name: " + appName);
sb.AppendLine("Type: " + appType);
sb.AppendLine("Version: " + appVersion);
sb.AppendLine("Health state: " + healthState);
sb.AppendLine("Status: " + status);
// App's Service(s)...
sb.AppendLine("\n\tServices:");
var serviceList = await this.FabricClientInstance.QueryManager.GetServiceListAsync(app.ApplicationName).ConfigureAwait(true);
var replicaList = await this.FabricClientInstance.QueryManager.GetDeployedReplicaListAsync(this.NodeName, app.ApplicationName).ConfigureAwait(true);
foreach (var service in serviceList)
{
var kind = service.ServiceKind.ToString();
var type = service.ServiceTypeName;
var serviceManifestversion = service.ServiceManifestVersion;
var serviceName = service.ServiceName;
var serviceDescription = await this.FabricClientInstance.ServiceManager.GetServiceDescriptionAsync(serviceName).ConfigureAwait(true);
var processModel = serviceDescription.ServicePackageActivationMode.ToString();
foreach (var rep in replicaList)
{
if (service.ServiceName != rep.ServiceName)
{
continue;
}
// Get established port count per service...
int procId = (int)rep.HostProcessId;
int ports = -1, ephemeralPorts = -1;
if (procId > -1)
{
ports = NetworkUsage.GetActivePortCount(procId);
ephemeralPorts = NetworkUsage.GetActiveEphemeralPortCount(procId);
}
sb.AppendLine("\tService Name: " + serviceName.OriginalString);
sb.AppendLine("\tTypeName: " + type);
sb.AppendLine("\tKind: " + kind);
sb.AppendLine("\tProcessModel: " + processModel);
sb.AppendLine("\tServiceManifest Version: " + serviceManifestversion);
if (ports > -1)
{
sb.AppendLine("\tActive Ports: " + ports);
}
if (ephemeralPorts > -1)
{
sb.AppendLine("\tActive Ephemeral Ports: " + ephemeralPorts);
}
sb.AppendLine();
// ETW...
if (this.IsEtwEnabled)
{
Logger.EtwLogger?.Write(
$"FabricObserverDataEvent",
new
{
Level = 0, // Info
Node = this.NodeName,
Observer = this.ObserverName,
AppName = appName,
AppType = appType,
AppVersion = appVersion,
AppHealthState = healthState,
AppStatus = status,
ServiceName = serviceName.OriginalString,
ServiceTypeName = type,
Kind = kind,
ProcessModel = processModel,
ServiceManifestVersion = serviceManifestversion,
ActivePorts = ports,
EphemeralPorts = ephemeralPorts,
});
}
break;
}
}
}
}
ret = sb.ToString();
sb.Clear();
}
finally
{
sreader.Dispose();
xreader.Dispose();
}
return(ret);
}
private async Task GetComputerInfoAsync(CancellationToken token)
{
ManagementObjectSearcher win32OsInfo = null;
ManagementObjectCollection results = null;
var sb = new StringBuilder();
var diskUsage = new DiskUsage();
string osName = string.Empty;
string osVersion = string.Empty;
int numProcs = 0;
string lastBootTime = string.Empty;
string installDate = string.Empty;
int logicalProcessorCount = Environment.ProcessorCount;
int totalVirtualMem = 0;
string osLang = string.Empty;
double freePhysicalMem = 0;
double freeVirtualMem = 0;
try
{
win32OsInfo = new ManagementObjectSearcher("SELECT Caption,Version,Status,OSLanguage,NumberOfProcesses,FreePhysicalMemory,FreeVirtualMemory,TotalVirtualMemorySize,TotalVisibleMemorySize,InstallDate,LastBootUpTime FROM Win32_OperatingSystem");
results = win32OsInfo.Get();
foreach (var prop in results)
{
token.ThrowIfCancellationRequested();
foreach (var p in prop.Properties)
{
token.ThrowIfCancellationRequested();
string name = p.Name;
string value = p.Value.ToString();
if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(value))
{
continue;
}
switch (name.ToLower())
{
case "caption":
osName = value;
break;
case "numberofprocesses":
_ = int.TryParse(value, out numProcs);
break;
case "status":
this.osStatus = value;
break;
case "oslanguage":
osLang = value;
break;
case "version":
osVersion = value;
break;
default:
{
if (name.ToLower().Contains("bootuptime"))
{
value = ManagementDateTimeConverter.ToDateTime(value).ToUniversalTime().ToString("o");
lastBootTime = value;
}
else if (name.ToLower().Contains("date"))
{
value = ManagementDateTimeConverter.ToDateTime(value).ToUniversalTime().ToString("o");
installDate = value;
}
else if (name.ToLower().Contains("memory"))
{
// For output.
int i = int.Parse(value) / 1024 / 1024;
// TotalVisible only needs to be set once.
if (name.ToLower().Contains("totalvisible"))
{
this.totalVisibleMemoryGb = i;
}
else if (name.ToLower().Contains("totalvirtual"))
{
totalVirtualMem = i;
}
else if (name.ToLower().Contains("freephysical"))
{
_ = double.TryParse(value, out freePhysicalMem);
}
else if (name.ToLower().Contains("freevirtual"))
{
_ = double.TryParse(value, out freeVirtualMem);
}
}
break;
}
}
}
}
// Active, bound ports.
var activePorts = NetworkUsage.GetActivePortCount();
// Active, ephemeral ports.
var activeEphemeralPorts = NetworkUsage.GetActiveEphemeralPortCount();
var(lowPortOs, highPortOs) = NetworkUsage.TupleGetDynamicPortRange();
string osEphemeralPortRange = string.Empty;
var fabricAppPortRange = string.Empty;
var clusterManifestXml = this.IsTestRun ? File.ReadAllText(
this.TestManifestPath) : await this.FabricClientInstance.ClusterManager.GetClusterManifestAsync(
this.AsyncClusterOperationTimeoutSeconds, this.Token).ConfigureAwait(false);
var(lowPortApp, highPortApp) =
NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(
this.FabricServiceContext.NodeContext.NodeType,
clusterManifestXml);
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
// WU AutoUpdate
string auMessage = "WindowsUpdateAutoDownloadEnabled: ";
if (this.auStateUnknown)
{
auMessage += "Unknown";
}
else
{
auMessage += this.isWindowsUpdateAutoDownloadEnabled;
}
// OS info.
_ = sb.AppendLine("OS Information:\r\n");
_ = sb.AppendLine($"Name: {osName}");
_ = sb.AppendLine($"Version: {osVersion}");
_ = sb.AppendLine($"InstallDate: {installDate}");
_ = sb.AppendLine($"LastBootUpTime*: {lastBootTime}");
_ = sb.AppendLine(auMessage);
_ = sb.AppendLine($"OSLanguage: {osLang}");
_ = sb.AppendLine($"OSHealthStatus*: {this.osStatus}");
_ = sb.AppendLine($"NumberOfProcesses*: {numProcs}");
if (lowPortOs > -1)
{
osEphemeralPortRange = $"{lowPortOs} - {highPortOs}";
_ = sb.AppendLine($"WindowsEphemeralTCPPortRange: {osEphemeralPortRange} (Active*: {activeEphemeralPorts})");
}
if (lowPortApp > -1)
{
fabricAppPortRange = $"{lowPortApp} - {highPortApp}";
_ = sb.AppendLine($"FabricApplicationTCPPortRange: {fabricAppPortRange}");
}
if (firewalls > -1)
{
_ = sb.AppendLine($"ActiveFirewallRules*: {firewalls}");
}
if (activePorts > -1)
{
_ = sb.AppendLine($"TotalActiveTCPPorts*: {activePorts}");
}
// Hardware info.
// Proc/Mem
_ = sb.AppendLine($"{Environment.NewLine}Hardware Information:{Environment.NewLine}");
_ = sb.AppendLine($"LogicalProcessorCount: {logicalProcessorCount}");
_ = sb.AppendLine($"TotalVirtualMemorySize: {totalVirtualMem} GB");
_ = sb.AppendLine($"TotalVisibleMemorySize: {this.totalVisibleMemoryGb} GB");
_ = sb.AppendLine($"FreePhysicalMemory*: {Math.Round(freePhysicalMem / 1024 / 1024, 2)} GB");
_ = sb.AppendLine($"FreeVirtualMemory*: {Math.Round(freeVirtualMem / 1024 / 1024, 2)} GB");
// Disk
var drivesInformationTuple = diskUsage.GetCurrentDiskSpaceTotalAndUsedPercentAllDrives(SizeUnit.Gigabytes);
var logicalDriveCount = drivesInformationTuple.Count;
string driveInfo = string.Empty;
_ = sb.AppendLine($"LogicalDriveCount: {logicalDriveCount}");
foreach (var(driveName, diskSize, percentConsumed) in drivesInformationTuple)
{
string systemDrv = "Data";
if (Environment.SystemDirectory.Substring(0, 1) == driveName)
{
systemDrv = "System";
}
string drvSize = $"Drive {driveName} ({systemDrv}) Size: {diskSize} GB";
string drvConsumed = $"Drive {driveName} ({systemDrv}) Consumed*: {percentConsumed}%";
_ = sb.AppendLine(drvSize);
_ = sb.AppendLine(drvConsumed);
driveInfo += $"{drvSize}{Environment.NewLine}{drvConsumed}{Environment.NewLine}";
}
string osHotFixes = GetWindowsHotFixes(token);
if (!string.IsNullOrEmpty(osHotFixes))
{
_ = sb.AppendLine($"\nWindows Patches/Hot Fixes*:\n\n{osHotFixes}");
}
// Dynamic info qualifier (*)
_ = sb.AppendLine($"\n* Dynamic data.");
this.osReport = sb.ToString();
// ETW.
if (this.IsEtwEnabled)
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
HealthState = "Ok",
Node = this.NodeName,
Observer = this.ObserverName,
OS = osName,
OSVersion = osVersion,
OSInstallDate = installDate,
AutoUpdateEnabled = this.auStateUnknown ? "Unknown" : this.isWindowsUpdateAutoDownloadEnabled.ToString(),
LastBootUpTime = lastBootTime,
WindowsAutoUpdateEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = this.totalVisibleMemoryGb,
AvailablePhysicalMemoryGB = Math.Round(freePhysicalMem / 1024 / 1024, 2),
AvailableVirtualMemoryGB = Math.Round(freeVirtualMem / 1024 / 1024, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = numProcs,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = GetWindowsHotFixes(token, false).Replace("\r\n", ", ").TrimEnd(','),
});
}
// Telemetry
if (this.IsTelemetryProviderEnabled && this.IsObserverTelemetryEnabled)
{
this.TelemetryClient?.ReportMetricAsync(
new MachineTelemetryData
{
HealthState = "Ok",
Node = this.NodeName,
Observer = this.ObserverName,
OS = osName,
OSVersion = osVersion,
OSInstallDate = installDate,
LastBootUpTime = lastBootTime,
WindowsUpdateAutoDownloadEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = this.totalVisibleMemoryGb,
AvailablePhysicalMemoryGB = Math.Round(freePhysicalMem / 1024 / 1024, 2),
AvailableVirtualMemoryGB = Math.Round(freeVirtualMem / 1024 / 1024, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = numProcs,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = GetWindowsHotFixes(token, false).Replace("\r\n", ", ").TrimEnd(','),
}, this.Token);
}
}
catch (ManagementException)
{
}
catch (Exception e)
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Error,
$"Unhandled exception processing OS information:{Environment.NewLine}{e}");
throw;
}
finally
{
results?.Dispose();
win32OsInfo?.Dispose();
diskUsage?.Dispose();
_ = sb.Clear();
}
}
