private async Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
await Task.Run(
() =>
{
token.ThrowIfCancellationRequested();
try
{
// Ports.
int activePortCountTotal = NetworkUsage.GetActivePortCount();
int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
for (int i = 0; i < 30; i++)
{
token.ThrowIfCancellationRequested();
if (this.CpuWarningUsageThresholdPct > 0 &&
this.CpuWarningUsageThresholdPct <= 100)
{
this.allCpuDataPrivTime.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
}
if (this.MemWarningUsageThresholdMb > 0)
{
this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
}
if (this.MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(
ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
}
Thread.Sleep(250);
}
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync: {e.Message}: \n {e.StackTrace}");
}
throw;
}
}, token).ConfigureAwait(true);
}
private Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
try
{
// Ports.
int activePortCountTotal = NetworkUsage.GetActivePortCount();
int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
// Note: Please make sure you understand the normal state of your nodes
// with respect to the machine resource use and/or abuse by your service(s).
// For example, if it is normal for your services to consume 90% of available CPU and memory
// as part of the work they perform under normal traffic flow, then it doesn't make sense to warn or
// error on these conditions.
// TODO: Look into making this a long running background task with signaling.
TimeSpan duration = TimeSpan.FromSeconds(30);
if (this.MonitorDuration > TimeSpan.MinValue)
{
duration = this.MonitorDuration;
}
// Warn up the counters.
_ = this.perfCounters.PerfCounterGetProcessorInfo();
_ = this.perfCounters.PerfCounterGetMemoryInfoMb();
while (this.stopwatch.Elapsed <= duration)
{
token.ThrowIfCancellationRequested();
if (this.CpuWarningUsageThresholdPct > 0 &&
this.CpuWarningUsageThresholdPct <= 100)
{
this.AllCpuTimeData.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
}
if (this.MemWarningUsageThresholdMb > 0)
{
this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
}
if (this.MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(
ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
}
Thread.Sleep(250);
}
}
catch (OperationCanceledException)
{
}
catch (Exception e)
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync:{Environment.NewLine}{e}");
throw;
}
return(Task.CompletedTask);
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
List <ReplicaOrInstanceMonitoringInfo> repOrInstList;
if (!string.IsNullOrEmpty(application.TargetType))
{
repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetType).ConfigureAwait(true);
}
else
{
repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.Target)).ConfigureAwait(true);
}
if (repOrInstList.Count == 0)
{
ObserverLogger.LogInfo("No target or targetType specified.");
return;
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
Token.ThrowIfCancellationRequested();
int processid = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level.
currentProcess = Process.GetProcessById(processid);
Token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process.
if (!allAppCpuData.Any(list => list.Id == id))
{
allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id));
allAppMemDataMB.Add(new FabricResourceUsageData <long>(ErrorWarningProperty.TotalMemoryConsumptionMb, id));
allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id));
allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id));
allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id));
}
// CPU (all cores).
int i = Environment.ProcessorCount + 10;
while (!currentProcess.HasExited && i > 0)
{
Token.ThrowIfCancellationRequested();
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process)).
var processMem = perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
allAppMemDataMB.FirstOrDefault(x => x.Id == id).Data.Add((long)processMem);
// Memory (percent in use (total)).
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.TotalMemory;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(usedPct);
}
--i;
Thread.Sleep(250);
}
// Total and Ephemeral ports..
allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
WriteToLogWithLevel(
ObserverName,
$"MonitorAsync failed to find current service process for {application.Target}/n{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
WriteToLogWithLevel(
ObserverName,
$"Unhandled exception in MonitorAsync: \n {e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
List <ReplicaOrInstanceMonitoringInfo> repOrInstList;
if (this.IsTestRun)
{
repOrInstList = this.ReplicaOrInstanceList;
}
else
{
if (!string.IsNullOrEmpty(application.TargetAppType))
{
repOrInstList = await this
.GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetAppType)
.ConfigureAwait(true);
}
else
{
repOrInstList = await this
.GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.TargetApp))
.ConfigureAwait(true);
}
if (repOrInstList.Count == 0)
{
this.ObserverLogger.LogInfo("No targetApp or targetAppType specified.");
return;
}
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
this.Token.ThrowIfCancellationRequested();
var timer = new Stopwatch();
int processId = (int)repOrInst.HostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level.
currentProcess = Process.GetProcessById(processId);
this.Token.ThrowIfCancellationRequested();
var procName = currentProcess.ProcessName;
string appNameOrType = GetAppNameOrType(repOrInst);
var id = $"{appNameOrType}:{procName}";
// Add new resource data structures for each app service process.
if (this.allAppCpuData.All(list => list.Id != id))
{
this.allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppMemDataMb.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.TotalMemoryConsumptionMb, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id, this.DataCapacity, this.UseCircularBuffer));
this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id, 1));
this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id, 1));
}
TimeSpan duration = TimeSpan.FromSeconds(15);
if (this.MonitorDuration > TimeSpan.MinValue)
{
duration = this.MonitorDuration;
}
// Warm up the counters.
_ = cpuUsage.GetCpuUsageProcess(currentProcess);
_ = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
timer.Start();
while (!currentProcess.HasExited && timer.Elapsed <= duration)
{
this.Token.ThrowIfCancellationRequested();
// CPU (all cores).
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process)).
var processMem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
this.allAppMemDataMb.FirstOrDefault(x => x.Id == id).Data.Add(processMem);
// Memory (percent in use (total)).
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.TotalMemory;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(Math.Round(usedPct, 1));
}
Thread.Sleep(250);
}
timer.Stop();
timer.Reset();
// Total and Ephemeral ports..
this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"MonitorAsync failed to find current service process for {application.TargetApp}/n{e}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Unhandled exception in MonitorAsync: \n {e}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}
