private async Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
{
await Task.Run(
() =>
{
token.ThrowIfCancellationRequested();
try
{
// Ports.
int activePortCountTotal = NetworkUsage.GetActivePortCount();
int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
this.activePortsData.Data.Add(activePortCountTotal);
this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);
// Firewall rules.
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
this.firewallData.Data.Add(firewalls);
// CPU and Memory.
for (int i = 0; i < 30; i++)
{
token.ThrowIfCancellationRequested();
if (this.CpuWarningUsageThresholdPct > 0 &&
this.CpuWarningUsageThresholdPct <= 100)
{
this.allCpuDataPrivTime.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
}
if (this.MemWarningUsageThresholdMB > 0)
{
this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMB());
}
if (this.MemoryWarningLimitPercent > 0)
{
this.allMemDataPercentUsed.Data.Add(ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().Item2);
}
Thread.Sleep(250);
}
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
this.HealthReporter.ReportFabricObserverServiceHealth(
this.FabricServiceContext.ServiceName.OriginalString,
this.ObserverName,
HealthState.Warning,
$"Unhandled exception in GetSystemCpuMemoryValuesAsync: {e.Message}: \n {e.StackTrace}");
}
throw;
}
}, token).ConfigureAwait(true);
}
private async Task MonitorAppAsync(ApplicationInfo application)
{
var repOrInstList = await this.GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.Target)).ConfigureAwait(true);
if (repOrInstList.Count == 0)
{
return;
}
Process currentProcess = null;
foreach (var repOrInst in repOrInstList)
{
this.Token.ThrowIfCancellationRequested();
int processid = (int)repOrInst.ReplicaHostProcessId;
var cpuUsage = new CpuUsage();
try
{
// App level...
currentProcess = Process.GetProcessById(processid);
this.Token.ThrowIfCancellationRequested();
if (currentProcess == null)
{
continue;
}
var procName = currentProcess.ProcessName;
// Add new resource data structures for each app service process...
var id = $"{application.Target.Replace("fabric:/", string.Empty)}:{procName}";
if (!this.allAppCpuData.Any(list => list.Id == id))
{
this.allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id));
this.allAppMemDataMB.Add(new FabricResourceUsageData <long>(ErrorWarningProperty.TotalMemoryConsumptionMB, id));
this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id));
this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id));
this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id));
}
// CPU (all cores)...
int i = Environment.ProcessorCount + 10;
while (!currentProcess.HasExited && i > 0)
{
this.Token.ThrowIfCancellationRequested();
int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);
if (cpu >= 0)
{
this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
}
// Memory (private working set (process))...
var mem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMB(currentProcess.ProcessName);
this.allAppMemDataMB.FirstOrDefault(x => x.Id == id).Data.Add((long)mem);
// Memory (percent in use (total))...
var memInfo = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
long totalMem = memInfo.Item1;
if (totalMem > -1)
{
double usedPct = Math.Round(((double)(mem * 100)) / (totalMem * 1024), 2);
this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(usedPct);
}
--i;
Thread.Sleep(250);
}
// Total and Ephemeral ports....
this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));
this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
.Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
}
catch (Exception e)
{
if (e is Win32Exception)
{
this.WriteToLogWithLevel(
this.ObserverName,
$"MonitorAsync failed to find current service process for {application.Target}",
LogLevel.Information);
}
else
{
if (!(e is OperationCanceledException))
{
this.WriteToLogWithLevel(
this.ObserverName,
$"Unhandled exception in MonitorAsync: \n {e.ToString()}",
LogLevel.Warning);
}
throw;
}
}
finally
{
currentProcess?.Dispose();
currentProcess = null;
}
}
}