private async Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
        {
            await Task.Run(
                () =>
            {
                token.ThrowIfCancellationRequested();

                try
                {
                    // Ports.
                    int activePortCountTotal    = NetworkUsage.GetActivePortCount();
                    int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
                    this.activePortsData.Data.Add(activePortCountTotal);
                    this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);

                    // Firewall rules.
                    int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
                    this.firewallData.Data.Add(firewalls);

                    // CPU and Memory.
                    for (int i = 0; i < 30; i++)
                    {
                        token.ThrowIfCancellationRequested();

                        if (this.CpuWarningUsageThresholdPct > 0 &&
                            this.CpuWarningUsageThresholdPct <= 100)
                        {
                            this.allCpuDataPrivTime.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
                        }

                        if (this.MemWarningUsageThresholdMb > 0)
                        {
                            this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
                        }

                        if (this.MemoryWarningLimitPercent > 0)
                        {
                            this.allMemDataPercentUsed.Data.Add(
                                ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
                        }

                        Thread.Sleep(250);
                    }
                }
                catch (Exception e)
                {
                    if (!(e is OperationCanceledException))
                    {
                        this.HealthReporter.ReportFabricObserverServiceHealth(
                            this.FabricServiceContext.ServiceName.OriginalString,
                            this.ObserverName,
                            HealthState.Warning,
                            $"Unhandled exception in GetSystemCpuMemoryValuesAsync: {e.Message}: \n {e.StackTrace}");
                    }

                    throw;
                }
            }, token).ConfigureAwait(true);
        }
예제 #2
0
        private Task GetSystemCpuMemoryValuesAsync(CancellationToken token)
        {
            token.ThrowIfCancellationRequested();

            try
            {
                // Ports.
                int activePortCountTotal    = NetworkUsage.GetActivePortCount();
                int ephemeralPortCountTotal = NetworkUsage.GetActiveEphemeralPortCount();
                this.activePortsData.Data.Add(activePortCountTotal);
                this.ephemeralPortsData.Data.Add(ephemeralPortCountTotal);

                // Firewall rules.
                int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
                this.firewallData.Data.Add(firewalls);

                // CPU and Memory.
                // Note: Please make sure you understand the normal state of your nodes
                // with respect to the machine resource use and/or abuse by your service(s).
                // For example, if it is normal for your services to consume 90% of available CPU and memory
                // as part of the work they perform under normal traffic flow, then it doesn't make sense to warn or
                // error on these conditions.
                // TODO: Look into making this a long running background task with signaling.
                TimeSpan duration = TimeSpan.FromSeconds(30);

                if (this.MonitorDuration > TimeSpan.MinValue)
                {
                    duration = this.MonitorDuration;
                }

                // Warn up the counters.
                _ = this.perfCounters.PerfCounterGetProcessorInfo();
                _ = this.perfCounters.PerfCounterGetMemoryInfoMb();

                while (this.stopwatch.Elapsed <= duration)
                {
                    token.ThrowIfCancellationRequested();

                    if (this.CpuWarningUsageThresholdPct > 0 &&
                        this.CpuWarningUsageThresholdPct <= 100)
                    {
                        this.AllCpuTimeData.Data.Add(this.perfCounters.PerfCounterGetProcessorInfo());
                    }

                    if (this.MemWarningUsageThresholdMb > 0)
                    {
                        this.allMemDataCommittedBytes.Data.Add(this.perfCounters.PerfCounterGetMemoryInfoMb());
                    }

                    if (this.MemoryWarningLimitPercent > 0)
                    {
                        this.allMemDataPercentUsed.Data.Add(
                            ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse().PercentInUse);
                    }

                    Thread.Sleep(250);
                }
            }
            catch (OperationCanceledException)
            {
            }
            catch (Exception e)
            {
                this.HealthReporter.ReportFabricObserverServiceHealth(
                    this.FabricServiceContext.ServiceName.OriginalString,
                    this.ObserverName,
                    HealthState.Warning,
                    $"Unhandled exception in GetSystemCpuMemoryValuesAsync:{Environment.NewLine}{e}");

                throw;
            }

            return(Task.CompletedTask);
        }
예제 #3
0
        private async Task MonitorAppAsync(ApplicationInfo application)
        {
            List <ReplicaOrInstanceMonitoringInfo> repOrInstList;

            if (!string.IsNullOrEmpty(application.TargetType))
            {
                repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetType).ConfigureAwait(true);
            }
            else
            {
                repOrInstList = await GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.Target)).ConfigureAwait(true);
            }

            if (repOrInstList.Count == 0)
            {
                ObserverLogger.LogInfo("No target or targetType specified.");
                return;
            }

            Process currentProcess = null;

            foreach (var repOrInst in repOrInstList)
            {
                Token.ThrowIfCancellationRequested();

                int processid = (int)repOrInst.HostProcessId;
                var cpuUsage  = new CpuUsage();

                try
                {
                    // App level.
                    currentProcess = Process.GetProcessById(processid);

                    Token.ThrowIfCancellationRequested();

                    var    procName      = currentProcess.ProcessName;
                    string appNameOrType = GetAppNameOrType(repOrInst);

                    var id = $"{appNameOrType}:{procName}";

                    // Add new resource data structures for each app service process.
                    if (!allAppCpuData.Any(list => list.Id == id))
                    {
                        allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id));
                        allAppMemDataMB.Add(new FabricResourceUsageData <long>(ErrorWarningProperty.TotalMemoryConsumptionMb, id));
                        allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id));
                        allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id));
                        allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id));
                    }

                    // CPU (all cores).
                    int i = Environment.ProcessorCount + 10;

                    while (!currentProcess.HasExited && i > 0)
                    {
                        Token.ThrowIfCancellationRequested();

                        int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);

                        if (cpu >= 0)
                        {
                            allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
                        }

                        // Memory (private working set (process)).
                        var processMem = perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
                        allAppMemDataMB.FirstOrDefault(x => x.Id == id).Data.Add((long)processMem);

                        // Memory (percent in use (total)).
                        var  memInfo  = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
                        long totalMem = memInfo.TotalMemory;

                        if (totalMem > -1)
                        {
                            double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
                            allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(usedPct);
                        }

                        --i;

                        Thread.Sleep(250);
                    }

                    // Total and Ephemeral ports..
                    allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
                    .Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));

                    allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
                    .Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
                }
                catch (Exception e)
                {
                    if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
                    {
                        WriteToLogWithLevel(
                            ObserverName,
                            $"MonitorAsync failed to find current service process for {application.Target}/n{e}",
                            LogLevel.Information);
                    }
                    else
                    {
                        if (!(e is OperationCanceledException))
                        {
                            WriteToLogWithLevel(
                                ObserverName,
                                $"Unhandled exception in MonitorAsync: \n {e}",
                                LogLevel.Warning);
                        }

                        throw;
                    }
                }
                finally
                {
                    currentProcess?.Dispose();
                    currentProcess = null;
                }
            }
        }
예제 #4
0
        private async Task MonitorAppAsync(ApplicationInfo application)
        {
            List <ReplicaOrInstanceMonitoringInfo> repOrInstList;

            if (this.IsTestRun)
            {
                repOrInstList = this.ReplicaOrInstanceList;
            }
            else
            {
                if (!string.IsNullOrEmpty(application.TargetAppType))
                {
                    repOrInstList = await this
                                    .GetDeployedApplicationReplicaOrInstanceListAsync(null, application.TargetAppType)
                                    .ConfigureAwait(true);
                }
                else
                {
                    repOrInstList = await this
                                    .GetDeployedApplicationReplicaOrInstanceListAsync(new Uri(application.TargetApp))
                                    .ConfigureAwait(true);
                }

                if (repOrInstList.Count == 0)
                {
                    this.ObserverLogger.LogInfo("No targetApp or targetAppType specified.");
                    return;
                }
            }

            Process currentProcess = null;

            foreach (var repOrInst in repOrInstList)
            {
                this.Token.ThrowIfCancellationRequested();

                var timer     = new Stopwatch();
                int processId = (int)repOrInst.HostProcessId;
                var cpuUsage  = new CpuUsage();

                try
                {
                    // App level.
                    currentProcess = Process.GetProcessById(processId);

                    this.Token.ThrowIfCancellationRequested();

                    var    procName      = currentProcess.ProcessName;
                    string appNameOrType = GetAppNameOrType(repOrInst);

                    var id = $"{appNameOrType}:{procName}";

                    // Add new resource data structures for each app service process.
                    if (this.allAppCpuData.All(list => list.Id != id))
                    {
                        this.allAppCpuData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalCpuTime, id, this.DataCapacity, this.UseCircularBuffer));
                        this.allAppMemDataMb.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.TotalMemoryConsumptionMb, id, this.DataCapacity, this.UseCircularBuffer));
                        this.allAppMemDataPercent.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.TotalMemoryConsumptionPct, id, this.DataCapacity, this.UseCircularBuffer));
                        this.allAppTotalActivePortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalActivePorts, id, 1));
                        this.allAppEphemeralPortsData.Add(new FabricResourceUsageData <int>(ErrorWarningProperty.TotalEphemeralPorts, id, 1));
                    }

                    TimeSpan duration = TimeSpan.FromSeconds(15);

                    if (this.MonitorDuration > TimeSpan.MinValue)
                    {
                        duration = this.MonitorDuration;
                    }

                    // Warm up the counters.
                    _ = cpuUsage.GetCpuUsageProcess(currentProcess);
                    _ = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);

                    timer.Start();

                    while (!currentProcess.HasExited && timer.Elapsed <= duration)
                    {
                        this.Token.ThrowIfCancellationRequested();

                        // CPU (all cores).
                        int cpu = cpuUsage.GetCpuUsageProcess(currentProcess);

                        if (cpu >= 0)
                        {
                            this.allAppCpuData.FirstOrDefault(x => x.Id == id).Data.Add(cpu);
                        }

                        // Memory (private working set (process)).
                        var processMem = this.perfCounters.PerfCounterGetProcessPrivateWorkingSetMb(currentProcess.ProcessName);
                        this.allAppMemDataMb.FirstOrDefault(x => x.Id == id).Data.Add(processMem);

                        // Memory (percent in use (total)).
                        var  memInfo  = ObserverManager.TupleGetTotalPhysicalMemorySizeAndPercentInUse();
                        long totalMem = memInfo.TotalMemory;

                        if (totalMem > -1)
                        {
                            double usedPct = Math.Round(((double)(processMem * 100)) / (totalMem * 1024), 2);
                            this.allAppMemDataPercent.FirstOrDefault(x => x.Id == id).Data.Add(Math.Round(usedPct, 1));
                        }

                        Thread.Sleep(250);
                    }

                    timer.Stop();
                    timer.Reset();

                    // Total and Ephemeral ports..
                    this.allAppTotalActivePortsData.FirstOrDefault(x => x.Id == id)
                    .Data.Add(NetworkUsage.GetActivePortCount(currentProcess.Id));

                    this.allAppEphemeralPortsData.FirstOrDefault(x => x.Id == id)
                    .Data.Add(NetworkUsage.GetActiveEphemeralPortCount(currentProcess.Id));
                }
                catch (Exception e)
                {
                    if (e is Win32Exception || e is ArgumentException || e is InvalidOperationException)
                    {
                        this.WriteToLogWithLevel(
                            this.ObserverName,
                            $"MonitorAsync failed to find current service process for {application.TargetApp}/n{e}",
                            LogLevel.Information);
                    }
                    else
                    {
                        if (!(e is OperationCanceledException))
                        {
                            this.WriteToLogWithLevel(
                                this.ObserverName,
                                $"Unhandled exception in MonitorAsync: \n {e}",
                                LogLevel.Warning);
                        }

                        throw;
                    }
                }
                finally
                {
                    currentProcess?.Dispose();
                    currentProcess = null;
                }
            }
        }