Beispiel #1
0
        private void ReportHealth(string property, HealthState state, string description = Source)
        {
            LogHelper.Log("Report health for node={0}; source={1};property={2};state={3};description={4}", NodeName, Source, property, state.ToString(), description);
            HealthInformation healthInformation = new HealthInformation(Source, property, state);

            healthInformation.Description = description;
            HealthReport healthReport = new NodeHealthReport(NodeName, healthInformation);

            try
            {
                HealthManager.ReportHealth(healthReport);
            }
            catch (Exception exception)
            {
                Assert.Fail("Report health throws exception {0}", exception.Message);
            }

            bool healthEventUpdated = false;

            while (!healthEventUpdated)
            {
                Thread.Sleep(HealthUpdateDelayTime);
                NodeHealth nodeHealth = HealthManager.GetNodeHealthAsync(NodeName).Result;
                var        eventQuery = from healthEvent in nodeHealth.HealthEvents
                                        where healthEvent.HealthInformation.SourceId == Source &&
                                        healthEvent.HealthInformation.Property == property &&
                                        healthEvent.HealthInformation.HealthState == state
                                        select healthEvent;
                healthEventUpdated = eventQuery.Any();
            }
        }
        public void SendReportForNode(HealthState healthState, string message)
        {
            var error   = "";
            var handler = HandlersFactory.GetProfilerHandler(_settingService, _loggerService);

            handler.Start(LOG_TAG, "SendReportForNode", null);

            try
            {
                HealthReport healthReport = new NodeHealthReport(_nodeName,
                                                                 new HealthInformation(_serviceName, message, healthState));

                healthReport.HealthInformation.TimeToLive        = TimeSpan.FromMinutes(_settingService.GetHealthIssuesTimeToLive());
                healthReport.HealthInformation.RemoveWhenExpired = false;
                ReportHealth(healthReport);

                SendReportForPartition(healthState, message);
            }
            catch (Exception ex)
            {
                error = ex.Message;
                /* Ignore */
            }
            finally
            {
                handler.Stop(error);
            }
        }
        /// <summary>
        /// Returns a health report
        /// </summary>
        /// <param name="context">The service fabric context that the health report is for</param>
        /// <param name="reportSourceId">The unique reporting source id</param>
        /// <param name="propertyName">The name of the health property being reported on</param>
        /// <param name="state">The current state of the health property</param>
        /// <param name="timeToLive">The time to live of the health report</param>
        /// <param name="reportType">The entity type the report is for</param>
        /// <returns>A health report for the appropriate reporting entity</returns>
        public static HealthReport GetHealthReport(ServiceContext context, string reportSourceId, string propertyName, HealthState state, ReportTypes reportType, TimeSpan timeToLive)
        {
            HealthReport report;
            var          information = new HealthInformation(reportSourceId, propertyName, state);

            information.Description       = $"{ propertyName } health state { Enum.GetName(typeof(HealthState), state) }";
            information.RemoveWhenExpired = true;
            information.TimeToLive        = timeToLive;
            information.SequenceNumber    = HealthInformation.AutoSequenceNumber;

            switch (reportType)
            {
            case ReportTypes.Cluster:
                report = new ClusterHealthReport(information);
                break;

            case ReportTypes.Application:
                report = new ApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), information);
                break;

            case ReportTypes.DeployedApplication:
                report = new DeployedApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.NodeContext.NodeName, information);
                break;

            case ReportTypes.Service:
                report = new ServiceHealthReport(context.ServiceName, information);
                break;

            case ReportTypes.DeployedService:
                report = new DeployedServicePackageHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.CodePackageActivationContext.GetServiceManifestName(), context.NodeContext.NodeName, information);
                break;

            case ReportTypes.Node:
                report = new NodeHealthReport(context.NodeContext.NodeName, information);
                break;

            case ReportTypes.Instance:
                if (context is StatelessServiceContext)
                {
                    report = new StatelessServiceInstanceHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
                }
                else
                {
                    report = new StatefulServiceReplicaHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
                }
                break;

            default:
                throw new ArgumentException("Unknown health type", nameof(reportType));
            }

            return(report);
        }
Beispiel #4
0
        private void SendHealthReport(NodeHealthReport healthReport)
        {
            if (this.client == null)
            {
                this.client = new FabricClient();
            }

            this.client.HealthManager.ReportHealth(healthReport);
            ServiceEventSource.Current.ServiceMessage(
                this.Context,
                $"Reported {healthReport.HealthInformation.HealthState} on {healthReport.NodeName} for patch {healthReport.HealthInformation.Property}");
        }
Beispiel #5
0
        private void AddReport(string nodeName, string key, string value, HealthState healthState)
        {
            // Don't specify a time to live here. Because if infrastructure service is down, then
            // all role instances (nodes in WinFab terms) are reported as down in the health report.
            // i.e. it appears as though the whole cluster is down. In that case, WinFab upgrades
            // etc. won't work since they perform checks on health status.
            // Instead, we are specifying a time to live only at the service health level
            var hi = new HealthInformation(HealthConstants.HealthReportSourceId, key, healthState)
            {
                Description = value
            };

            var healthReport = new NodeHealthReport(nodeName, hi);

            healthClient.ReportHealth(healthReport);
        }
Beispiel #6
0
        private void ReportNodeHealth(string property, string nodeName, HealthState healthState, string description)
        {
            var healthInfo = new HealthInformation(
                InfrastructureService.Constants.HealthReportSourceId,
                property,
                healthState)
            {
                Description       = description,
                TimeToLive        = this.configReader.NodeHealthEventTTL,
                RemoveWhenExpired = true,
            };

            var report = new NodeHealthReport(nodeName, healthInfo);

            this.healthClient.ReportHealth(report);
        }
Beispiel #7
0
        private static void InternalSendNodeHealthReport(
            string description,
            HealthState healthState,
            string subProperty,
            TimeSpan timeToLive,
            bool removeWhenExpired)
        {
            if (Disabled)
            {
                return;
            }

            InitializeClient();
            var property = string.IsNullOrEmpty(subProperty) ? Property : string.Join(".", Property, subProperty);

            HealthInformation healthInfo;

            if (timeToLive != TimeSpan.Zero)
            {
                healthInfo = new HealthInformation(SourceId, property, healthState)
                {
                    Description       = description,
                    TimeToLive        = timeToLive,
                    RemoveWhenExpired = removeWhenExpired
                };
            }
            else
            {
                healthInfo = new HealthInformation(SourceId, property, healthState)
                {
                    Description       = description,
                    TimeToLive        = timeToLive,
                    RemoveWhenExpired = removeWhenExpired
                };
            }

            var nodeHealthReport = new NodeHealthReport(Utility.FabricNodeName, healthInfo);

            try
            {
                fabricClient.HealthManager.ReportHealth(nodeHealthReport);
            }
            catch (Exception e)
            {
                Utility.TraceSource.WriteExceptionAsWarning(TraceType, e);
            }
        }
Beispiel #8
0
        private void ReportInstalledPatch(string patchId, string installedOn)
        {
            var healthInformation = new HealthInformation(
                HealthReportSource,
                patchId,
                HealthState.Ok)
            {
                Description       = $"Patch {patchId} was installed on {installedOn}",
                RemoveWhenExpired = this.settings.HealthReport.RemoveWhenExpired,
                TimeToLive        = TimeSpan.FromSeconds(this.settings.HealthReport.TimeToLiveSeconds),
                SequenceNumber    = 0
            };

            var healthReport = new NodeHealthReport(this.Context.NodeContext.NodeName, healthInformation);

            this.SendHealthReport(healthReport);
        }
Beispiel #9
0
        private void ReportMissingPatch(string patchId)
        {
            var healthInformation = new HealthInformation(
                HealthReportSource,
                patchId,
                this.settings.HealthReport.ReportMissingAs)
            {
                Description       = $"Patch {patchId} is NOT installed.",
                RemoveWhenExpired = this.settings.HealthReport.RemoveWhenExpired,
                TimeToLive        = TimeSpan.FromSeconds(this.settings.HealthReport.TimeToLiveSeconds),
                SequenceNumber    = 0
            };

            var healthReport = new NodeHealthReport(this.Context.NodeContext.NodeName, healthInformation);

            this.SendHealthReport(healthReport);
        }
        public void ReportHealthToServiceFabric(HealthReport healthReport)
        {
            if (healthReport == null)
            {
                return;
            }

            // There is no real need to change Immediate to true here for errors/warnings. This only adds unecessary stress to the
            // Health subsystem.
            var sendOptions = new HealthReportSendOptions {
                Immediate = false
            };

            // Quickly send OK (clears warning/errors states).
            if (healthReport.State == HealthState.Ok)
            {
                sendOptions.Immediate = true;
            }

            var timeToLive = TimeSpan.FromMinutes(5);

            if (healthReport.HealthReportTimeToLive != default)
            {
                timeToLive = healthReport.HealthReportTimeToLive;
            }

            // In order for multiple Error/Warning/Ok events to show up in SFX Details view from observer instances,
            // Event Source Ids must be unique, thus the seemingly strange conditionals inside the cases below:
            // The apparent duplicity in OR checks is for the case when the incoming report is an OK report, where there is
            // no error code, but the specific ErrorWarningProperty is known.
            string property;

            switch (healthReport.Observer)
            {
            case ObserverConstants.AppObserverName:
                property = "AppHealth";
                break;

            case ObserverConstants.CertificateObserverName:
                property = "SecurityHealth";
                break;

            case ObserverConstants.DiskObserverName:
                property = "DiskHealth";
                break;

            case ObserverConstants.FabricSystemObserverName:
                property = "FabricSystemServiceHealth";
                break;

            case ObserverConstants.NetworkObserverName:
                property = "NetworkingHealth";
                break;

            case ObserverConstants.OsObserverName:
                property = "MachineInformation";
                break;

            case ObserverConstants.NodeObserverName:
                property = "MachineResourceHealth";
                break;

            default:
                property = "FOGenericHealth";
                break;
            }

            string sourceId = healthReport.Observer;

            if (!string.IsNullOrEmpty(healthReport.Code))
            {
                // Only use FOErrorWarningCode for source
                sourceId = $"{healthReport.Code}";
            }

            var healthInformation = new HealthInformation(sourceId, property, healthReport.State)
            {
                Description       = healthReport.HealthMessage,
                TimeToLive        = timeToLive,
                RemoveWhenExpired = true,
            };

            // Log event only if ObserverWebApi (REST Log reader.) app is deployed.
            if (ObserverManager.ObserverWebAppDeployed &&
                healthReport.EmitLogEvent)
            {
                if (healthReport.State == HealthState.Error)
                {
                    this.logger.LogError(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else if (healthReport.State == HealthState.Warning)
                {
                    this.logger.LogWarning(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else
                {
                    this.logger.LogInfo(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
            }

            // To SFX and Telemetry provider.
            if (healthReport.ReportType == HealthReportType.Application && healthReport.AppName != null)
            {
                var appHealthReport = new ApplicationHealthReport(healthReport.AppName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(appHealthReport, sendOptions);
            }
            else
            {
                var nodeHealthReport = new NodeHealthReport(healthReport.NodeName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(nodeHealthReport, sendOptions);
            }
        }
        public void ReportHealthToServiceFabric(HealthReport healthReport)
        {
            if (healthReport == null)
            {
                return;
            }

            // There is no real need to change Immediate to true here for errors/warnings. This only adds unecessary stress to the
            // Health subsystem.
            var sendOptions = new HealthReportSendOptions {
                Immediate = false
            };

            // Quickly send OK (clears warning/errors states).
            if (healthReport.State == HealthState.Ok)
            {
                sendOptions.Immediate = true;
            }

            var timeToLive = TimeSpan.FromMinutes(5);

            if (healthReport.HealthReportTimeToLive != default)
            {
                timeToLive = healthReport.HealthReportTimeToLive;
            }

            // Set property for health event.
            string property = healthReport.Property;

            if (string.IsNullOrEmpty(property))
            {
                switch (healthReport.Observer)
                {
                case ObserverConstants.AppObserverName:
                    property = "ApplicationHealth";
                    break;

                case ObserverConstants.CertificateObserverName:
                    property = "SecurityHealth";
                    break;

                case ObserverConstants.DiskObserverName:
                    property = "DiskHealth";
                    break;

                case ObserverConstants.FabricSystemObserverName:
                    property = "FabricSystemServiceHealth";
                    break;

                case ObserverConstants.NetworkObserverName:
                    property = "NetworkHealth";
                    break;

                case ObserverConstants.OsObserverName:
                    property = "MachineInformation";
                    break;

                case ObserverConstants.NodeObserverName:
                    property = "MachineResourceHealth";
                    break;

                default:
                    property = "FOGenericHealth";
                    break;
                }
            }

            string        sourceId   = healthReport.Observer;
            TelemetryData healthData = healthReport.HealthData;

            if (!string.IsNullOrEmpty(healthReport.Code))
            {
                // Only use FOErrorWarningCode for source
                sourceId += $"({healthReport.Code})";
            }

            string errWarnPreamble = string.Empty;

            if (healthReport.State == HealthState.Error ||
                healthReport.State == HealthState.Warning)
            {
                errWarnPreamble =
                    $"{healthReport.Observer} detected " +
                    $"{Enum.GetName(typeof(HealthState), healthReport.State)} threshold breach. ";

                // OSObserver does not monitor resources and therefore does not support related usage threshold configuration.
                if (healthReport.Observer == ObserverConstants.OsObserverName &&
                    property == "OSConfiguration")
                {
                    errWarnPreamble = $"{ObserverConstants.OsObserverName} detected potential problem with OS configuration: ";
                    property        = "OSConfiguration";
                }
            }

            string message = $"{errWarnPreamble}{healthReport.HealthMessage}";

            if (healthData != null)
            {
                message = JsonConvert.SerializeObject(healthData);
            }

            var healthInformation = new HealthInformation(sourceId, property, healthReport.State)
            {
                Description       = $"{message}",
                TimeToLive        = timeToLive,
                RemoveWhenExpired = true,
            };

            // Log event only if ObserverWebApi (REST API Log reader service) app is deployed.
            if (ObserverManager.ObserverWebAppDeployed &&
                healthReport.EmitLogEvent)
            {
                if (healthReport.State == HealthState.Error)
                {
                    this.logger.LogError(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else if (healthReport.State == HealthState.Warning)
                {
                    this.logger.LogWarning(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else
                {
                    this.logger.LogInfo(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
            }

            // To SFX.
            if (healthReport.ReportType == HealthReportType.Application && healthReport.AppName != null)
            {
                var appHealthReport = new ApplicationHealthReport(healthReport.AppName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(appHealthReport, sendOptions);
            }
            else
            {
                var nodeHealthReport = new NodeHealthReport(healthReport.NodeName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(nodeHealthReport, sendOptions);
            }
        }
Beispiel #12
0
        public void ReportHealthToServiceFabric(Utilities.HealthReport healthReport)
        {
            if (healthReport == null)
            {
                return;
            }

            // There is no real need to change Immediate to true here for errors/warnings. This only adds unecessary stress to the
            // Health subsystem...
            var sendOptions = new HealthReportSendOptions {
                Immediate = false
            };

            // Quickly send OK (clears warning/errors states)...
            if (healthReport.State == HealthState.Ok)
            {
                sendOptions.Immediate = true;
            }

            var timeToLive = TimeSpan.FromMinutes(5);

            if (healthReport.HealthReportTimeToLive != default(TimeSpan))
            {
                timeToLive = healthReport.HealthReportTimeToLive;
            }

            string kind = string.Empty;

            if (healthReport.Code != null)
            {
                kind = healthReport.Code + ": ";
            }

            string source = healthReport.Observer;
            string property;

            // In order for multiple Error/Warning/Ok events to show up in SFX Details view from observer instances,
            // Event Source Ids must be unique, thus the seemingly strange conditionals inside the cases below:
            // The apparent duplicity in OR checks is for the case when the incoming report is an OK report, where there is
            // no error code, but the specific ErrorWarningProperty is known...
            switch (healthReport.Observer)
            {
            case ObserverConstants.AppObserverName:
                property = "AppHealth";

                if (healthReport.Code == ErrorWarningCode.WarningCpuTime ||
                    healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalCpuTime)
                {
                    source += "(CPU)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryPercentUsed ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionPct)
                {
                    source += "(Memory%)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryCommitted ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionMB)
                {
                    source += "(MemoryMB)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyActiveEphemeralPorts ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalEphemeralPorts)
                {
                    source += "(ActiveEphemeralPorts)";
                }

                break;

            case ObserverConstants.CertificateObserverName:
                property = "SecurityHealth";
                break;

            case ObserverConstants.DiskObserverName:
                property = "DiskHealth";

                if (healthReport.Code == ErrorWarningCode.WarningDiskAverageQueueLength ||
                    healthReport.ResourceUsageDataProperty == ErrorWarningProperty.DiskAverageQueueLength)
                {
                    source += "(DiskQueueLength)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningDiskSpacePercentUsed ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.DiskSpaceUsagePercentage)
                {
                    source += "(DiskSpace%)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningDiskSpaceMB ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.DiskSpaceUsageMB)
                {
                    source += "(DiskSpaceMB)";
                }

                break;

            case ObserverConstants.FabricSystemObserverName:
                property = "FabricSystemServiceHealth";

                if (healthReport.Code == ErrorWarningCode.WarningCpuTime ||
                    healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalCpuTime)
                {
                    source += "(CPU)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryPercentUsed ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionPct)
                {
                    source += "(Memory%)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryCommitted ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionMB)
                {
                    source += "(MemoryMB)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyActiveTcpPorts ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalActivePorts)
                {
                    source += "(ActivePorts)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyActiveEphemeralPorts ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalEphemeralPorts)
                {
                    source += "(ActiveEphemeralPorts)";
                }

                break;

            case ObserverConstants.NetworkObserverName:
                property = "NetworkingHealth";
                break;

            case ObserverConstants.OSObserverName:
                property = "MachineInformation";
                break;

            case ObserverConstants.NodeObserverName:
                property = "MachineResourceHealth";

                if (healthReport.Code == ErrorWarningCode.WarningCpuTime ||
                    healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalCpuTime)
                {
                    source += "(CPU)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyFirewallRules ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalActiveFirewallRules)
                {
                    source += "(FirewallRules)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryPercentUsed ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionPct)
                {
                    source += "(Memory%)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningMemoryCommitted ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalMemoryConsumptionMB)
                {
                    source += "(MemoryMB)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyActiveTcpPorts ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalActivePorts)
                {
                    source += "(ActivePorts)";
                }
                else if (healthReport.Code == ErrorWarningCode.WarningTooManyActiveEphemeralPorts ||
                         healthReport.ResourceUsageDataProperty == ErrorWarningProperty.TotalEphemeralPorts)
                {
                    source += "(ActiveEphemeralPorts)";
                }

                break;

            default:
                property = "FOGenericHealth";
                break;
            }

            var healthInformation = new HealthInformation(source, property, healthReport.State)
            {
                Description       = kind + healthReport.HealthMessage,
                TimeToLive        = timeToLive,
                RemoveWhenExpired = true,
            };

            // Log event only if ObserverWebApi (REST Log reader...) app is deployed...
            if (ObserverManager.ObserverWebAppDeployed &&
                healthReport.EmitLogEvent)
            {
                if (healthReport.State == HealthState.Error)
                {
                    this.logger.LogError(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else if (healthReport.State == HealthState.Warning)
                {
                    this.logger.LogWarning(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else
                {
                    this.logger.LogInfo(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
            }

            // To SFX and Telemetry provider...
            if (healthReport.ReportType == HealthReportType.Application && healthReport.AppName != null)
            {
                var report = new ApplicationHealthReport(healthReport.AppName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(report, sendOptions);
            }
            else
            {
                var report = new NodeHealthReport(healthReport.NodeName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(report, sendOptions);
            }
        }
Beispiel #13
0
        /// <summary>
        /// Reports the health of a node.
        /// </summary>
        /// <param name="ttl"></param>
        public async Task ReportNodeHealthAndLoadAsync(TimeSpan ttl)
        {
            const int         MB  = 1048576;
            HealthInformation hi  = null;
            NodeHealthReport  nhr = null;

            try
            {
                // Get the global memory load and report as a node health parameter.
                NativeMethods.MEMORYSTATUSEX msex = new NativeMethods.MEMORYSTATUSEX();
                if (NativeMethods.GlobalMemoryStatus(ref msex))
                {
                    HealthState hs = (msex.dwMemoryLoad > 80) ? HealthState.Warning : (msex.dwMemoryLoad > 95) ? HealthState.Error : HealthState.Ok;

                    // Save the current memory load.
                    MemoryLoad = msex.dwMemoryLoad;

                    // Create the health information to report to Service Fabric.
                    hi                   = new HealthInformation("NodeHealth", "MemoryLoad", hs);
                    hi.TimeToLive        = (0.0 <= ttl.TotalMilliseconds) ? TimeSpan.FromSeconds(30) : ttl;
                    hi.Description       = $"Percent of memory in used on this node: {msex.dwMemoryLoad}";
                    hi.RemoveWhenExpired = true;
                    hi.SequenceNumber    = HealthInformation.AutoSequenceNumber;

                    // Create a replica health report.
                    nhr = new NodeHealthReport(Context.NodeContext.NodeName, hi);
                    ServiceFabricClient.HealthManager.ReportHealth(nhr);
                }

                // Create the health information and send report to Service Fabric.
                hi                   = new HealthInformation("NodeHealth", "CPU", HealthState.Ok);
                hi.TimeToLive        = (0.0 <= ttl.TotalMilliseconds) ? TimeSpan.FromSeconds(30) : ttl;
                hi.Description       = $"Total CPU usage on this node: {_cpuCounter.NextValue()}";
                hi.RemoveWhenExpired = true;
                hi.SequenceNumber    = HealthInformation.AutoSequenceNumber;
                nhr                  = new NodeHealthReport(Context.NodeContext.NodeName, hi);
                ServiceFabricClient.HealthManager.ReportHealth(nhr);

                // Get the number of deployed replicas on this node for this service.
                int serviceReplicaCount = 0;
                var replicaList         = await ServiceFabricClient.QueryManager.GetDeployedReplicaListAsync(Context.NodeContext.NodeName, Application);

                for (int i = 0; i < replicaList.Count; i++)
                {
                    if (Context.ServiceName == replicaList[i].ServiceName)
                    {
                        serviceReplicaCount++;
                    }
                }

                DateTimeOffset oldSampleTime = _timeOfCpuSample;
                TimeSpan       oldCpuSample  = _cpuProcessTime;
                _cpuProcessTime  = Process.GetCurrentProcess().TotalProcessorTime;
                _timeOfCpuSample = DateTimeOffset.UtcNow;

                long processTicks        = (_cpuProcessTime - oldCpuSample).Ticks;
                long periodTicks         = (_timeOfCpuSample - oldSampleTime).Ticks;
                long cpuTicks            = (processTicks / periodTicks);
                long cpuPercent          = (cpuTicks / serviceReplicaCount) * 100;
                long partitionWorkingSet = ((Process.GetCurrentProcess().WorkingSet64 / MB) / serviceReplicaCount);

                // Report the partition load metrics.
                LoadMetric[] metrics = new LoadMetric[]
                {
                    new LoadMetric("PartitionCPU", (int)cpuPercent),
                    new LoadMetric("WorkingSetMB", Convert.ToInt32(partitionWorkingSet))
                };

                ReportLoad(metrics);
            }
            catch (Exception ex)
            {
                _eventSource.ServiceRequestFailed(Context.ServiceTypeName, Context.PartitionId, Context.ReplicaId, "ReportNodeHealthAndLoadAsync", ex.Message);
            }
        }
        /// <summary>
        /// Returns a health report
        /// </summary>
        /// <param name="context">The service fabric context that the health report is for</param>
        /// <param name="reportSourceId">The unique reporting source id</param>
        /// <param name="propertyName">The name of the health property being reported on</param>
        /// <param name="state">The current state of the health property</param>
        /// <param name="timeToLive">The time to live of the health report</param>
        /// <param name="reportType">The entity type the report is for</param>
        /// <returns>A health report for the appropriate reporting entity</returns>
        public static HealthReport GetHealthReport(ServiceContext context, string reportSourceId, string propertyName, HealthState state, ReportTypes reportType, TimeSpan timeToLive)
        {
            HealthReport report;
            var information = new HealthInformation(reportSourceId, propertyName, state);

            information.Description = $"{ propertyName } health state { Enum.GetName(typeof(HealthState), state) }";
            information.RemoveWhenExpired = true;
            information.TimeToLive = timeToLive;
            information.SequenceNumber = HealthInformation.AutoSequenceNumber;

            switch (reportType)
            {
                case ReportTypes.Cluster:
                    report = new ClusterHealthReport(information);
                    break;

                case ReportTypes.Application:
                    report = new ApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), information);
                    break;

                case ReportTypes.DeployedApplication:
                    report = new DeployedApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.NodeContext.NodeName, information);
                    break;

                case ReportTypes.Service:
                    report = new ServiceHealthReport(context.ServiceName, information);
                    break;

                case ReportTypes.DeployedService:
                    report = new DeployedServicePackageHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.CodePackageActivationContext.GetServiceManifestName(), context.NodeContext.NodeName, information);
                    break;

                case ReportTypes.Node:
                    report = new NodeHealthReport(context.NodeContext.NodeName, information);
                    break;

                case ReportTypes.Instance:
                    if (context is StatelessServiceContext)
                    {
                        report = new StatelessServiceInstanceHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
                    }
                    else
                    {
                        report = new StatefulServiceReplicaHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
                    }
                    break;

                default:
                    throw new ArgumentException("Unknown health type", nameof(reportType));
            }

            return report;
        }