예제 #1
0
        /// <inheritdoc/>
        public override Task ReportAsync(CancellationToken token)
        {
            this.Token.ThrowIfCancellationRequested();
            var timeToLiveWarning     = this.SetTimeToLiveWarning(this.runtime.Seconds);
            var portInformationReport = new Utilities.HealthReport
            {
                Observer      = this.ObserverName,
                NodeName      = this.NodeName,
                HealthMessage = $"Number of ports in use by Fabric services: {this.TotalActivePortCount}\n" +
                                $"Number of ephemeral ports in use by Fabric services: {this.TotalActiveEphemeralPortCount}",
                State = HealthState.Ok,
                HealthReportTimeToLive = timeToLiveWarning,
            };

            // TODO: Report on port count based on thresholds PortCountWarning/Error...
            this.HealthReporter.ReportHealthToServiceFabric(portInformationReport);

            // Reset ports counters...
            this.TotalActivePortCount          = 0;
            this.TotalActiveEphemeralPortCount = 0;

            // CPU
            this.ProcessResourceDataList(
                this.allCpuData,
                this.CpuErrorUsageThresholdPct,
                this.CpuWarnUsageThresholdPct);

            // Memory
            this.ProcessResourceDataList(
                this.allMemData,
                this.MemErrorUsageThresholdMB,
                this.MemWarnUsageThresholdMB);

            // Disk IO - Reads
            this.ProcessResourceDataList(
                this.allAppDiskReadsData,
                this.DiskErrorIOReadsThresholdMS,
                this.DiskWarnIOReadsThresholdMS);

            // Disk IO - Writes
            this.ProcessResourceDataList(
                this.allAppDiskWritesData,
                this.DiskErrorIOWritesThresholdMS,
                this.DiskWarnIOWritesThresholdMS);

            // Windows Event Log
            if (this.monitorWinEventLog)
            {
                // SF Eventlog Errors?
                // Write this out to a new file, for use by the web front end log viewer...
                // Format = HTML...
                int count   = this.evtRecordList.Count();
                var logPath = Path.Combine(this.ObserverLogger.LogFolderBasePath, "EventVwrErrors.txt");

                // Remove existing file...
                if (File.Exists(logPath))
                {
                    try
                    {
                        File.Delete(logPath);
                    }
                    catch (IOException)
                    {
                    }
                    catch (UnauthorizedAccessException)
                    {
                    }
                }

                if (count >= 10)
                {
                    var sb = new StringBuilder();

                    sb.AppendLine("<br/><div><strong>" +
                                  "<a href='javascript:toggle(\"evtContainer\")'>" +
                                  "<div id=\"plus\" style=\"display: inline; font-size: 25px;\">+</div> " + count +
                                  " Error Events in ServiceFabric and System</a> " +
                                  "Event logs</strong>.<br/></div>");

                    sb.AppendLine("<div id='evtContainer' style=\"display: none;\">");

                    foreach (var evt in this.evtRecordList.Distinct())
                    {
                        token.ThrowIfCancellationRequested();

                        try
                        {
                            // Access event properties:
                            sb.AppendLine("<div>" + evt.LogName + "</div>");
                            sb.AppendLine("<div>" + evt.LevelDisplayName + "</div>");
                            if (evt.TimeCreated.HasValue)
                            {
                                sb.AppendLine("<div>" + evt.TimeCreated.Value.ToShortDateString() + "</div>");
                            }

                            foreach (var prop in evt.Properties)
                            {
                                if (prop.Value != null && Convert.ToString(prop.Value).Length > 0)
                                {
                                    sb.AppendLine("<div>" + prop.Value + "</div>");
                                }
                            }
                        }
                        catch (EventLogException)
                        {
                        }
                    }

                    sb.AppendLine("</div>");

                    this.ObserverLogger.TryWriteLogFile(logPath, sb.ToString());
                    sb.Clear();
                }

                // Clean up...
                if (count > 0)
                {
                    this.evtRecordList.Clear();
                }
            }

            return(Task.CompletedTask);
        }
예제 #2
0
        /// <inheritdoc/>
        public override Task ReportAsync(CancellationToken token)
        {
            try
            {
                token.ThrowIfCancellationRequested();

                // OS Health...
                if (this.osStatus != null &&
                    this.osStatus.ToUpper() != "OK")
                {
                    string healthMessage = $"OS reporting unhealthy: {this.osStatus}";
                    var healthReport = new Utilities.HealthReport
                    {
                        Observer = this.ObserverName,
                        NodeName = this.NodeName,
                        HealthMessage = healthMessage,
                        State = HealthState.Error,
                        HealthReportTimeToLive = this.SetTimeToLiveWarning(),
                    };

                    this.HealthReporter.ReportHealthToServiceFabric(healthReport);

                    // This means this observer created a Warning or Error SF Health Report
                    this.HasActiveFabricErrorOrWarning = true;

                    // Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example...)...
                    if (this.IsTelemetryEnabled)
                    {
                        _ = this.ObserverTelemetryClient?.ReportHealthAsync(
                                FabricRuntime.GetActivationContext().ApplicationName,
                                this.FabricServiceContext.ServiceName.OriginalString,
                                "FabricObserver",
                                this.ObserverName,
                                $"{this.NodeName}/OS reporting unhealthy: {this.osStatus}",
                                HealthState.Error,
                                token);
                    }
                }
                else if (this.HasActiveFabricErrorOrWarning &&
                         this.osStatus != null &&
                         this.osStatus.ToUpper() == "OK")
                {
                    // Clear Error or Warning with an OK Health Report...
                    string healthMessage = $"OS reporting healthy: {this.osStatus}";
                    var healthReport = new Utilities.HealthReport
                    {
                        Observer = this.ObserverName,
                        NodeName = this.NodeName,
                        HealthMessage = healthMessage,
                        State = HealthState.Ok,
                        HealthReportTimeToLive = default(TimeSpan),
                    };

                    this.HealthReporter.ReportHealthToServiceFabric(healthReport);

                    // Reset internal health state...
                    this.HasActiveFabricErrorOrWarning = false;
                }

                var logPath = Path.Combine(this.ObserverLogger.LogFolderBasePath, "SysInfo.txt");

                // This file is used by the web application (log reader...)...
                if (!this.ObserverLogger.TryWriteLogFile(logPath, $"Last updated on {DateTime.UtcNow.ToString("M/d/yyyy HH:mm:ss")} UTC<br/>{this.osReport}"))
                {
                    this.HealthReporter.ReportFabricObserverServiceHealth(
                        this.FabricServiceContext.ServiceName.OriginalString,
                        this.ObserverName,
                        HealthState.Warning,
                        "Unable to create SysInfo.txt file...");
                }

                var osReport = new Utilities.HealthReport
                {
                    Observer = this.ObserverName,
                    HealthMessage = this.osReport,
                    State = HealthState.Ok,
                    NodeName = this.NodeName,
                    HealthReportTimeToLive = this.SetTimeToLiveWarning(),
                };

                this.HealthReporter.ReportHealthToServiceFabric(osReport);

                return Task.CompletedTask;
            }
            catch (Exception e)
            {
                this.HealthReporter.ReportFabricObserverServiceHealth(
                    this.FabricServiceContext.ServiceName.OriginalString,
                    this.ObserverName,
                    HealthState.Error,
                    $"Unhandled exception processing OS information: {e.Message}: \n {e.StackTrace}");
                throw;
            }
        }
예제 #3
0
        /// <inheritdoc/>
        public override async Task ReportAsync(CancellationToken token)
        {
            string app;
            var    timeToLiveWarning = this.SetTimeToLiveWarning();

            // Report on connection state.
            for (int j = 0; j < this.userEndpoints.Count; j++)
            {
                token.ThrowIfCancellationRequested();

                var deployedApps = await this.FabricClientInstance.QueryManager
                                   .GetDeployedApplicationListAsync(
                    this.NodeName,
                    new Uri(this.userEndpoints[j].AppTarget)).ConfigureAwait(true);

                // We only care about deployed apps.
                if (deployedApps == null || deployedApps.Count < 1)
                {
                    continue;
                }

                app = this.userEndpoints[j].AppTarget.Replace("fabric:/", string.Empty);

                for (int i = 0; i < this.connectionStatus.Count; i++)
                {
                    token.ThrowIfCancellationRequested();

                    var connStatus = this.connectionStatus[i];

                    if (!connStatus.Connected)
                    {
                        this.healthState = HealthState.Warning;
                        var healthMessage = "Outbound Internet connection failure detected for endpoint " + connStatus.HostName + "\n";

                        Utilities.HealthReport report = new Utilities.HealthReport
                        {
                            AppName                   = new Uri(this.userEndpoints[j].AppTarget),
                            Code                      = FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable,
                            EmitLogEvent              = true,
                            HealthMessage             = healthMessage,
                            HealthReportTimeToLive    = timeToLiveWarning,
                            State                     = this.healthState,
                            NodeName                  = this.NodeName,
                            Observer                  = this.ObserverName,
                            ReportType                = HealthReportType.Application,
                            ResourceUsageDataProperty = $"{ErrorWarningProperty.InternetConnectionFailure}: connStatus.HostName",
                        };

                        // Send health report Warning and log event locally.
                        this.HealthReporter.ReportHealthToServiceFabric(report);

                        // This means this observer created a Warning or Error SF Health Report
                        this.HasActiveFabricErrorOrWarning = true;

                        // Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example.).
                        if (this.IsTelemetryEnabled)
                        {
                            _ = this.ObserverTelemetryClient?.ReportHealthAsync(
                                Utilities.Telemetry.HealthScope.Application,
                                this.userEndpoints[j].AppTarget,
                                HealthState.Warning,
                                $"{this.NodeName}/{FOErrorWarningCodes.AppWarningNetworkEndpointUnreachable}: {healthMessage}",
                                this.ObserverName,
                                this.Token);
                        }
                    }
                    else
                    {
                        if (connStatus.Health == HealthState.Warning)
                        {
                            this.healthState = HealthState.Ok;
                            var healthMessage = "Outbound Internet connection test successful.";

                            // Clear existing Health Warning.
                            Utilities.HealthReport report = new Utilities.HealthReport
                            {
                                AppName                = new Uri(this.userEndpoints[j].AppTarget),
                                EmitLogEvent           = true,
                                HealthMessage          = healthMessage,
                                HealthReportTimeToLive = default(TimeSpan),
                                State      = this.healthState,
                                NodeName   = this.NodeName,
                                Observer   = this.ObserverName,
                                ReportType = HealthReportType.Application,
                            };

                            this.HealthReporter.ReportHealthToServiceFabric(report);

                            // Reset health state.
                            this.HasActiveFabricErrorOrWarning = false;
                        }
                    }
                }
            }

            // Clear
            this.connectionStatus.RemoveAll(conn => conn.Connected == true);
            this.connectionStatus.TrimExcess();
        }
예제 #4
0
        public void ProcessResourceDataReportHealth <T>(
            FabricResourceUsageData <T> data,
            T thresholdError,
            T thresholdWarning,
            TimeSpan healthReportTtl,
            HealthReportType healthReportType = HealthReportType.Node,
            string app = null,
            ReplicaMonitoringInfo replicaOrInstance = null,
            bool dumpOnError = false)
        {
            if (data == null)
            {
                throw new ArgumentException("Supply all required parameters with non-null value...");
            }

            var    thresholdName = "Minimum";
            bool   warningOrError = false;
            string repPartitionId = null, repOrInstanceId = null, name = null, id = null, procName = null;
            T      threshold   = thresholdWarning;
            var    healthState = HealthState.Ok;
            Uri    appName     = null;

            if (replicaOrInstance != null)
            {
                repPartitionId  = $"Partition: {replicaOrInstance.Partitionid}";
                repOrInstanceId = $"Replica: {replicaOrInstance.ReplicaOrInstanceId}";
                procName        = Process.GetProcessById((int)replicaOrInstance.ReplicaHostProcessId)?.ProcessName;
            }

            // Create a unique node id which may be used in the case of warnings or OK clears...
            if (app != null)
            {
                appName = new Uri(app);
                name    = app.Replace("fabric:/", string.Empty);
                id      = name + "_" + data.Property.Replace(" ", string.Empty);
            }

            // Telemetry...
            if (this.IsTelemetryEnabled)
            {
                _ = this.ObserverTelemetryClient?.ReportMetricAsync($"{this.NodeName}-{app}-{data.Id}-{data.Property}", data.AverageDataValue, this.Token);
            }

            // ETW...
            if (this.IsEtwEnabled)
            {
                Logger.EtwLogger?.Write(
                    $"FabricObserverDataEvent",
                    new
                {
                    Level    = 0,  // Info
                    Node     = this.NodeName,
                    Observer = this.ObserverName,
                    Property = data.Property,
                    Id       = data.Id,
                    Value    = $"{Math.Round(data.AverageDataValue)}",
                    Unit     = data.Units,
                });
            }

            // Health Error
            if (data.IsUnhealthy(thresholdError))
            {
                thresholdName  = "Maximum";
                threshold      = thresholdError;
                warningOrError = true;
                healthState    = HealthState.Error;

                // This is primarily useful for AppObserver, but makes sense to be
                // part of the base class for future use, like for FSO...
                if (replicaOrInstance != null && procName != null && dumpOnError)
                {
                    try
                    {
                        int procId = (int)replicaOrInstance.ReplicaHostProcessId;

                        if (!this.serviceDumpCountDictionary.ContainsKey(procName))
                        {
                            this.serviceDumpCountDictionary.Add(procName, 0);
                        }

                        if (this.serviceDumpCountDictionary[procName] < this.maxDumps)
                        {
                            // DumpServiceProcess defaults to a Full dump with
                            // process memory, handles and thread data...
                            bool success = this.DumpServiceProcess(procId);

                            if (success)
                            {
                                this.serviceDumpCountDictionary[procName]++;
                            }
                        }
                    }

                    // Ignore these, it just means no dmp will be created.This is not
                    // critical to FO... Log as info, not warning...
                    catch (ArgumentException ae)
                    {
                        this.ObserverLogger.LogInfo($"Unable to generate dmp file:\n{ae.ToString()}");
                    }
                    catch (InvalidOperationException ioe)
                    {
                        this.ObserverLogger.LogInfo($"Unable to generate dmp file:\n{ioe.ToString()}");
                    }
                }
            }

            // Health Warning
            if (!warningOrError && data.IsUnhealthy(thresholdWarning))
            {
                warningOrError = true;
                healthState    = HealthState.Warning;
            }

            if (warningOrError)
            {
                string errorWarningKind = null;

                if (data.Property.ToLower().Contains("cpu"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorCpuTime : ErrorWarningCode.WarningCpuTime;
                }
                else if (data.Property.ToLower().Contains("disk space"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorDiskSpace : ErrorWarningCode.WarningDiskSpace;
                }
                else if (data.Property == "Memory Consumption MB")
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorMemoryCommitted : ErrorWarningCode.WarningMemoryCommitted;
                }
                else if (data.Property == "Memory Consumption %")
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorMemoryPercentUsed : ErrorWarningCode.WarningMemoryPercentUsed;
                }
                else if (data.Property.Contains("Read"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorDiskIoReads : ErrorWarningCode.WarningDiskIoReads;
                }
                else if (data.Property.Contains("Write"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorDiskIoWrites : ErrorWarningCode.WarningDiskIoWrites;
                }
                else if (data.Property.Contains("Queue"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorDiskAverageQueueLength : ErrorWarningCode.WarningDiskAverageQueueLength;
                }
                else if (data.Property.Contains("Firewall"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorTooManyFirewallRules : ErrorWarningCode.WarningTooManyFirewallRules;
                }
                else if (data.Property.Contains("Ports"))
                {
                    errorWarningKind = (healthState == HealthState.Error) ? ErrorWarningCode.ErrorTooManyActivePorts : ErrorWarningCode.WarningTooManyActivePorts;
                }

                var healthMessage = new StringBuilder();

                if (name != null)
                {
                    healthMessage.Append($"{name} (Service Process: {procName}, {repPartitionId}, {repOrInstanceId}): ");
                }

                healthMessage.Append($"{data.Property} is at or above the specified {thresholdName} limit ({threshold}{data.Units})");
                healthMessage.AppendLine($" - Average {data.Property}: {Math.Round(data.AverageDataValue)}{data.Units}");

                // Set internal fabric health states...
                data.ActiveErrorOrWarning = true;

                // This means this observer created a Warning or Error SF Health Report
                this.HasActiveFabricErrorOrWarning = true;

                var healthReport = new Utilities.HealthReport
                {
                    AppName                = appName,
                    Code                   = errorWarningKind,
                    EmitLogEvent           = true,
                    HealthMessage          = healthMessage.ToString(),
                    HealthReportTimeToLive = healthReportTtl,
                    ReportType             = healthReportType,
                    State                  = healthState,
                    NodeName               = this.NodeName,
                    Observer               = this.ObserverName,
                };

                // Emit a Fabric Health Report and optionally a local log write...
                this.HealthReporter.ReportHealthToServiceFabric(healthReport);

                // Send Health Report as Telemetry event (perhaps it signals an Alert from App Insights, for example...)...
                if (this.IsTelemetryEnabled)
                {
                    _ = this.ObserverTelemetryClient?.ReportHealthAsync(
                        id,
                        this.FabricServiceContext.ServiceName.OriginalString,
                        "FabricObserver",
                        this.ObserverName,
                        $"{this.NodeName}/{errorWarningKind}/{data.Property}/{Math.Round(data.AverageDataValue)}",
                        healthState,
                        this.Token);
                }

                // ETW...
                if (this.IsEtwEnabled)
                {
                    Logger.EtwLogger?.Write(
                        $"FabricObserverDataEvent",
                        new
                    {
                        Level                  = (healthState == HealthState.Warning) ? 1 : 2,
                        Node                   = this.NodeName,
                        Observer               = this.ObserverName,
                        HealthEventErrorCode   = errorWarningKind,
                        HealthEventDescription = healthMessage.ToString(),
                        Property               = data.Property,
                        Id    = data.Id,
                        Value = $"{Math.Round(data.AverageDataValue)}",
                        Unit  = data.Units,
                    });
                }

                // Clean up sb...
                healthMessage.Clear();
            }
            else
            {
                if (data.ActiveErrorOrWarning)
                {
                    Utilities.HealthReport report = new Utilities.HealthReport
                    {
                        AppName                = appName,
                        EmitLogEvent           = true,
                        HealthMessage          = $"{data.Id}: {data.Property} is now within normal/expected range.",
                        HealthReportTimeToLive = default(TimeSpan),
                        ReportType             = healthReportType,
                        State    = HealthState.Ok,
                        NodeName = this.NodeName,
                        Observer = this.ObserverName,
                    };

                    // Emit an Ok Health Report to clear Fabric Health warning...
                    this.HealthReporter.ReportHealthToServiceFabric(report);

                    // Reset health states...
                    data.ActiveErrorOrWarning          = false;
                    this.HasActiveFabricErrorOrWarning = false;
                }
            }

            // No need to keep data in memory...
            data.Data.Clear();
            data.Data.TrimExcess();
        }
예제 #5
0
        /// <inheritdoc/>
        public override Task ReportAsync(CancellationToken token)
        {
            if (token.IsCancellationRequested)
            {
                return(Task.CompletedTask);
            }

            // Someone calling without observing first, must be run after a new run of ObserveAsync
            if (this.ExpiringWarnings == null ||
                this.ExpiredWarnings == null ||
                this.NotFoundWarnings == null)
            {
                return(Task.CompletedTask);
            }

            HealthReport healthReport;

            if (this.ExpiringWarnings.Count == 0 &&
                this.ExpiredWarnings.Count == 0 &&
                this.NotFoundWarnings.Count == 0)
            {
                healthReport = new HealthReport
                {
                    Observer               = this.ObserverName,
                    ReportType             = HealthReportType.Node,
                    EmitLogEvent           = true,
                    NodeName               = this.NodeName,
                    HealthMessage          = $"All cluster and monitored app certificates are healthy.",
                    State                  = HealthState.Ok,
                    HealthReportTimeToLive = this.RunInterval > TimeSpan.MinValue ? this.RunInterval : this.HealthReportTimeToLive,
                };

                this.HasActiveFabricErrorOrWarning = false;
            }
            else
            {
                string healthMessage = (this.ExpiredWarnings.Count == 0 ? string.Empty : (this.ExpiredWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j) + "\n")) +
                                       (this.NotFoundWarnings.Count == 0 ? string.Empty : (this.NotFoundWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j) + "\n")) +
                                       (this.ExpiringWarnings.Count == 0 ? string.Empty : this.ExpiringWarnings.Aggregate(string.Empty, (i, j) => i + "\n" + j));

                healthReport = new HealthReport
                {
                    Code                   = FOErrorWarningCodes.WarningCertificateExpiration,
                    Observer               = this.ObserverName,
                    ReportType             = HealthReportType.Node,
                    EmitLogEvent           = true,
                    NodeName               = this.NodeName,
                    HealthMessage          = healthMessage,
                    State                  = HealthState.Warning,
                    HealthReportTimeToLive = this.RunInterval > TimeSpan.MinValue ? this.RunInterval : this.HealthReportTimeToLive,
                };

                this.HasActiveFabricErrorOrWarning = true;
            }

            this.HealthReporter.ReportHealthToServiceFabric(healthReport);

            this.ExpiredWarnings  = null;
            this.ExpiringWarnings = null;
            this.NotFoundWarnings = null;

            this.LastRunDateTime = DateTime.Now;

            return(Task.CompletedTask);
        }
        public void ReportHealthToServiceFabric(HealthReport healthReport)
        {
            if (healthReport == null)
            {
                return;
            }

            // There is no real need to change Immediate to true here for errors/warnings. This only adds unecessary stress to the
            // Health subsystem.
            var sendOptions = new HealthReportSendOptions {
                Immediate = false
            };

            // Quickly send OK (clears warning/errors states).
            if (healthReport.State == HealthState.Ok)
            {
                sendOptions.Immediate = true;
            }

            var timeToLive = TimeSpan.FromMinutes(5);

            if (healthReport.HealthReportTimeToLive != default)
            {
                timeToLive = healthReport.HealthReportTimeToLive;
            }

            // In order for multiple Error/Warning/Ok events to show up in SFX Details view from observer instances,
            // Event Source Ids must be unique, thus the seemingly strange conditionals inside the cases below:
            // The apparent duplicity in OR checks is for the case when the incoming report is an OK report, where there is
            // no error code, but the specific ErrorWarningProperty is known.
            string property;

            switch (healthReport.Observer)
            {
            case ObserverConstants.AppObserverName:
                property = "AppHealth";
                break;

            case ObserverConstants.CertificateObserverName:
                property = "SecurityHealth";
                break;

            case ObserverConstants.DiskObserverName:
                property = "DiskHealth";
                break;

            case ObserverConstants.FabricSystemObserverName:
                property = "FabricSystemServiceHealth";
                break;

            case ObserverConstants.NetworkObserverName:
                property = "NetworkingHealth";
                break;

            case ObserverConstants.OSObserverName:
                property = "MachineInformation";
                break;

            case ObserverConstants.NodeObserverName:
                property = "MachineResourceHealth";
                break;

            default:
                property = "FOGenericHealth";
                break;
            }

            string sourceId = healthReport.Observer;

            if (!string.IsNullOrEmpty(healthReport.Code))
            {
                // Only use FOErrorWarningCode for source
                sourceId = $"{healthReport.Code}";
            }

            var healthInformation = new HealthInformation(sourceId, property, healthReport.State)
            {
                Description       = healthReport.HealthMessage,
                TimeToLive        = timeToLive,
                RemoveWhenExpired = true,
            };

            // Log event only if ObserverWebApi (REST Log reader.) app is deployed.
            if (ObserverManager.ObserverWebAppDeployed &&
                healthReport.EmitLogEvent)
            {
                if (healthReport.State == HealthState.Error)
                {
                    this.logger.LogError(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else if (healthReport.State == HealthState.Warning)
                {
                    this.logger.LogWarning(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
                else
                {
                    this.logger.LogInfo(healthReport.NodeName + ": {0}", healthInformation.Description);
                }
            }

            // To SFX and Telemetry provider.
            if (healthReport.ReportType == HealthReportType.Application && healthReport.AppName != null)
            {
                var appHealthReport = new ApplicationHealthReport(healthReport.AppName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(appHealthReport, sendOptions);
            }
            else
            {
                var nodeHealthReport = new NodeHealthReport(healthReport.NodeName, healthInformation);
                this.fabricClient.HealthManager.ReportHealth(nodeHealthReport, sendOptions);
            }
        }