public void ReportHealthAndLoad(object notused)
{
// Calculate the values and then remember current values for the next report.
long total = Controllers.VotesController._requestCount;
long diff = total - _lastCount;
long duration = Math.Max((long)DateTime.UtcNow.Subtract(_lastReport).TotalSeconds, 1L);
long rps = diff / duration;
_lastCount = total;
_lastReport = DateTime.UtcNow;
// Create the health information for this instance of the service and send report to Service Fabric.
HealthInformation hi = new HealthInformation("VotingServiceHealth", "Heartbeat", HealthState.Ok)
{
TimeToLive = _interval.Add(_interval),
Description = $"{diff} requests since last report. RPS: {rps} Total requests: {total}.",
RemoveWhenExpired = false,
SequenceNumber = HealthInformation.AutoSequenceNumber
};
var sshr = new StatelessServiceInstanceHealthReport(Context.PartitionId, Context.InstanceId, hi);
_client.HealthManager.ReportHealth(sshr);
// Report the load
Partition.ReportLoad(new[] { new LoadMetric("RPS", (int)rps) });
ServiceEventSource.Current.HealthReport(hi.SourceId, hi.Property, Enum.GetName(typeof(HealthState), hi.HealthState), Context.PartitionId, Context.ReplicaOrInstanceId, hi.Description);
}
protected override HealthReport GetHealthReport(HealthInformation healthInformation)
{
HealthReport replicaOrInstanceHealthReport = null;
switch (this.ParameterSetName)
{
/// Replica
case Constants.SendReplicaHealthReportstatefulServiceParamSetName:
replicaOrInstanceHealthReport =
new StatefulServiceReplicaHealthReport(this.PartitionId, this.ReplicaId, healthInformation);
break;
/// Instance
case Constants.SendReplicaHealthReportStatelessServiceParamSetName:
replicaOrInstanceHealthReport =
new StatelessServiceInstanceHealthReport(this.PartitionId, this.InstanceId, healthInformation);
break;
default:
throw new ArgumentException(string.Format(
CultureInfo.CurrentCulture,
StringResources.Error_ReplicaHealthReportUnknownParamSet,
ParameterSetName));
}
return(replicaOrInstanceHealthReport);
}
public void ReportHealthAndLoad(object notused)
{
// Calculate the values and then remember current values for the next report.
long total = Controllers.VotesController._requestCount;
long diff = total - _lastCount;
long duration = Math.Max((long)DateTime.UtcNow.Subtract(_lastReport).TotalSeconds, 1L);
long rps = diff / duration;
_lastCount = total;
_lastReport = DateTime.UtcNow;
// Create the health information for this instance of the service and send report to Service Fabric.
HealthInformation hi = new HealthInformation("VotingServiceHealth", "Heartbeat", HealthState.Ok)
{
TimeToLive = _interval.Add(_interval),
Description = $"{diff} requests since last report. RPS: {rps} Total requests: {total}.",
RemoveWhenExpired = false,
SequenceNumber = HealthInformation.AutoSequenceNumber
};
var sshr = new StatelessServiceInstanceHealthReport(Context.PartitionId, Context.InstanceId, hi);
_client.HealthManager.ReportHealth(sshr);
// Report the load
Partition.ReportLoad(new[] { new LoadMetric("RPS", (int)rps) });
ServiceEventSource.Current.HealthReport(
hi.SourceId, hi.Property, Enum.GetName(typeof(HealthState), hi.HealthState),
Context.PartitionId,
Context.ReplicaOrInstanceId,
hi.Description);
// Report failing health report to cause rollback
var nodeList = _client.QueryManager.
GetNodeListAsync(Context.NodeContext.NodeName).GetAwaiter().GetResult();
var node = nodeList[0];
if ("4" == node.UpgradeDomain || "3" == node.UpgradeDomain || "2" == node.UpgradeDomain)
{
hi = new HealthInformation(
"VotingServiceHealth",
"Error_Heartbeat",
HealthState.Error);
hi.TimeToLive = _interval.Add(_interval);
hi.Description = $"Bogus health error to force rollback.";
hi.RemoveWhenExpired = true;
hi.SequenceNumber = HealthInformation.AutoSequenceNumber;
sshr = new StatelessServiceInstanceHealthReport(
Context.PartitionId,
Context.InstanceId,
hi);
_client.HealthManager.ReportHealth(sshr);
}
}
private void ReportReplicaHealth(
ServiceFabricDiscoveryOptions options,
ServiceWrapper service,
Guid partitionId,
ReplicaWrapper replica,
HealthState state,
string description = null)
{
if (!options.ReportReplicasHealth)
{
return;
}
var healthInformation = new HealthInformation(
sourceId: HealthReportSourceId,
property: HealthReportProperty,
healthState: state)
{
Description = description,
TimeToLive = HealthReportTimeToLive(options),
RemoveWhenExpired = true,
};
HealthReport healthReport;
switch (service.ServiceKind)
{
case ServiceKind.Stateful:
healthReport = new StatefulServiceReplicaHealthReport(
partitionId: partitionId,
replicaId: replica.Id,
healthInformation: healthInformation);
break;
case ServiceKind.Stateless:
healthReport = new StatelessServiceInstanceHealthReport(
partitionId: partitionId,
instanceId: replica.Id,
healthInformation: healthInformation);
break;
default:
Log.ReplicaHealthReportFailedInvalidServiceKind(_logger, state, replica.Id, service.ServiceKind);
return;
}
var sendOptions = new HealthReportSendOptions { Immediate = state != HealthState.Ok }; // Report immediately if unhealthy
try
{
_serviceFabricCaller.ReportHealth(healthReport, sendOptions);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.ReplicaHealthReportFailed(_logger, state, replica.Id, ex);
}
}
private void SendHealthReport(InstanceIdentifier instanceIdentifier, HealthState state)
{
var healthInformation = new HealthInformation("Watchdog", "Healthcheck", state)
{
TimeToLive = TimeSpan.FromSeconds(30)
};
var healthReport = new StatelessServiceInstanceHealthReport(instanceIdentifier.PartitionId,
instanceIdentifier.InstanceId, healthInformation);
_fabricClient.HealthManager.ReportHealth(healthReport);
}
/// <summary>
/// Returns a health report
/// </summary>
/// <param name="context">The service fabric context that the health report is for</param>
/// <param name="reportSourceId">The unique reporting source id</param>
/// <param name="propertyName">The name of the health property being reported on</param>
/// <param name="state">The current state of the health property</param>
/// <param name="timeToLive">The time to live of the health report</param>
/// <param name="reportType">The entity type the report is for</param>
/// <returns>A health report for the appropriate reporting entity</returns>
public static HealthReport GetHealthReport(ServiceContext context, string reportSourceId, string propertyName, HealthState state, ReportTypes reportType, TimeSpan timeToLive)
{
HealthReport report;
var information = new HealthInformation(reportSourceId, propertyName, state);
information.Description = $"{ propertyName } health state { Enum.GetName(typeof(HealthState), state) }";
information.RemoveWhenExpired = true;
information.TimeToLive = timeToLive;
information.SequenceNumber = HealthInformation.AutoSequenceNumber;
switch (reportType)
{
case ReportTypes.Cluster:
report = new ClusterHealthReport(information);
break;
case ReportTypes.Application:
report = new ApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), information);
break;
case ReportTypes.DeployedApplication:
report = new DeployedApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.NodeContext.NodeName, information);
break;
case ReportTypes.Service:
report = new ServiceHealthReport(context.ServiceName, information);
break;
case ReportTypes.DeployedService:
report = new DeployedServicePackageHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.CodePackageActivationContext.GetServiceManifestName(), context.NodeContext.NodeName, information);
break;
case ReportTypes.Node:
report = new NodeHealthReport(context.NodeContext.NodeName, information);
break;
case ReportTypes.Instance:
if (context is StatelessServiceContext)
{
report = new StatelessServiceInstanceHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
}
else
{
report = new StatefulServiceReplicaHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
}
break;
default:
throw new ArgumentException("Unknown health type", nameof(reportType));
}
return(report);
}
/// <summary>
/// Posts a health report against the current service instance of Patch Orchestration Agent
/// </summary>
/// <param name="fabricClient">Fabric client to carry out HealthManager operation on cluster</param>
/// <param name="serviceContext">Context of the current service</param>
/// <param name="sourceId">SourceId for health report</param>
/// <param name="healthReportProperty">Property of the health report</param>
/// <param name="description">description of the health report</param>
/// <param name="healthState">HealthState for the health report</param>
/// <param name="timeToLiveInMinutes">Time to live for health report</param>
internal static void PostServiceInstanceHealthReport(FabricClient fabricClient, ServiceContext serviceContext, string sourceId, string healthReportProperty, string description,
HealthState healthState, long timeToLiveInMinutes = -1)
{
HealthInformation healthInformation = new HealthInformation(sourceId, healthReportProperty,
healthState);
healthInformation.RemoveWhenExpired = true;
healthInformation.Description = description;
if (timeToLiveInMinutes >= 0)
{
healthInformation.TimeToLive = TimeSpan.FromMinutes(timeToLiveInMinutes);
}
StatelessServiceInstanceHealthReport serviceInstanceHealthReport =
new StatelessServiceInstanceHealthReport(serviceContext.PartitionId,
serviceContext.ReplicaOrInstanceId, healthInformation);
fabricClient.HealthManager.ReportHealth(serviceInstanceHealthReport);
}
public Task UpdateStatusAsync(string serviceName, string instance, string subStatusName, HealthStatus status, string message)
{
var healthInfo = new HealthInformation(
GetType().FullName,
subStatusName,
MapStatus(status))
{
Description = message
};
System.Fabric.Health.HealthReport report;
if (instance == null)
{
report = new ServiceHealthReport(
GetServiceUri(),
healthInfo
);
}
else if (_isStateful)
{
report = new StatefulServiceReplicaHealthReport(
_context.PartitionId,
_context.ReplicaOrInstanceId,
healthInfo
);
}
else
{
report = new StatelessServiceInstanceHealthReport(
_context.PartitionId,
_context.ReplicaOrInstanceId,
healthInfo
);
}
_fabricClient.HealthManager.ReportHealth(report, new HealthReportSendOptions {
Immediate = true
});
return(Task.CompletedTask);
}
/// <summary>
/// This function generates Service Fabric Health Reports that will show up in SFX.
/// </summary>
/// <param name="healthReport">Utilities.HealthReport instance.</param>
public void ReportHealthToServiceFabric(HealthReport healthReport)
{
if (healthReport == null)
{
return;
}
// There is no real need to change Immediate to true here for errors/warnings. This only adds unecessary stress to the
// Health subsystem.
var sendOptions = new HealthReportSendOptions {
Immediate = false
};
// Quickly send OK (clears warning/errors states).
if (healthReport.State == HealthState.Ok)
{
sendOptions.Immediate = true;
}
var timeToLive = TimeSpan.FromMinutes(5);
if (healthReport.HealthReportTimeToLive != default)
{
timeToLive = healthReport.HealthReportTimeToLive;
}
TelemetryData healthData = healthReport.HealthData;
string errWarnPreamble = string.Empty;
if (healthReport.State == HealthState.Error ||
healthReport.State == HealthState.Warning)
{
errWarnPreamble =
$"{healthReport.Observer} detected " +
$"{Enum.GetName(typeof(HealthState), healthReport.State)} threshold breach. ";
// OSObserver does not monitor resources and therefore does not support related usage threshold configuration.
if (healthReport.Observer == ObserverConstants.OSObserverName &&
healthReport.Property == "OSConfiguration")
{
errWarnPreamble = $"{ObserverConstants.OSObserverName} detected potential problem with OS configuration: ";
}
}
string message = $"{errWarnPreamble}{healthReport.HealthMessage}";
if (healthData != null)
{
message = JsonConvert.SerializeObject(healthData);
}
if (string.IsNullOrEmpty(healthReport.SourceId))
{
healthReport.SourceId = healthReport.Observer;
}
if (string.IsNullOrEmpty(healthReport.Property))
{
switch (healthReport.Observer)
{
case ObserverConstants.AppObserverName:
healthReport.Property = "ApplicationHealth";
break;
case ObserverConstants.CertificateObserverName:
healthReport.Property = "SecurityHealth";
break;
case ObserverConstants.DiskObserverName:
healthReport.Property = "DiskHealth";
break;
case ObserverConstants.FabricSystemObserverName:
healthReport.Property = "FabricSystemServiceHealth";
break;
case ObserverConstants.NetworkObserverName:
healthReport.Property = "NetworkHealth";
break;
case ObserverConstants.OSObserverName:
healthReport.Property = "MachineInformation";
break;
case ObserverConstants.NodeObserverName:
healthReport.Property = "MachineResourceHealth";
break;
default:
healthReport.Property = $"{healthReport.Observer}_HealthProperty";
break;
}
;
}
var healthInformation = new HealthInformation(healthReport.SourceId, healthReport.Property, healthReport.State)
{
Description = $"{message}",
TimeToLive = timeToLive,
RemoveWhenExpired = true,
};
// Log health event locally.
if (healthReport.EmitLogEvent)
{
if (healthReport.State == HealthState.Error)
{
this.logger.LogError(healthReport.NodeName + ": {0}", healthInformation.Description);
}
else if (healthReport.State == HealthState.Warning)
{
this.logger.LogWarning(healthReport.NodeName + ": {0}", healthInformation.Description);
}
else
{
this.logger.LogInfo(healthReport.NodeName + ": {0}", healthInformation.Description);
}
}
// To SFX.
if (healthReport.ReportType == HealthReportType.Application && healthReport.AppName != null)
{
var appHealthReport = new ApplicationHealthReport(healthReport.AppName, healthInformation);
this.fabricClient.HealthManager.ReportHealth(appHealthReport, sendOptions);
}
else if (healthReport.ReportType == HealthReportType.Service && healthReport.ServiceName != null)
{
var serviceHealthReport = new ServiceHealthReport(healthReport.ServiceName, healthInformation);
this.fabricClient.HealthManager.ReportHealth(serviceHealthReport, sendOptions);
}
else if (healthReport.ReportType == HealthReportType.StatefulService &&
healthReport.PartitionId != Guid.Empty && healthReport.ReplicaOrInstanceId > 0)
{
var statefulServiceHealthReport = new StatefulServiceReplicaHealthReport(healthReport.PartitionId, healthReport.ReplicaOrInstanceId, healthInformation);
this.fabricClient.HealthManager.ReportHealth(statefulServiceHealthReport, sendOptions);
}
else if (healthReport.ReportType == HealthReportType.StatelessService &&
healthReport.PartitionId != Guid.Empty && healthReport.ReplicaOrInstanceId > 0)
{
var statelessServiceHealthReport = new StatelessServiceInstanceHealthReport(healthReport.PartitionId, healthReport.ReplicaOrInstanceId, healthInformation);
this.fabricClient.HealthManager.ReportHealth(statelessServiceHealthReport, sendOptions);
}
else if (healthReport.ReportType == HealthReportType.Partition && healthReport.PartitionId != Guid.Empty)
{
var partitionHealthReport = new PartitionHealthReport(healthReport.PartitionId, healthInformation);
this.fabricClient.HealthManager.ReportHealth(partitionHealthReport, sendOptions);
}
else if (healthReport.ReportType == HealthReportType.DeployedApplication && healthReport.AppName != null)
{
var deployedApplicationHealthReport = new DeployedApplicationHealthReport(healthReport.AppName, healthReport.NodeName, healthInformation);
this.fabricClient.HealthManager.ReportHealth(deployedApplicationHealthReport, sendOptions);
}
else
{
var nodeHealthReport = new NodeHealthReport(healthReport.NodeName, healthInformation);
this.fabricClient.HealthManager.ReportHealth(nodeHealthReport, sendOptions);
}
}
/// <summary>
/// Returns a health report
/// </summary>
/// <param name="context">The service fabric context that the health report is for</param>
/// <param name="reportSourceId">The unique reporting source id</param>
/// <param name="propertyName">The name of the health property being reported on</param>
/// <param name="state">The current state of the health property</param>
/// <param name="timeToLive">The time to live of the health report</param>
/// <param name="reportType">The entity type the report is for</param>
/// <returns>A health report for the appropriate reporting entity</returns>
public static HealthReport GetHealthReport(ServiceContext context, string reportSourceId, string propertyName, HealthState state, ReportTypes reportType, TimeSpan timeToLive)
{
HealthReport report;
var information = new HealthInformation(reportSourceId, propertyName, state);
information.Description = $"{ propertyName } health state { Enum.GetName(typeof(HealthState), state) }";
information.RemoveWhenExpired = true;
information.TimeToLive = timeToLive;
information.SequenceNumber = HealthInformation.AutoSequenceNumber;
switch (reportType)
{
case ReportTypes.Cluster:
report = new ClusterHealthReport(information);
break;
case ReportTypes.Application:
report = new ApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), information);
break;
case ReportTypes.DeployedApplication:
report = new DeployedApplicationHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.NodeContext.NodeName, information);
break;
case ReportTypes.Service:
report = new ServiceHealthReport(context.ServiceName, information);
break;
case ReportTypes.DeployedService:
report = new DeployedServicePackageHealthReport(new Uri(context.CodePackageActivationContext.ApplicationName), context.CodePackageActivationContext.GetServiceManifestName(), context.NodeContext.NodeName, information);
break;
case ReportTypes.Node:
report = new NodeHealthReport(context.NodeContext.NodeName, information);
break;
case ReportTypes.Instance:
if (context is StatelessServiceContext)
{
report = new StatelessServiceInstanceHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
}
else
{
report = new StatefulServiceReplicaHealthReport(context.PartitionId, context.ReplicaOrInstanceId, information);
}
break;
default:
throw new ArgumentException("Unknown health type", nameof(reportType));
}
return report;
}