public void TestApplcationWithNullDeltaHealthPolicy()
{
CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper());
PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy();
clusterUpgradePolicy.DeltaHealthPolicy = null;
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(
totalApplicationCount: 6,
totalNodeCount: 5,
unhealthyApplicationsCount: 0,
unhealthyNodeCount: 0);
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
clusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
VerifyCommandProcessorClusterUpgradeDescription(
clusterUpgradePolicy,
result,
maxPercentageUnhealthyApplication: 100,
expectedtMaxPercentUnhealthyPerServicesType: null);
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ClusterHealth obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
writer.WriteProperty(obj.AggregatedHealthState, "AggregatedHealthState", HealthStateConverter.Serialize);
if (obj.HealthEvents != null)
{
writer.WriteEnumerableProperty(obj.HealthEvents, "HealthEvents", HealthEventConverter.Serialize);
}
if (obj.UnhealthyEvaluations != null)
{
writer.WriteEnumerableProperty(obj.UnhealthyEvaluations, "UnhealthyEvaluations", HealthEvaluationWrapperConverter.Serialize);
}
if (obj.HealthStatistics != null)
{
writer.WriteProperty(obj.HealthStatistics, "HealthStatistics", HealthStatisticsConverter.Serialize);
}
if (obj.NodeHealthStates != null)
{
writer.WriteEnumerableProperty(obj.NodeHealthStates, "NodeHealthStates", NodeHealthStateConverter.Serialize);
}
if (obj.ApplicationHealthStates != null)
{
writer.WriteEnumerableProperty(obj.ApplicationHealthStates, "ApplicationHealthStates", ApplicationHealthStateConverter.Serialize);
}
writer.WriteEndObject();
}
public void ClusterHealthSerializationTest()
{
ClusterHealth clusterHealth = this.random.CreateRandom <ClusterHealth>();
var appHS = new ApplicationHealthState()
{
ApplicationName = new Uri("fabric:/" + "testAppManifest" + this.random.CreateRandom <string>()),
AggregatedHealthState = this.random.CreateRandom <HealthState>()
};
var nodeHS = new NodeHealthState()
{
NodeName = "NodeName_" + this.random.CreateRandom <string>(),
AggregatedHealthState = this.random.CreateRandom <HealthState>()
};
clusterHealth.ApplicationHealthStates = new List <ApplicationHealthState>()
{
appHS
};
clusterHealth.NodeHealthStates = new List <NodeHealthState>()
{
nodeHS
};
TestUsingSerializer(this.Serializer, clusterHealth);
}
/// <summary>
/// Reports the overall cluster health to the telemetry provider.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task ReportClusterHealthAsync(CancellationToken cancellationToken)
{
// Called from RunAsync, don't let an exception out so the service will start, but log the exception because the service won't work.
try
{
ClusterHealth health = await _client.HealthManager.GetClusterHealthAsync(TimeSpan.FromSeconds(4), cancellationToken);
if (null != health)
{
// Report the aggregated cluster health.
await
this._telemetry.ReportHealthAsync(
this.Context.ServiceName.AbsoluteUri,
this.Context.PartitionId.ToString(),
this.Context.ReplicaOrInstanceId.ToString(),
"Cluster",
"Aggregated Cluster Health",
health.AggregatedHealthState,
cancellationToken);
// Get the state of each of the applications running within the cluster. Report anything that is unhealthy.
foreach (ApplicationHealthState appHealth in health.ApplicationHealthStates)
{
if (HealthState.Ok != appHealth.AggregatedHealthState)
{
await
this._telemetry.ReportHealthAsync(
appHealth.ApplicationName.AbsoluteUri,
this.Context.ServiceName.AbsoluteUri,
this.Context.PartitionId.ToString(),
this.Context.ReplicaOrInstanceId.ToString(),
this.Context.NodeContext.NodeName,
appHealth.AggregatedHealthState,
cancellationToken);
}
}
// Get the state of each of the nodes running within the cluster.
foreach (NodeHealthState nodeHealth in health.NodeHealthStates)
{
if (HealthState.Ok != nodeHealth.AggregatedHealthState)
{
await
this._telemetry.ReportHealthAsync(
this.Context.NodeContext.NodeName,
this.Context.ServiceName.AbsoluteUri,
this.Context.PartitionId.ToString(),
this.Context.NodeContext.NodeType,
this.Context.NodeContext.IPAddressOrFQDN,
nodeHealth.AggregatedHealthState,
cancellationToken);
}
}
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Error($"Exception: {ex.Message} at {ex.StackTrace}.");
}
}
private async Task ProcessClusterUpgradeAsync(
PaasClusterUpgradeDescription upgradeDesc,
CancellationToken token)
{
var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, token);
var clusterHealthQueryTask = this.fabricClientWrapper.GetClusterHealthAsync(Constants.MaxOperationTimeout, token);
await Task.WhenAll(
upgradeProgressTask,
clusterHealthQueryTask);
FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask);
ClusterHealth currentClusterHealth = GetResultFromTask(clusterHealthQueryTask);
ClusterUpgradeCommandParameter upgradeCommandParameter = null;
try
{
upgradeCommandParameter = await this.commandParameterGenerator.GetCommandParameterAsync(
currentUpgradeProgress,
currentClusterHealth,
upgradeDesc,
token);
await UpgradeClusterAsync(upgradeCommandParameter, token);
}
finally
{
Cleanup(upgradeCommandParameter);
}
}
private static async Task DumpClusterHealthAsync(FabricClient fc)
{
FabricClient.HealthClient hm = fc.HealthManager;
ClusterHealth clusterHealth = await hm.GetClusterHealthAsync();
WriteLine($"Cluster: State={clusterHealth.AggregatedHealthState}");
foreach (HealthEvent healthEvent in clusterHealth.HealthEvents)
{
healthEvent.WriteHealth();
}
foreach (HealthEvaluation healthEval in clusterHealth.UnhealthyEvaluations)
{
WriteLine(healthEval);
}
foreach (NodeHealthState nodeHealth in clusterHealth.NodeHealthStates)
{
WriteLine($"Node: State={nodeHealth.AggregatedHealthState}, Name={nodeHealth.NodeName}");
}
foreach (ApplicationHealthState appHealthState in clusterHealth.ApplicationHealthStates)
{
WriteLine($"App: State={appHealthState.AggregatedHealthState}, Name={appHealthState.ApplicationName}");
}
//await hm.GetNodeHealthAsync()
ApplicationHealth appHealth = await hm.GetApplicationHealthAsync(new Uri(@"fabric:/"));
WriteLine($"App: State={appHealth.AggregatedHealthState}, Name={appHealth.ApplicationName}");
foreach (HealthEvent healthEvent in appHealth.HealthEvents)
{
healthEvent.WriteHealth();
}
}
public async Task <ClusterInfo> Info()
{
NodeList nodes = await this.query.GetNodesAsync();
ApplicationTypeList appTypes = await this.query.GetApplicationTypesAsync();
ApplicationList applications = await this.query.GetApplicationsAsync();
ClusterLoadInformation clusterLoadInfo = await this.query.GetClusterLoadAsync();
ClusterHealth clusterHealth = await this.query.GetClusterHealthAsync();
ProvisionedFabricCodeVersion version = await this.query.GetFabricVersion();
long serviceCount = 0;
long partitionCount = 0;
long replicaCount = 0;
foreach (Node node in nodes)
{
DeployedApplicationList deployedApplicationList = await this.query.GetDeployedApplicationsAsync(node.NodeName);
foreach (DeployedApplication deployedApplication in deployedApplicationList)
{
DeployedServiceReplicaList deployedReplicas =
await this.query.GetDeployedReplicasAsync(node.NodeName, deployedApplication.ApplicationName);
replicaCount += deployedReplicas.Count;
}
}
foreach (Application application in applications)
{
ServiceList services = await this.query.GetServicesAsync(application.ApplicationName);
serviceCount += services.Count;
}
return(new ClusterInfo(
clusterHealth.AggregatedHealthState.ToString(),
version != null ? version.CodeVersion : "not based",
nodes.Select(x => x.NodeType).Distinct().Count(),
appTypes.Count(),
nodes.Select(x => x.FaultDomain.ToString()).Distinct().Count(),
nodes.Select(x => x.UpgradeDomain).Distinct().Count(),
nodes.Count,
applications.Count,
serviceCount,
partitionCount, // TODO: partition count
replicaCount,
clusterLoadInfo.LastBalancingStartTimeUtc,
clusterLoadInfo.LastBalancingEndTimeUtc));
}
/// <summary>
/// Overloaded ToString function for formatting the output on the console.
/// </summary>
/// <param name="clusterHealth"> Object of type ClusterHealth </param>
/// <returns>
/// Returns formatted string.
/// </returns>
public static string ToString(ClusterHealth clusterHealth)
{
var strBuilder = new StringBuilder();
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "AggregatedHealthState", clusterHealth.AggregatedHealthState));
strBuilder.Append(Environment.NewLine);
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "NodeHealthStates", OutputFormatter.ToString(clusterHealth.NodeHealthStates.ToList())));
strBuilder.Append(Environment.NewLine);
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "ApplicationHealthStates", OutputFormatter.ToString(clusterHealth.ApplicationHealthStates.ToList())));
strBuilder.Append(Environment.NewLine);
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "HealthEvents", OutputFormatter.ToString(clusterHealth.HealthEvents.ToList())));
strBuilder.Append(Environment.NewLine);
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "HealthStatistics", OutputFormatter.ToString(clusterHealth.HealthStatistics)));
strBuilder.Append(Environment.NewLine);
return(strBuilder.ToString());
}
/// <summary>
/// Compute cluster health
/// </summary>
protected virtual void ComputeClusterHealth()
{
var clusterHealth = new ClusterHealth();
foreach (var performanceCounter in _performanceCounters.ToList())
{
if (!clusterHealth.PerformanceCounters.ContainsKey(performanceCounter.Key) &&
!clusterHealth.PerformanceCounters.TryAdd(performanceCounter.Key,
Convert.ToInt64(performanceCounter.Value)))
{
Logger.LogWarning($"Could not add performance counter {performanceCounter.Key}.");
}
}
ClusterMetrics.Health = clusterHealth;
var clusterMetricSubmitted = ClusterMetricSubmitted;
clusterMetricSubmitted?.Invoke(this, new ClusterMetricsEventArgs(ClusterMetrics));
}
public void TestDefaultHealthPolicy()
{
CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper());
PaasClusterUpgradePolicy defaultPaasClusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy();
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(5, 5);
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
defaultPaasClusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
VerifyCommandProcessorClusterUpgradeDescription(
defaultPaasClusterUpgradePolicy,
result,
maxPercentageUnhealthyApplication: 0,
expectedtMaxPercentUnhealthyPerServicesType: null);
}
public void TestApplcationWithUnhealthySystemServiceHealthPolicy()
{
var testFabricClientWrapper = new TestFabricClientWrapper();
CommandParameterGenerator generator = new CommandParameterGenerator(testFabricClientWrapper);
PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicyWithSystemServicesPolicies();
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(
totalApplicationCount: 6,
totalNodeCount: 5,
unhealthyApplicationsCount: 0,
unhealthyNodeCount: 0,
isSystemServicesUnhealthy: true);
Service fssService = new StatefulService(new Uri("fabric:/System/ImageStoreService"), "FileStoreService", "1.0", true, HealthState.Error, ServiceStatus.Active);
Service cmService = new StatefulService(new Uri("fabric:/System/ClusterManagerService"), "ClusterManager", "1.0", true, HealthState.Ok, ServiceStatus.Active);
Service hmService = new StatefulService(new Uri("fabric:/System/HealthManagerService"), "HealthManager", "1.0", true, HealthState.Ok, ServiceStatus.Active);
testFabricClientWrapper.GetServicesResult = new ServiceList(new List <Service>()
{
fssService, cmService, hmService
});
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
clusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
var expectedtMaxPercentUnhealthyPerServicesType = new Dictionary <string, byte>();
expectedtMaxPercentUnhealthyPerServicesType.Add("FileStoreService", 100);
expectedtMaxPercentUnhealthyPerServicesType.Add("ClusterManager", 0);
expectedtMaxPercentUnhealthyPerServicesType.Add("HealthManager", 0);
VerifyCommandProcessorClusterUpgradeDescription(
clusterUpgradePolicy,
result,
0,
expectedtMaxPercentUnhealthyPerServicesType);
}
/// <summary>
/// Post the cluster patching status as events on CoordinatorService
/// </summary>
public async Task PostClusterPatchingStatus(CancellationToken cancellationToken)
{
try
{
NodeList nodeList = await this.fabricClient.QueryManager.GetNodeListAsync(null, null, this.DefaultTimeoutForOperation, cancellationToken);
IList <RepairTask> claimedTaskList = await this.GetClaimedRepairTasks(nodeList, cancellationToken);
RepairTaskList processingTaskList = await this.GetRepairTasksUnderProcessing(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
if (claimedTaskList.Any())
{
if (!processingTaskList.Any())
{
// This means that repair tasks are not getting approved.
ClusterHealth clusterHealth = await this.fabricClient.HealthManager.GetClusterHealthAsync();
if (clusterHealth.AggregatedHealthState == HealthState.Error)
{
// Reset Count
postUpdateCount = 0;
string warningDescription = " Cluster is currently unhealthy. Nodes are currently not getting patched by Patch Orchestration Application. Please ensure the cluster becomes healthy for patching to continue.";
await PostWarningOnCoordinatorService(warningDescription, 1);
}
else
{
postUpdateCount++;
if (postUpdateCount > 60)
{
// Reset Count and throw a warning on the service saying we dont know the reason. But POA not is not approving tasks.
postUpdateCount = 0;
string warningDescription = "Patch Orchestration Application is currently not patching nodes. This could be possible if there is some node which is stuck in disabling state for long time.";
await PostWarningOnCoordinatorService(warningDescription, 61);
}
}
}
else
{
// Reset Count
postUpdateCount = 0;
await PostRMTaskNodeUpdate(cancellationToken);
}
}
else
{
// Reset Count
postUpdateCount = 0;
if (processingTaskList.Any())
{
await PostRMTaskNodeUpdate(cancellationToken);
}
else
{
// Post the health event saying that there is no repair task and things are working fine.
string description = "No claimed tasks and no processing tasks are found.";
HealthManagerHelper.PostNodeHealthReport(this.fabricClient, this.context.ServiceName, ClusterPatchingStatusProperty, description, HealthState.Ok, -1);
}
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ErrorMessage("PostClusterPatchingStatus failed with exception {0}", ex.ToString());
}
}
public async Task <ClusterUpgradeCommandParameter> GetCommandParameterAsync(
FabricUpgradeProgress currentUpgradeProgress,
ClusterHealth currentClusterHealth,
PaasClusterUpgradeDescription response,
CancellationToken token)
{
if (response == null ||
(response.TargetClusterManifestBase64 == null && response.TargetMsiUri == null))
{
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Null response");
return(null);
}
if (currentClusterHealth == null)
{
Trace.WriteWarning(TraceType, "currentClusterHealth is null. ContinousNullClusterHealthCount = {0}", this.continousNullClusterHealthCount);
if (++continousNullClusterHealthCount < 5)
{
Trace.WriteWarning(TraceType, "Returning null since upgrade will not be initiated");
return(null);
}
Trace.WriteWarning(TraceType, "Upgrading even though currentClusterHealth is null.");
}
else
{
this.continousNullClusterHealthCount = 0;
}
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Processing WRP response");
ClusterManifestType targetClusterManifest = FromBase64ClusterManifest(response.TargetClusterManifestBase64);
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: response.TargetMsiVersion = {0}, currentUpgradeProgress.TargetCodeVersion = {1}, targetClusterManifest.Version = {2}, currentUpgradeProgress.UpgradeState = {3}",
response.TargetMsiVersion,
currentUpgradeProgress != null ? currentUpgradeProgress.TargetCodeVersion : "NA",
targetClusterManifest != null ? targetClusterManifest.Version : "NA",
currentUpgradeProgress != null ? currentUpgradeProgress.UpgradeState.ToString() : "NA");
string currentCodeVersion = string.Empty;
string currentConfigVersion = string.Empty;
bool isUpgradeStateFailed = false;
if (currentUpgradeProgress != null)
{
currentCodeVersion = currentUpgradeProgress.TargetCodeVersion;
currentConfigVersion = currentUpgradeProgress.TargetConfigVersion;
isUpgradeStateFailed = currentUpgradeProgress.UpgradeState == FabricUpgradeState.Failed;
}
if (response.TargetMsiVersion == currentCodeVersion &&
(targetClusterManifest != null && targetClusterManifest.Version == currentConfigVersion) &&
!isUpgradeStateFailed)
{
return(null);
}
string clusterConfigFilePath = null;
string clusterMsiPath = null;
string clusterConfigVersion = null;
var info = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
var tempPath = info.FullName;
if (isUpgradeStateFailed || (targetClusterManifest != null && targetClusterManifest.Version != currentConfigVersion))
{
clusterConfigFilePath = Path.Combine(tempPath, "ClusterManifest.xml");
using (var fs = new FileStream(clusterConfigFilePath, FileMode.Create))
{
using (TextWriter writer = new StreamWriter(fs, new UTF8Encoding()))
{
// Serialize using the XmlTextWriter.
this.clusterManifestSerializer.Serialize(writer, targetClusterManifest);
}
}
clusterConfigVersion = targetClusterManifest.Version;
}
if (isUpgradeStateFailed || (response.TargetMsiUri != null && response.TargetMsiVersion != currentCodeVersion))
{
clusterMsiPath = await DownloadMsiAsync(response.TargetMsiUri, response.TargetMsiVersion, tempPath);
}
Trace.WriteInfo(TraceType, "ClusterConfigFile {0}, ClusterMsiFile {1}", clusterConfigFilePath, clusterMsiPath);
var parameter = new ClusterUpgradeCommandParameter()
{
CodeVersion = response.TargetMsiVersion,
CodeFilePath = clusterMsiPath,
ConfigFilePath = clusterConfigFilePath,
ConfigVersion = clusterConfigVersion
};
parameter.UpgradeDescription = await this.GetClusterUpgradeDescriptionAsync(response.UpgradePolicy, currentClusterHealth, token);
return(parameter);
}
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
{
if (paasUpgradePolicy == null)
{
return(null);
}
CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
{
ForceRestart = paasUpgradePolicy.ForceRestart,
HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout,
HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration,
HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration,
UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout,
UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout,
};
if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
{
return(upgradeDescription);
}
upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();
var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;
if (paasHealthPolicy != null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes;
if (paasHealthPolicy.ApplicationHealthPolicies != null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
keyValuePair => keyValuePair.Key,
KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
}
}
var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;
if (paasDeltaHealthPolicy != null)
{
upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
}
else
{
int totalAppCount = 0, unhealthyAppCount = 0;
if (currentClusterHealth == null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
Trace.WriteWarning(
TraceType,
"currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
}
else if (currentClusterHealth.ApplicationHealthStates != null)
{
var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
appHealthState =>
{
if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
return(true);
});
unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
totalAppCount = filteredAppHealthStates.Count();
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
unhealthyAppCount,
totalAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
Trace.WriteInfo(
TraceType,
"Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
totalAppCount,
unhealthyAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
{
foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
{
var applicationName = applicationDeltaHealthPolicy.Key;
var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
{
// no policy provided
continue;
}
ApplicationHealthState matchingHealthState = null;
if (currentClusterHealth != null)
{
matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
}
if (matchingHealthState == null)
{
Trace.WriteWarning(
TraceType,
"Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
applicationName);
// the application is not found in the cluster
continue;
}
Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);
// Compute the total and unhealthy services by ServiceType for this application
Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
foreach (var service in serviceList)
{
HealthStats serviceTypeHealthstats;
if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
{
serviceTypeHealthstats = new HealthStats();
serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
}
if (service.HealthState == HealthState.Error)
{
serviceTypeHealthstats.UnhealthyCount++;
}
serviceTypeHealthstats.TotalCount++;
}
// For each service type specific healthy policy provided, compute the delta health policy
if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
{
var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;
HealthStats stats;
if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
{
byte maxUnhealthyPercentage =
CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
stats.TotalCount,
stats.UnhealthyCount,
serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
}
else
{
Trace.WriteWarning(
TraceType,
"ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
serviceTypeName,
applicationName);
continue;
}
}
}
// If default service type delta policy is specified, compute the delta health policy for ServiceType
// which does not have an explicit policy
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
{
foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
{
var serviceTypeName = serviceTypeHealthStatsKeyValue.Key;
var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;
if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
{
// Explicit policy has been specified
continue;
}
byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.TotalCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.UnhealthyCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
}
}
if (commandProcessorServiceTypeHealthPolicies.Any())
{
if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
}
CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
{
applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
{
SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
};
upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
applicationName,
applicationHealthPolicy);
}
foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
{
if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
{
applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
}
applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
}
}
}
}
}
return(upgradeDescription);
}
