public void ClusterHealthSerializationTest()
{
ClusterHealth clusterHealth = this.random.CreateRandom <ClusterHealth>();
var appHS = new ApplicationHealthState()
{
ApplicationName = new Uri("fabric:/" + "testAppManifest" + this.random.CreateRandom <string>()),
AggregatedHealthState = this.random.CreateRandom <HealthState>()
};
var nodeHS = new NodeHealthState()
{
NodeName = "NodeName_" + this.random.CreateRandom <string>(),
AggregatedHealthState = this.random.CreateRandom <HealthState>()
};
clusterHealth.ApplicationHealthStates = new List <ApplicationHealthState>()
{
appHS
};
clusterHealth.NodeHealthStates = new List <NodeHealthState>()
{
nodeHS
};
TestUsingSerializer(this.Serializer, clusterHealth);
}
/// <summary>
/// Overloaded ToString function for formatting the output on the console.
/// </summary>
/// <param name="applicationHealthState"> Object of type ApplicationHealthState </param>
/// <returns>
/// Returns formatted string.
/// </returns>
public static string ToString(ApplicationHealthState applicationHealthState)
{
var strBuilder = new StringBuilder();
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0, -21} : {1}", "ApplicationName", applicationHealthState.Name));
strBuilder.Append(Environment.NewLine);
strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0, -21} : {1}", "AggregatedHealthState", applicationHealthState.AggregatedHealthState));
strBuilder.Append(Environment.NewLine);
return(strBuilder.ToString());
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ApplicationHealthState obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
writer.WriteProperty(obj.AggregatedHealthState, "AggregatedHealthState", HealthStateConverter.Serialize);
if (obj.Name != null)
{
writer.WriteProperty(obj.Name, "Name", ApplicationNameConverter.Serialize);
}
writer.WriteEndObject();
}
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
{
if (paasUpgradePolicy == null)
{
return(null);
}
CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
{
ForceRestart = paasUpgradePolicy.ForceRestart,
HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout,
HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration,
HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration,
UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout,
UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout,
};
if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
{
return(upgradeDescription);
}
upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();
var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;
if (paasHealthPolicy != null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes;
if (paasHealthPolicy.ApplicationHealthPolicies != null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
keyValuePair => keyValuePair.Key,
KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
}
}
var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;
if (paasDeltaHealthPolicy != null)
{
upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
}
else
{
int totalAppCount = 0, unhealthyAppCount = 0;
if (currentClusterHealth == null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
Trace.WriteWarning(
TraceType,
"currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
}
else if (currentClusterHealth.ApplicationHealthStates != null)
{
var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
appHealthState =>
{
if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
return(true);
});
unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
totalAppCount = filteredAppHealthStates.Count();
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
unhealthyAppCount,
totalAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
Trace.WriteInfo(
TraceType,
"Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
totalAppCount,
unhealthyAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
{
foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
{
var applicationName = applicationDeltaHealthPolicy.Key;
var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
{
// no policy provided
continue;
}
ApplicationHealthState matchingHealthState = null;
if (currentClusterHealth != null)
{
matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
}
if (matchingHealthState == null)
{
Trace.WriteWarning(
TraceType,
"Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
applicationName);
// the application is not found in the cluster
continue;
}
Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);
// Compute the total and unhealthy services by ServiceType for this application
Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
foreach (var service in serviceList)
{
HealthStats serviceTypeHealthstats;
if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
{
serviceTypeHealthstats = new HealthStats();
serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
}
if (service.HealthState == HealthState.Error)
{
serviceTypeHealthstats.UnhealthyCount++;
}
serviceTypeHealthstats.TotalCount++;
}
// For each service type specific healthy policy provided, compute the delta health policy
if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
{
var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;
HealthStats stats;
if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
{
byte maxUnhealthyPercentage =
CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
stats.TotalCount,
stats.UnhealthyCount,
serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
}
else
{
Trace.WriteWarning(
TraceType,
"ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
serviceTypeName,
applicationName);
continue;
}
}
}
// If default service type delta policy is specified, compute the delta health policy for ServiceType
// which does not have an explicit policy
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
{
foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
{
var serviceTypeName = serviceTypeHealthStatsKeyValue.Key;
var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;
if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
{
// Explicit policy has been specified
continue;
}
byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.TotalCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.UnhealthyCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
}
}
if (commandProcessorServiceTypeHealthPolicies.Any())
{
if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
}
CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
{
applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
{
SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
};
upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
applicationName,
applicationHealthPolicy);
}
foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
{
if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
{
applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
}
applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
}
}
}
}
}
return(upgradeDescription);
}
private static ClusterHealth CreateDefaultClusterHealth(
int totalApplicationCount,
int totalNodeCount,
int unhealthyApplicationsCount = 0,
int unhealthyNodeCount = 0,
bool isSystemServicesUnhealthy = false)
{
HealthState aggregatedHealthState =
(unhealthyApplicationsCount > 0 || unhealthyNodeCount > 0 || isSystemServicesUnhealthy) ? HealthState.Error : HealthState.Ok;
List <ApplicationHealthState> applicationHealthStates = new List <ApplicationHealthState>();
for (int i = 0; i < totalApplicationCount; i++)
{
HealthState healthState = HealthState.Ok;
if (unhealthyApplicationsCount > 0)
{
healthState = HealthState.Error;
unhealthyApplicationsCount--;
}
ApplicationHealthState appHealthState = new ApplicationHealthState()
{
AggregatedHealthState = healthState,
ApplicationName = new Uri(string.Format("fabric:/app{0}", i))
};
applicationHealthStates.Add(appHealthState);
}
ApplicationHealthState systemServiceAppHealthState = new ApplicationHealthState()
{
AggregatedHealthState = isSystemServicesUnhealthy ? HealthState.Error : HealthState.Ok,
ApplicationName = new Uri("fabric:/System")
};
applicationHealthStates.Add(systemServiceAppHealthState);
List <NodeHealthState> nodeHealthStates = new List <NodeHealthState>();
for (int i = 0; i < totalNodeCount; i++)
{
HealthState healthState = HealthState.Ok;
if (unhealthyNodeCount > 0)
{
healthState = HealthState.Error;
unhealthyNodeCount--;
}
NodeHealthState nodeHealthState = new NodeHealthState()
{
AggregatedHealthState = healthState,
NodeName = "_node_" + i.ToString()
};
nodeHealthStates.Add(nodeHealthState);
}
return(new ClusterHealth()
{
AggregatedHealthState = aggregatedHealthState,
ApplicationHealthStates = applicationHealthStates,
NodeHealthStates = nodeHealthStates,
HealthEvents = new List <HealthEvent>(),
UnhealthyEvaluations = new List <HealthEvaluation>()
});
}
