public void TestApplcationWithNullDeltaHealthPolicy()
{
CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper());
PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy();
clusterUpgradePolicy.DeltaHealthPolicy = null;
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(
totalApplicationCount: 6,
totalNodeCount: 5,
unhealthyApplicationsCount: 0,
unhealthyNodeCount: 0);
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
clusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
VerifyCommandProcessorClusterUpgradeDescription(
clusterUpgradePolicy,
result,
maxPercentageUnhealthyApplication: 100,
expectedtMaxPercentUnhealthyPerServicesType: null);
}
private static PaasClusterUpgradePolicy CreateDefaultPaasClusterUpgradePolicyWithSystemServicesPolicies()
{
PaasApplicationHealthPolicy systemServiceHealthPolicy = new PaasApplicationHealthPolicy();
systemServiceHealthPolicy.DefaultServiceTypeHealthPolicy = new PaasServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = 100
};
PaasApplicationDeltaHealthPolicy systemServiceDeltaHealthPolicy = new PaasApplicationDeltaHealthPolicy();
systemServiceDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy = new PaasServiceTypeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyServices = 0
};
PaasClusterUpgradePolicy policy = CreateDefaultPaasClusterUpgradePolicy();
policy.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, PaasApplicationHealthPolicy>();
policy.HealthPolicy.ApplicationHealthPolicies.Add("fabric:/System", systemServiceHealthPolicy);
policy.DeltaHealthPolicy.ApplicationDeltaHealthPolicies = new Dictionary <string, PaasApplicationDeltaHealthPolicy>();
policy.DeltaHealthPolicy.ApplicationDeltaHealthPolicies.Add("fabric:/System", systemServiceDeltaHealthPolicy);
return(policy);
}
private static PaasClusterUpgradePolicy CreateDefaultPaasClusterUpgradePolicy()
{
PaasClusterUpgradePolicy policy = new PaasClusterUpgradePolicy()
{
ForceRestart = false,
UpgradeReplicaSetCheckTimeout = TimeSpan.MaxValue,
HealthCheckWaitDuration = TimeSpan.FromMinutes(5),
HealthCheckStableDuration = TimeSpan.FromMinutes(5),
HealthCheckRetryTimeout = TimeSpan.FromMinutes(30),
UpgradeTimeout = TimeSpan.FromHours(12),
UpgradeDomainTimeout = TimeSpan.FromHours(2)
};
policy.HealthPolicy = new PaasClusterUpgradeHealthPolicy()
{
MaxPercentUnhealthyApplications = 100,
MaxPercentUnhealthyNodes = 100,
ApplicationHealthPolicies = null
};
policy.DeltaHealthPolicy = new PaasClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyApplications = 0,
MaxPercentDeltaUnhealthyNodes = 0,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = 0,
ApplicationDeltaHealthPolicies = null
};
return(policy);
}
public void TestDefaultHealthPolicy()
{
CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper());
PaasClusterUpgradePolicy defaultPaasClusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy();
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(5, 5);
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
defaultPaasClusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
VerifyCommandProcessorClusterUpgradeDescription(
defaultPaasClusterUpgradePolicy,
result,
maxPercentageUnhealthyApplication: 0,
expectedtMaxPercentUnhealthyPerServicesType: null);
}
public void TestApplcationWithUnhealthySystemServiceHealthPolicy()
{
var testFabricClientWrapper = new TestFabricClientWrapper();
CommandParameterGenerator generator = new CommandParameterGenerator(testFabricClientWrapper);
PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicyWithSystemServicesPolicies();
ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(
totalApplicationCount: 6,
totalNodeCount: 5,
unhealthyApplicationsCount: 0,
unhealthyNodeCount: 0,
isSystemServicesUnhealthy: true);
Service fssService = new StatefulService(new Uri("fabric:/System/ImageStoreService"), "FileStoreService", "1.0", true, HealthState.Error, ServiceStatus.Active);
Service cmService = new StatefulService(new Uri("fabric:/System/ClusterManagerService"), "ClusterManager", "1.0", true, HealthState.Ok, ServiceStatus.Active);
Service hmService = new StatefulService(new Uri("fabric:/System/HealthManagerService"), "HealthManager", "1.0", true, HealthState.Ok, ServiceStatus.Active);
testFabricClientWrapper.GetServicesResult = new ServiceList(new List <Service>()
{
fssService, cmService, hmService
});
CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync(
clusterUpgradePolicy,
defaultClusterHealth,
CancellationToken.None).Result;
var expectedtMaxPercentUnhealthyPerServicesType = new Dictionary <string, byte>();
expectedtMaxPercentUnhealthyPerServicesType.Add("FileStoreService", 100);
expectedtMaxPercentUnhealthyPerServicesType.Add("ClusterManager", 0);
expectedtMaxPercentUnhealthyPerServicesType.Add("HealthManager", 0);
VerifyCommandProcessorClusterUpgradeDescription(
clusterUpgradePolicy,
result,
0,
expectedtMaxPercentUnhealthyPerServicesType);
}
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
{
if (paasUpgradePolicy == null)
{
return(null);
}
CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
{
ForceRestart = paasUpgradePolicy.ForceRestart,
HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout,
HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration,
HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration,
UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout,
UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout,
};
if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
{
return(upgradeDescription);
}
upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();
var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;
if (paasHealthPolicy != null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes;
if (paasHealthPolicy.ApplicationHealthPolicies != null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
keyValuePair => keyValuePair.Key,
KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
}
}
var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;
if (paasDeltaHealthPolicy != null)
{
upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
}
else
{
int totalAppCount = 0, unhealthyAppCount = 0;
if (currentClusterHealth == null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
Trace.WriteWarning(
TraceType,
"currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
}
else if (currentClusterHealth.ApplicationHealthStates != null)
{
var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
appHealthState =>
{
if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
return(true);
});
unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
totalAppCount = filteredAppHealthStates.Count();
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
unhealthyAppCount,
totalAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
Trace.WriteInfo(
TraceType,
"Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
totalAppCount,
unhealthyAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
{
foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
{
var applicationName = applicationDeltaHealthPolicy.Key;
var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
{
// no policy provided
continue;
}
ApplicationHealthState matchingHealthState = null;
if (currentClusterHealth != null)
{
matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
}
if (matchingHealthState == null)
{
Trace.WriteWarning(
TraceType,
"Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
applicationName);
// the application is not found in the cluster
continue;
}
Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);
// Compute the total and unhealthy services by ServiceType for this application
Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
foreach (var service in serviceList)
{
HealthStats serviceTypeHealthstats;
if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
{
serviceTypeHealthstats = new HealthStats();
serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
}
if (service.HealthState == HealthState.Error)
{
serviceTypeHealthstats.UnhealthyCount++;
}
serviceTypeHealthstats.TotalCount++;
}
// For each service type specific healthy policy provided, compute the delta health policy
if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
{
var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;
HealthStats stats;
if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
{
byte maxUnhealthyPercentage =
CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
stats.TotalCount,
stats.UnhealthyCount,
serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
}
else
{
Trace.WriteWarning(
TraceType,
"ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
serviceTypeName,
applicationName);
continue;
}
}
}
// If default service type delta policy is specified, compute the delta health policy for ServiceType
// which does not have an explicit policy
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
{
foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
{
var serviceTypeName = serviceTypeHealthStatsKeyValue.Key;
var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;
if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
{
// Explicit policy has been specified
continue;
}
byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.TotalCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.UnhealthyCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
}
}
if (commandProcessorServiceTypeHealthPolicies.Any())
{
if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
}
CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
{
applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
{
SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
};
upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
applicationName,
applicationHealthPolicy);
}
foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
{
if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
{
applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
}
applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
}
}
}
}
}
return(upgradeDescription);
}
private void VerifyCommandProcessorClusterUpgradeDescription(
PaasClusterUpgradePolicy source,
CommandProcessorClusterUpgradeDescription result,
byte maxPercentageUnhealthyApplication,
Dictionary <string, byte> expectedtMaxPercentUnhealthyPerServicesType)
{
Assert.AreEqual(source.ForceRestart, result.ForceRestart);
Assert.AreEqual(source.HealthCheckRetryTimeout, result.HealthCheckRetryTimeout);
Assert.AreEqual(source.HealthCheckStableDuration, result.HealthCheckStableDuration);
Assert.AreEqual(source.HealthCheckWaitDuration, result.HealthCheckWaitDuration);
Assert.AreEqual(source.UpgradeDomainTimeout, result.UpgradeDomainTimeout);
Assert.AreEqual(source.UpgradeTimeout, result.UpgradeTimeout);
Assert.AreEqual(source.UpgradeReplicaSetCheckTimeout, result.UpgradeReplicaSetCheckTimeout);
if (source.HealthPolicy == null)
{
Assert.IsNull(result.HealthPolicy);
}
else
{
Assert.IsNotNull(result.HealthPolicy);
Assert.AreEqual(source.HealthPolicy.MaxPercentUnhealthyNodes, result.HealthPolicy.MaxPercentUnhealthyNodes);
Assert.AreEqual(maxPercentageUnhealthyApplication, result.HealthPolicy.MaxPercentUnhealthyApplications);
if (source.HealthPolicy.ApplicationHealthPolicies == null)
{
Assert.IsNull(result.HealthPolicy.ApplicationHealthPolicies);
}
else
{
Assert.IsNotNull(result.HealthPolicy.ApplicationHealthPolicies);
Assert.AreEqual(source.HealthPolicy.ApplicationHealthPolicies.Count(), result.HealthPolicy.ApplicationHealthPolicies.Count());
foreach (var keyVaulePair in result.HealthPolicy.ApplicationHealthPolicies)
{
Assert.AreEqual(
source.HealthPolicy.ApplicationHealthPolicies[keyVaulePair.Key].DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices,
keyVaulePair.Value.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices);
if (keyVaulePair.Value.SerivceTypeHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in keyVaulePair.Value.SerivceTypeHealthPolicies)
{
string serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
byte expectedtMaxPercentUnhealthy;
if (expectedtMaxPercentUnhealthyPerServicesType.TryGetValue(
serviceTypeName, out expectedtMaxPercentUnhealthy))
{
Assert.AreEqual(expectedtMaxPercentUnhealthy, serviceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices);
}
else
{
Assert.AreEqual(
source.HealthPolicy.ApplicationHealthPolicies[keyVaulePair.Key].SerivceTypeHealthPolicies[serviceTypeName].MaxPercentUnhealthyServices,
serviceTypeHealthPolicyKeyValue.Value);
}
}
}
}
}
}
if (source.DeltaHealthPolicy == null)
{
Assert.IsNull(result.DeltaHealthPolicy);
}
else
{
Assert.IsNotNull(result.DeltaHealthPolicy);
Assert.AreEqual(source.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
result.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes);
Assert.AreEqual(source.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes,
result.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes);
}
}
