public ClusterCommandProcessor(
IConfigStore configStore,
string configSectionName,
IFabricClientWrapper fabricClientWrapper,
CommandParameterGenerator commandParameterGenerator,
NodeStatusManager nodeStatusManager,
JsonSerializer jsonSerializer)
{
configStore.ThrowIfNull(nameof(configStore));
configSectionName.ThrowIfNullOrWhiteSpace(nameof(configSectionName));
fabricClientWrapper.ThrowIfNull(nameof(fabricClientWrapper));
commandParameterGenerator.ThrowIfNull(nameof(commandParameterGenerator));
nodeStatusManager.ThrowIfNull(nameof(nodeStatusManager));
jsonSerializer.ThrowIfNull(nameof(jsonSerializer));
this.configStore = configStore;
this.configSectionName = configSectionName;
this.fabricClientWrapper = fabricClientWrapper;
this.commandParameterGenerator = commandParameterGenerator;
this.nodeStatusManager = nodeStatusManager;
this.serializer = jsonSerializer;
}
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
{
if (paasUpgradePolicy == null)
{
return(null);
}
CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
{
ForceRestart = paasUpgradePolicy.ForceRestart,
HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout,
HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration,
HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration,
UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout,
UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout,
};
if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
{
return(upgradeDescription);
}
upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();
var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;
if (paasHealthPolicy != null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes;
if (paasHealthPolicy.ApplicationHealthPolicies != null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
keyValuePair => keyValuePair.Key,
KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
}
}
var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;
if (paasDeltaHealthPolicy != null)
{
upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
}
else
{
int totalAppCount = 0, unhealthyAppCount = 0;
if (currentClusterHealth == null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
Trace.WriteWarning(
TraceType,
"currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
}
else if (currentClusterHealth.ApplicationHealthStates != null)
{
var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
appHealthState =>
{
if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
return(true);
});
unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
totalAppCount = filteredAppHealthStates.Count();
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
unhealthyAppCount,
totalAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
Trace.WriteInfo(
TraceType,
"Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
totalAppCount,
unhealthyAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
{
foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
{
var applicationName = applicationDeltaHealthPolicy.Key;
var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
{
// no policy provided
continue;
}
ApplicationHealthState matchingHealthState = null;
if (currentClusterHealth != null)
{
matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
}
if (matchingHealthState == null)
{
Trace.WriteWarning(
TraceType,
"Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
applicationName);
// the application is not found in the cluster
continue;
}
Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);
// Compute the total and unhealthy services by ServiceType for this application
Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
foreach (var service in serviceList)
{
HealthStats serviceTypeHealthstats;
if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
{
serviceTypeHealthstats = new HealthStats();
serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
}
if (service.HealthState == HealthState.Error)
{
serviceTypeHealthstats.UnhealthyCount++;
}
serviceTypeHealthstats.TotalCount++;
}
// For each service type specific healthy policy provided, compute the delta health policy
if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
{
var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;
HealthStats stats;
if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
{
byte maxUnhealthyPercentage =
CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
stats.TotalCount,
stats.UnhealthyCount,
serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
}
else
{
Trace.WriteWarning(
TraceType,
"ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
serviceTypeName,
applicationName);
continue;
}
}
}
// If default service type delta policy is specified, compute the delta health policy for ServiceType
// which does not have an explicit policy
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
{
foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
{
var serviceTypeName = serviceTypeHealthStatsKeyValue.Key;
var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;
if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
{
// Explicit policy has been specified
continue;
}
byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.TotalCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.UnhealthyCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
}
}
if (commandProcessorServiceTypeHealthPolicies.Any())
{
if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
}
CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
{
applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
{
SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
};
upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
applicationName,
applicationHealthPolicy);
}
foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
{
if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
{
applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
}
applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
}
}
}
}
}
return(upgradeDescription);
}