public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
{
if (paasUpgradePolicy == null)
{
return(null);
}
CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
{
ForceRestart = paasUpgradePolicy.ForceRestart,
HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout,
HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration,
HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration,
UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout,
UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout,
};
if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
{
return(upgradeDescription);
}
upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();
var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;
if (paasHealthPolicy != null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes;
if (paasHealthPolicy.ApplicationHealthPolicies != null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
keyValuePair => keyValuePair.Key,
KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
}
}
var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;
if (paasDeltaHealthPolicy != null)
{
upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
}
else
{
int totalAppCount = 0, unhealthyAppCount = 0;
if (currentClusterHealth == null)
{
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
Trace.WriteWarning(
TraceType,
"currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
}
else if (currentClusterHealth.ApplicationHealthStates != null)
{
var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
appHealthState =>
{
if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
{
return(false);
}
return(true);
});
unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
totalAppCount = filteredAppHealthStates.Count();
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
unhealthyAppCount,
totalAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
Trace.WriteInfo(
TraceType,
"Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
totalAppCount,
unhealthyAppCount,
paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
}
if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
{
foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
{
var applicationName = applicationDeltaHealthPolicy.Key;
var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
{
// no policy provided
continue;
}
ApplicationHealthState matchingHealthState = null;
if (currentClusterHealth != null)
{
matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
}
if (matchingHealthState == null)
{
Trace.WriteWarning(
TraceType,
"Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
applicationName);
// the application is not found in the cluster
continue;
}
Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);
// Compute the total and unhealthy services by ServiceType for this application
Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
foreach (var service in serviceList)
{
HealthStats serviceTypeHealthstats;
if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
{
serviceTypeHealthstats = new HealthStats();
serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
}
if (service.HealthState == HealthState.Error)
{
serviceTypeHealthstats.UnhealthyCount++;
}
serviceTypeHealthstats.TotalCount++;
}
// For each service type specific healthy policy provided, compute the delta health policy
if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
{
foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
{
var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key;
var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;
HealthStats stats;
if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
{
byte maxUnhealthyPercentage =
CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
stats.TotalCount,
stats.UnhealthyCount,
serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
}
else
{
Trace.WriteWarning(
TraceType,
"ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
serviceTypeName,
applicationName);
continue;
}
}
}
// If default service type delta policy is specified, compute the delta health policy for ServiceType
// which does not have an explicit policy
if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
{
foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
{
var serviceTypeName = serviceTypeHealthStatsKeyValue.Key;
var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;
if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
{
// Explicit policy has been specified
continue;
}
byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.TotalCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
commandProcessorServiceTypeHealthPolicies.Add(
serviceTypeName,
new CommandProcessorServiceTypeHealthPolicy()
{
MaxPercentUnhealthyServices = maxUnhealthyPercentage
});
Trace.WriteInfo(
TraceType,
"Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
serviceTypeName,
applicationName,
maxUnhealthyPercentage,
serviceTypeHealthStats.UnhealthyCount,
serviceTypeHealthStats.UnhealthyCount,
paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
}
}
if (commandProcessorServiceTypeHealthPolicies.Any())
{
if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
{
upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
}
CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
{
applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
{
SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
};
upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
applicationName,
applicationHealthPolicy);
}
foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
{
if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
{
applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
}
applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
}
}
}
}
}
return(upgradeDescription);
}
/// <summary>
/// This function starts the upgrade process
/// </summary>
/// <param name="commandDescription"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task StartUpgradeFabricAsync(
CommandProcessorClusterUpgradeDescription commandDescription,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
Trace.WriteInfo(
TraceType,
"StartUpgradeFabricAsync - Started");
var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
{
FailureAction = UpgradeFailureAction.Rollback
};
var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
MonitoringPolicy = rollingUpgradeMonitoringPolicy,
};
if (commandDescription != null)
{
if (commandDescription.HealthCheckRetryTimeout.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
}
if (commandDescription.HealthCheckStableDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
}
if (commandDescription.HealthCheckWaitDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
}
if (commandDescription.UpgradeDomainTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
}
if (commandDescription.UpgradeTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
}
if (commandDescription.ForceRestart.HasValue)
{
policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
}
if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
}
if (commandDescription.HealthPolicy != null)
{
policyDescription.HealthPolicy = new ClusterHealthPolicy
{
MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
};
if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
{
foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
{
CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
commandProcessorApplicationHealthPolicyKeyValue.Value;
if (commandProcessorApplicationHealthPolicy == null)
{
continue;
}
var applicationHealthPolicy = new ApplicationHealthPolicy();
if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
};
}
if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
{
foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
{
if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
{
continue;
}
ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
};
applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
}
}
policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
}
}
}
if (commandDescription.DeltaHealthPolicy != null)
{
policyDescription.EnableDeltaHealthEvaluation = true;
policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
}
}
;
// Specify the target code and configuration version and upgrade mode.
var upgradeDescription = new FabricUpgradeDescription()
{
TargetCodeVersion = targetCodeVersion,
TargetConfigVersion = targetConfigVersion,
UpgradePolicyDescription = policyDescription
};
Trace.WriteInfo(TraceType, "Start upgrade");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.UpgradeFabricAsync(
upgradeDescription,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.UpgradeFabricErrors.Value,
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
}
