Ejemplo n.º 1
0
        /// <inheritdoc/>
        protected override void ProcessRecordInternal()
        {
            var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
                maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
                maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
                maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);

            var applicationHealthPolicy = new ApplicationHealthPolicy(
                considerWarningAsError: this.ConsiderWarningAsError,
                maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
                defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
                serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);

            var result = this.ServiceFabricClient.Applications.GetApplicationHealthUsingPolicyAsync(
                applicationId: this.ApplicationId,
                eventsHealthStateFilter: this.EventsHealthStateFilter,
                deployedApplicationsHealthStateFilter: this.DeployedApplicationsHealthStateFilter,
                servicesHealthStateFilter: this.ServicesHealthStateFilter,
                excludeHealthStatistics: this.ExcludeHealthStatistics,
                applicationHealthPolicy: applicationHealthPolicy,
                serverTimeout: this.ServerTimeout,
                cancellationToken: this.CancellationToken).GetAwaiter().GetResult();

            if (result != null)
            {
                this.WriteObject(this.FormatOutput(result));
            }
        }
        /// <inheritdoc/>
        protected override void ProcessRecordInternal()
        {
            var registryCredential = new RegistryCredential(
                registryUserName: this.RegistryUserName,
                registryPassword: this.RegistryPassword,
                passwordEncrypted: this.PasswordEncrypted);

            var monitoringPolicyDescription = new MonitoringPolicyDescription(
                failureAction: this.FailureAction,
                healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
                healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
                healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
                upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
                upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds);

            var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
                maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
                maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
                maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);

            var applicationHealthPolicy = new ApplicationHealthPolicy(
                considerWarningAsError: this.ConsiderWarningAsError,
                maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
                defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
                serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);

            var composeDeploymentUpgradeDescription = new ComposeDeploymentUpgradeDescription(
                deploymentName: this.DeploymentName,
                composeFileContent: this.ComposeFileContent,
                upgradeKind: this.UpgradeKind,
                registryCredential: registryCredential,
                rollingUpgradeMode: this.RollingUpgradeMode,
                upgradeReplicaSetCheckTimeoutInSeconds: this.UpgradeReplicaSetCheckTimeoutInSeconds,
                forceRestart: this.ForceRestart,
                monitoringPolicy: monitoringPolicyDescription,
                applicationHealthPolicy: applicationHealthPolicy);

            this.ServiceFabricClient.ComposeDeployments.StartComposeDeploymentUpgradeAsync(
                deploymentName: this.DeploymentName,
                composeDeploymentUpgradeDescription: composeDeploymentUpgradeDescription,
                serverTimeout: this.ServerTimeout,
                cancellationToken: this.CancellationToken).GetAwaiter().GetResult();

            Console.WriteLine("Success!");
        }
        /// <summary>
        /// Serializes the object to JSON.
        /// </summary>
        /// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
        /// <param name="obj">The object to serialize to JSON.</param>
        internal static void Serialize(JsonWriter writer, ServiceTypeHealthPolicy obj)
        {
            // Required properties are always serialized, optional properties are serialized when not null.
            writer.WriteStartObject();
            if (obj.MaxPercentUnhealthyPartitionsPerService != null)
            {
                writer.WriteProperty(obj.MaxPercentUnhealthyPartitionsPerService, "MaxPercentUnhealthyPartitionsPerService", JsonWriterExtensions.WriteIntValue);
            }

            if (obj.MaxPercentUnhealthyReplicasPerPartition != null)
            {
                writer.WriteProperty(obj.MaxPercentUnhealthyReplicasPerPartition, "MaxPercentUnhealthyReplicasPerPartition", JsonWriterExtensions.WriteIntValue);
            }

            if (obj.MaxPercentUnhealthyServices != null)
            {
                writer.WriteProperty(obj.MaxPercentUnhealthyServices, "MaxPercentUnhealthyServices", JsonWriterExtensions.WriteIntValue);
            }

            writer.WriteEndObject();
        }
        /// <inheritdoc/>
        protected override void ProcessRecordInternal()
        {
            var monitoringPolicyDescription = new MonitoringPolicyDescription(
                failureAction: this.FailureAction,
                healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
                healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
                healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
                upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
                upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds);

            var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
                maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
                maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
                maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);

            var applicationHealthPolicy = new ApplicationHealthPolicy(
                considerWarningAsError: this.ConsiderWarningAsError,
                maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
                defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
                serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);

            var applicationUpgradeDescription = new ApplicationUpgradeDescription(
                name: $"fabric:/{this.ApplicationId}",
                targetApplicationTypeVersion: this.TargetApplicationTypeVersion,
                parameters: this.Parameters?.ToDictionary <string, string>(),
                upgradeKind: this.UpgradeKind,
                rollingUpgradeMode: this.RollingUpgradeMode,
                upgradeReplicaSetCheckTimeoutInSeconds: this.UpgradeReplicaSetCheckTimeoutInSeconds,
                forceRestart: this.ForceRestart,
                monitoringPolicy: monitoringPolicyDescription,
                applicationHealthPolicy: applicationHealthPolicy);

            this.ServiceFabricClient.Applications.StartApplicationUpgradeAsync(
                applicationId: this.ApplicationId,
                applicationUpgradeDescription: applicationUpgradeDescription,
                serverTimeout: this.ServerTimeout,
                cancellationToken: this.CancellationToken).GetAwaiter().GetResult();

            Console.WriteLine("Success!");
        }
        /// <inheritdoc/>
        protected override void ProcessRecordInternal()
        {
            var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
                maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
                maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
                maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);

            var applicationHealthPolicy = new ApplicationHealthPolicy(
                considerWarningAsError: this.ConsiderWarningAsError,
                maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
                defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
                serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);

            var rollingUpgradeUpdateDescription = new RollingUpgradeUpdateDescription(
                rollingUpgradeMode: this.RollingUpgradeMode,
                forceRestart: this.ForceRestart,
                replicaSetCheckTimeoutInMilliseconds: this.ReplicaSetCheckTimeoutInMilliseconds,
                failureAction: this.FailureAction,
                healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
                healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
                healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
                upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
                upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds,
                instanceCloseDelayDurationInSeconds: this.InstanceCloseDelayDurationInSeconds);

            var applicationUpgradeUpdateDescription = new ApplicationUpgradeUpdateDescription(
                name: this.Name,
                upgradeKind: this.UpgradeKind,
                applicationHealthPolicy: applicationHealthPolicy,
                updateDescription: rollingUpgradeUpdateDescription);

            this.ServiceFabricClient.Applications.UpdateApplicationUpgradeAsync(
                applicationId: this.ApplicationId,
                applicationUpgradeUpdateDescription: applicationUpgradeUpdateDescription,
                serverTimeout: this.ServerTimeout,
                cancellationToken: this.CancellationToken).GetAwaiter().GetResult();

            Console.WriteLine("Success!");
        }
Ejemplo n.º 6
0
        public override async Task UpdateAsync(IOperationDescription description, IFabricOperationResult result, IOperationContext context)
        {
            Trace.WriteInfo(TraceType, "UpdateAsync called");
            description.ThrowIfNull(nameof(description));
            result.ThrowIfNull(nameof(result));
            result.OperationStatus.ThrowIfNull(nameof(result.OperationStatus));
            result.QueryResult.ThrowIfNull(nameof(result.QueryResult));
            context.ThrowIfNull(nameof(context));

            string errorMessage;
            if (!this.ValidObjectType<ApplicationOperationDescription>(description, out errorMessage))
            {
                throw new InvalidCastException(errorMessage);
            }

            if (!this.ValidObjectType<ApplicationFabricQueryResult>(result.QueryResult, out errorMessage))
            {
                throw new InvalidCastException(errorMessage);
            }

            var appDescription = (ApplicationOperationDescription)description;
            appDescription.TypeVersion.ThrowIfNullOrWhiteSpace(nameof(appDescription.TypeVersion));
            appDescription.ApplicationUri.ThrowIfNull(nameof(appDescription.ApplicationUri));
            var appResult = (ApplicationFabricQueryResult)result.QueryResult;

            var healthPolicy = new ApplicationHealthPolicy();
            var defaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy();
            if (appDescription.UpgradePolicy?.ApplicationHealthPolicy != null)
            {
                healthPolicy.ConsiderWarningAsError =
                    appDescription.UpgradePolicy.ApplicationHealthPolicy.ConsiderWarningAsError;
                healthPolicy.MaxPercentUnhealthyDeployedApplications =
                    appDescription.UpgradePolicy.ApplicationHealthPolicy.MaxPercentUnhealthyDeployedApplications;

                if (appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
                {
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyPartitionsPerService =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyPartitionsPerService;
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyReplicasPerPartition =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyReplicasPerPartition;
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyServices;
                    healthPolicy.DefaultServiceTypeHealthPolicy = defaultServiceTypeHealthPolicy;
                }
            }

            var monitoringPolicy = new RollingUpgradeMonitoringPolicy();
            if (appDescription.UpgradePolicy?.RollingUpgradeMonitoringPolicy != null)
            {
                monitoringPolicy.FailureAction =
                    (UpgradeFailureAction) Enum.Parse(
                        typeof (UpgradeFailureAction),
                        appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.FailureAction.ToString(),
                        true);
                monitoringPolicy.HealthCheckRetryTimeout =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckRetryTimeout;
                monitoringPolicy.HealthCheckStableDuration =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckStableDuration;
                monitoringPolicy.HealthCheckWaitDuration =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckWaitDuration;
                monitoringPolicy.UpgradeDomainTimeout =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeDomainTimeout;
                monitoringPolicy.UpgradeTimeout = appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeTimeout;
            }

            var policyDescription = new MonitoredRollingApplicationUpgradePolicyDescription()
            {
                UpgradeMode = RollingUpgradeMode.Monitored,
                HealthPolicy = healthPolicy,
                MonitoringPolicy = monitoringPolicy
            };

            if (appDescription.UpgradePolicy != null)
            {
                if (appDescription.UpgradePolicy.ForceRestart.HasValue)
                {
                    policyDescription.ForceRestart = appDescription.UpgradePolicy.ForceRestart.Value;
                }

                if (appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.HasValue)
                {
                    policyDescription.UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.Value;
                }
            }

            // Update
            var updateDescription = new ApplicationUpdateDescription(appDescription.ApplicationUri)
            {
                RemoveApplicationCapacity = appDescription.RemoveApplicationCapacity
            };

            if (appDescription.MinimumNodes.HasValue)
            {
                updateDescription.MinimumNodes = appDescription.MinimumNodes.Value;
            }

            if (appDescription.MaximumNodes.HasValue)
            {
                updateDescription.MaximumNodes = appDescription.MaximumNodes.Value;
            }

            if (appDescription.Metrics != null && appDescription.Metrics.Any())
            {
                updateDescription.Metrics = new List<ApplicationMetricDescription>();
                foreach (var metric in appDescription.Metrics)
                {
                    updateDescription.Metrics.Add(
                        new ApplicationMetricDescription()
                        {
                            Name = metric.Name,
                            MaximumNodeCapacity = metric.MaximumCapacity,
                            NodeReservationCapacity = metric.ReservationCapacity,
                            TotalApplicationCapacity = metric.TotalApplicationCapacity
                        });
                }
            }

            Trace.WriteInfo(
                TraceType,
                "UpdateAsync: Updating application. Name: {0}. Timeout: {1}",
                updateDescription.ApplicationName,
                context.OperationTimeout);
            await this.fabricClient.ApplicationManager.UpdateApplicationAsync(
                updateDescription,
                context.OperationTimeout,
                context.CancellationToken);
            Trace.WriteInfo(TraceType, "UpdateAsync: Application update call accepted");

            // Upgrade
            var upgradeDescription = new ApplicationUpgradeDescription()
            {
                ApplicationName = appDescription.ApplicationUri,
                TargetApplicationTypeVersion = appDescription.TypeVersion,
                UpgradePolicyDescription = policyDescription
            };

            if (appDescription.Parameters != null)
            {
                upgradeDescription.ApplicationParameters.Add(appDescription.Parameters.ToNameValueCollection());
            }

            try
            {
                Trace.WriteInfo(
                    TraceType,
                    "UpdateAsync: Upgrading application. Name: {0}. Version: {1}. TargetVersion: {2}. Timeout: {3}",
                    upgradeDescription.ApplicationName,
                    appResult.Application.ApplicationTypeVersion,
                    upgradeDescription.TargetApplicationTypeVersion,
                    context.OperationTimeout);
                await this.fabricClient.ApplicationManager.UpgradeApplicationAsync(
                    upgradeDescription,
                    context.OperationTimeout,
                    context.CancellationToken);
                Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade call accepted");
            }
            catch (FabricException fe)
            {
                if (fe.ErrorCode == FabricErrorCode.ApplicationAlreadyInTargetVersion)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: Application already in target version: {0}.", upgradeDescription.TargetApplicationTypeVersion);
                    return;
                }

                if (fe.ErrorCode == FabricErrorCode.ApplicationUpgradeInProgress)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade in progress with same version: {0}", upgradeDescription.TargetApplicationTypeVersion);                 
                }
                else
                {
                    Trace.WriteError(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", fe);
                    throw;
                }                
            }
            catch (Exception e)
            {
                Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", e);
                throw;
            }

            // UpgradeUpdate
            ApplicationUpgradeProgress upgradeProgress = null;
            try
            {
                upgradeProgress = await this.fabricClient.ApplicationManager.GetApplicationUpgradeProgressAsync(
                        appDescription.ApplicationUri,
                        context.OperationTimeout,
                        context.CancellationToken);
            }
            catch (Exception)
            {
                Trace.WriteWarning(
                    TraceType,
                    "UpdateAsync: Failed to get the application upgrade progress to determine the parameters for upgrade update. Need to retry the operation. Application Name: {0}",
                    appDescription.ApplicationUri);

                throw new FabricTransientException();
            }
                       
            ApplicationUpgradeUpdateDescription upgradeUpdateDescription;
            if (upgradeProgress.UpgradeState == ApplicationUpgradeState.RollingBackInProgress)
            {
                // During rollback we can only change below properties
                upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
                {
                    ApplicationName = appDescription.ApplicationUri,
                    UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout,
                    ForceRestart = appDescription.UpgradePolicy.ForceRestart
                };
            }
            else
            {
                upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
                {
                    ApplicationName = appDescription.ApplicationUri,
                    UpgradeDomainTimeout = monitoringPolicy.UpgradeDomainTimeout,
                    UpgradeTimeout = monitoringPolicy.UpgradeTimeout,
                    HealthCheckRetryTimeout = monitoringPolicy.HealthCheckRetryTimeout,
                    HealthCheckWaitDuration = monitoringPolicy.HealthCheckWaitDuration,
                    HealthCheckStableDuration = monitoringPolicy.HealthCheckStableDuration,
                    UpgradeReplicaSetCheckTimeout = policyDescription.UpgradeReplicaSetCheckTimeout,
                    ForceRestart = policyDescription.ForceRestart,
                    FailureAction = monitoringPolicy.FailureAction,
                    HealthPolicy = healthPolicy
                };
            }

            try
            {               

                Trace.WriteInfo(
                    TraceType,
                    "UpdateAsync: Updating application upgrade in progress. Name: {0}. Timeout: {1}",
                    upgradeUpdateDescription.ApplicationName,
                    context.OperationTimeout);
                await this.fabricClient.ApplicationManager.UpdateApplicationUpgradeAsync(
                    upgradeUpdateDescription,
                    context.OperationTimeout,
                    context.CancellationToken);
                Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade call accepted");
            }
            catch (FabricException fe)
            {
                if (fe.ErrorCode == FabricErrorCode.ApplicationNotUpgrading)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: No application upgrade in progress");
                    return;
                }
            }
            catch (Exception e)
            {
                Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade encountered an exception: {0}", e);
                throw;
            }
        }
Ejemplo n.º 7
0
        /// <summary>
        /// This function starts the upgrade process
        /// </summary>
        /// <param name="commandDescription"></param>
        /// <param name="targetCodeVersion"></param>
        /// <param name="targetConfigVersion"></param>
        /// <param name="timeoutHelper"></param>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        private async Task StartUpgradeFabricAsync(
            CommandProcessorClusterUpgradeDescription commandDescription,
            string targetCodeVersion,
            string targetConfigVersion,
            TimeoutHelper timeoutHelper,
            CancellationToken cancellationToken)
        {
            Trace.WriteInfo(
                TraceType,
                "StartUpgradeFabricAsync - Started");

            var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
            {
                FailureAction = UpgradeFailureAction.Rollback
            };

            var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
            {
                UpgradeMode      = RollingUpgradeMode.Monitored,
                MonitoringPolicy = rollingUpgradeMonitoringPolicy,
            };

            if (commandDescription != null)
            {
                if (commandDescription.HealthCheckRetryTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
                }

                if (commandDescription.HealthCheckStableDuration.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
                }

                if (commandDescription.HealthCheckWaitDuration.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
                }

                if (commandDescription.UpgradeDomainTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
                }

                if (commandDescription.UpgradeTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
                }

                if (commandDescription.ForceRestart.HasValue)
                {
                    policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
                }

                if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
                {
                    policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
                }

                if (commandDescription.HealthPolicy != null)
                {
                    policyDescription.HealthPolicy = new ClusterHealthPolicy
                    {
                        MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
                        MaxPercentUnhealthyNodes        = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
                    };

                    if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
                    {
                        foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
                        {
                            CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
                                commandProcessorApplicationHealthPolicyKeyValue.Value;

                            if (commandProcessorApplicationHealthPolicy == null)
                            {
                                continue;
                            }

                            var applicationHealthPolicy = new ApplicationHealthPolicy();
                            if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
                            {
                                applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
                                {
                                    MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
                                };
                            }

                            if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
                            {
                                foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
                                {
                                    if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
                                    {
                                        continue;
                                    }

                                    ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
                                    {
                                        MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
                                    };

                                    applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
                                }
                            }

                            policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
                        }
                    }
                }

                if (commandDescription.DeltaHealthPolicy != null)
                {
                    policyDescription.EnableDeltaHealthEvaluation = true;
                    policyDescription.UpgradeHealthPolicy         = new ClusterUpgradeHealthPolicy()
                    {
                        MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
                        MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
                    };
                }
            }
            ;

            // Specify the target code and configuration version and upgrade mode.
            var upgradeDescription = new FabricUpgradeDescription()
            {
                TargetCodeVersion        = targetCodeVersion,
                TargetConfigVersion      = targetConfigVersion,
                UpgradePolicyDescription = policyDescription
            };

            Trace.WriteInfo(TraceType, "Start upgrade");

            await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
                                                                            this.fabricClient.ClusterManager.UpgradeFabricAsync(
                                                                                upgradeDescription,
                                                                                timeoutHelper.GetOperationTimeout(),
                                                                                cancellationToken),
                                                                            FabricClientRetryErrors.UpgradeFabricErrors.Value,
                                                                            timeoutHelper.GetOperationTimeout(),
                                                                            cancellationToken).ConfigureAwait(false);
        }