Esempio n. 1
0
        protected override void ProcessRecord()
        {
            try
            {
                if (this.ReplicaQuorumTimeoutSec.HasValue)
                {
                    this.WriteWarning(StringResources.PowerShell_ReplicaQuorumTimeoutSec_Deprecated);

                    if (!this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
                    {
                        this.UpgradeReplicaSetCheckTimeoutSec = this.ReplicaQuorumTimeoutSec.Value;
                    }
                }

                if (this.RestartProcess)
                {
                    this.WriteWarning(StringResources.PowerShell_RestartProcess_Deprecated);

                    if (!this.ForceRestart)
                    {
                        this.ForceRestart = this.RestartProcess;
                    }
                }

                RollingUpgradePolicyDescription policyDescription;
                if (this.Monitored)
                {
                    var monitoringPolicy = new RollingUpgradeMonitoringPolicy();

                    if (this.FailureAction == UpgradeFailureAction.Invalid)
                    {
                        this.FailureAction = Constants.DefaultUpgradeFailureAction;
                    }

                    monitoringPolicy.FailureAction = this.FailureAction;

                    if (this.HealthCheckRetryTimeoutSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value);
                    }

                    if (this.HealthCheckWaitDurationSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value);
                    }

                    if (this.HealthCheckStableDurationSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value);
                    }

                    if (this.UpgradeDomainTimeoutSec.HasValue)
                    {
                        monitoringPolicy.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value);
                    }

                    if (this.UpgradeTimeoutSec.HasValue)
                    {
                        monitoringPolicy.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value);
                    }

                    var monitoredPolicyDescription = new MonitoredRollingApplicationUpgradePolicyDescription
                    {
                        UpgradeMode      = RollingUpgradeMode.Monitored,
                        ForceRestart     = this.ForceRestart,
                        MonitoringPolicy = monitoringPolicy
                    };

                    policyDescription = monitoredPolicyDescription;

                    if (this.IsUpdatingHealthPolicy())
                    {
                        if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess(
                                //// description shows up for "-WhatIf"
                                string.Format(
                                    CultureInfo.InvariantCulture,
                                    "{0} {1}",
                                    StringResources.PowerShell_HealthPolicyUpgradeCaption,
                                    StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning),
                                //// warning and caption show up when prompting for confirmation
                                StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning,
                                StringResources.PowerShell_HealthPolicyUpgradeCaption))
                        {
                            return;
                        }

                        var healthPolicy = new ApplicationHealthPolicy();

                        if (this.ConsiderWarningAsError.HasValue)
                        {
                            healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value;
                        }

                        if (this.DefaultServiceTypeHealthPolicy != null)
                        {
                            healthPolicy.DefaultServiceTypeHealthPolicy = this.ParseServiceTypeHealthPolicy(this.DefaultServiceTypeHealthPolicy);
                        }

                        if (this.MaxPercentUnhealthyDeployedApplications.HasValue)
                        {
                            healthPolicy.MaxPercentUnhealthyDeployedApplications = this.MaxPercentUnhealthyDeployedApplications.Value;
                        }

                        if (this.ServiceTypeHealthPolicyMap != null)
                        {
                            foreach (DictionaryEntry entry in this.ServiceTypeHealthPolicyMap)
                            {
                                healthPolicy.ServiceTypeHealthPolicyMap.Add(entry.Key as string, this.ParseServiceTypeHealthPolicy(entry.Value as string));
                            }
                        }

                        monitoredPolicyDescription.HealthPolicy = healthPolicy;
                    }
                }
                else if (this.UnmonitoredManual)
                {
                    policyDescription = new RollingUpgradePolicyDescription
                    {
                        UpgradeMode  = RollingUpgradeMode.UnmonitoredManual,
                        ForceRestart = this.ForceRestart,
                    };
                }
                else
                {
                    policyDescription = new RollingUpgradePolicyDescription
                    {
                        UpgradeMode  = RollingUpgradeMode.UnmonitoredAuto,
                        ForceRestart = this.ForceRestart,
                    };
                }

                if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
                {
                    policyDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value);
                }

                var upgradeDescription = new ApplicationUpgradeDescription
                {
                    ApplicationName = this.ApplicationName,
                    TargetApplicationTypeVersion = this.ApplicationTypeVersion,
                    UpgradePolicyDescription     = policyDescription
                };
                upgradeDescription.ApplicationParameters.Add(this.GetNameValueCollection(this.ApplicationParameter));
                this.UpgradeApplication(upgradeDescription);
            }
            catch (Exception exception)
            {
                this.ThrowTerminatingError(
                    exception,
                    Constants.UpgradeApplicationErrorId,
                    null);
            }
        }
Esempio n. 2
0
        public override async Task UpdateAsync(IOperationDescription description, IFabricOperationResult result, IOperationContext context)
        {
            Trace.WriteInfo(TraceType, "UpdateAsync called");
            description.ThrowIfNull(nameof(description));
            result.ThrowIfNull(nameof(result));
            result.OperationStatus.ThrowIfNull(nameof(result.OperationStatus));
            result.QueryResult.ThrowIfNull(nameof(result.QueryResult));
            context.ThrowIfNull(nameof(context));

            string errorMessage;
            if (!this.ValidObjectType<ApplicationOperationDescription>(description, out errorMessage))
            {
                throw new InvalidCastException(errorMessage);
            }

            if (!this.ValidObjectType<ApplicationFabricQueryResult>(result.QueryResult, out errorMessage))
            {
                throw new InvalidCastException(errorMessage);
            }

            var appDescription = (ApplicationOperationDescription)description;
            appDescription.TypeVersion.ThrowIfNullOrWhiteSpace(nameof(appDescription.TypeVersion));
            appDescription.ApplicationUri.ThrowIfNull(nameof(appDescription.ApplicationUri));
            var appResult = (ApplicationFabricQueryResult)result.QueryResult;

            var healthPolicy = new ApplicationHealthPolicy();
            var defaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy();
            if (appDescription.UpgradePolicy?.ApplicationHealthPolicy != null)
            {
                healthPolicy.ConsiderWarningAsError =
                    appDescription.UpgradePolicy.ApplicationHealthPolicy.ConsiderWarningAsError;
                healthPolicy.MaxPercentUnhealthyDeployedApplications =
                    appDescription.UpgradePolicy.ApplicationHealthPolicy.MaxPercentUnhealthyDeployedApplications;

                if (appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
                {
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyPartitionsPerService =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyPartitionsPerService;
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyReplicasPerPartition =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyReplicasPerPartition;
                    defaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices =
                        appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
                            .MaxPercentUnhealthyServices;
                    healthPolicy.DefaultServiceTypeHealthPolicy = defaultServiceTypeHealthPolicy;
                }
            }

            var monitoringPolicy = new RollingUpgradeMonitoringPolicy();
            if (appDescription.UpgradePolicy?.RollingUpgradeMonitoringPolicy != null)
            {
                monitoringPolicy.FailureAction =
                    (UpgradeFailureAction) Enum.Parse(
                        typeof (UpgradeFailureAction),
                        appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.FailureAction.ToString(),
                        true);
                monitoringPolicy.HealthCheckRetryTimeout =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckRetryTimeout;
                monitoringPolicy.HealthCheckStableDuration =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckStableDuration;
                monitoringPolicy.HealthCheckWaitDuration =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckWaitDuration;
                monitoringPolicy.UpgradeDomainTimeout =
                    appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeDomainTimeout;
                monitoringPolicy.UpgradeTimeout = appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeTimeout;
            }

            var policyDescription = new MonitoredRollingApplicationUpgradePolicyDescription()
            {
                UpgradeMode = RollingUpgradeMode.Monitored,
                HealthPolicy = healthPolicy,
                MonitoringPolicy = monitoringPolicy
            };

            if (appDescription.UpgradePolicy != null)
            {
                if (appDescription.UpgradePolicy.ForceRestart.HasValue)
                {
                    policyDescription.ForceRestart = appDescription.UpgradePolicy.ForceRestart.Value;
                }

                if (appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.HasValue)
                {
                    policyDescription.UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.Value;
                }
            }

            // Update
            var updateDescription = new ApplicationUpdateDescription(appDescription.ApplicationUri)
            {
                RemoveApplicationCapacity = appDescription.RemoveApplicationCapacity
            };

            if (appDescription.MinimumNodes.HasValue)
            {
                updateDescription.MinimumNodes = appDescription.MinimumNodes.Value;
            }

            if (appDescription.MaximumNodes.HasValue)
            {
                updateDescription.MaximumNodes = appDescription.MaximumNodes.Value;
            }

            if (appDescription.Metrics != null && appDescription.Metrics.Any())
            {
                updateDescription.Metrics = new List<ApplicationMetricDescription>();
                foreach (var metric in appDescription.Metrics)
                {
                    updateDescription.Metrics.Add(
                        new ApplicationMetricDescription()
                        {
                            Name = metric.Name,
                            MaximumNodeCapacity = metric.MaximumCapacity,
                            NodeReservationCapacity = metric.ReservationCapacity,
                            TotalApplicationCapacity = metric.TotalApplicationCapacity
                        });
                }
            }

            Trace.WriteInfo(
                TraceType,
                "UpdateAsync: Updating application. Name: {0}. Timeout: {1}",
                updateDescription.ApplicationName,
                context.OperationTimeout);
            await this.fabricClient.ApplicationManager.UpdateApplicationAsync(
                updateDescription,
                context.OperationTimeout,
                context.CancellationToken);
            Trace.WriteInfo(TraceType, "UpdateAsync: Application update call accepted");

            // Upgrade
            var upgradeDescription = new ApplicationUpgradeDescription()
            {
                ApplicationName = appDescription.ApplicationUri,
                TargetApplicationTypeVersion = appDescription.TypeVersion,
                UpgradePolicyDescription = policyDescription
            };

            if (appDescription.Parameters != null)
            {
                upgradeDescription.ApplicationParameters.Add(appDescription.Parameters.ToNameValueCollection());
            }

            try
            {
                Trace.WriteInfo(
                    TraceType,
                    "UpdateAsync: Upgrading application. Name: {0}. Version: {1}. TargetVersion: {2}. Timeout: {3}",
                    upgradeDescription.ApplicationName,
                    appResult.Application.ApplicationTypeVersion,
                    upgradeDescription.TargetApplicationTypeVersion,
                    context.OperationTimeout);
                await this.fabricClient.ApplicationManager.UpgradeApplicationAsync(
                    upgradeDescription,
                    context.OperationTimeout,
                    context.CancellationToken);
                Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade call accepted");
            }
            catch (FabricException fe)
            {
                if (fe.ErrorCode == FabricErrorCode.ApplicationAlreadyInTargetVersion)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: Application already in target version: {0}.", upgradeDescription.TargetApplicationTypeVersion);
                    return;
                }

                if (fe.ErrorCode == FabricErrorCode.ApplicationUpgradeInProgress)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade in progress with same version: {0}", upgradeDescription.TargetApplicationTypeVersion);                 
                }
                else
                {
                    Trace.WriteError(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", fe);
                    throw;
                }                
            }
            catch (Exception e)
            {
                Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", e);
                throw;
            }

            // UpgradeUpdate
            ApplicationUpgradeProgress upgradeProgress = null;
            try
            {
                upgradeProgress = await this.fabricClient.ApplicationManager.GetApplicationUpgradeProgressAsync(
                        appDescription.ApplicationUri,
                        context.OperationTimeout,
                        context.CancellationToken);
            }
            catch (Exception)
            {
                Trace.WriteWarning(
                    TraceType,
                    "UpdateAsync: Failed to get the application upgrade progress to determine the parameters for upgrade update. Need to retry the operation. Application Name: {0}",
                    appDescription.ApplicationUri);

                throw new FabricTransientException();
            }
                       
            ApplicationUpgradeUpdateDescription upgradeUpdateDescription;
            if (upgradeProgress.UpgradeState == ApplicationUpgradeState.RollingBackInProgress)
            {
                // During rollback we can only change below properties
                upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
                {
                    ApplicationName = appDescription.ApplicationUri,
                    UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout,
                    ForceRestart = appDescription.UpgradePolicy.ForceRestart
                };
            }
            else
            {
                upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
                {
                    ApplicationName = appDescription.ApplicationUri,
                    UpgradeDomainTimeout = monitoringPolicy.UpgradeDomainTimeout,
                    UpgradeTimeout = monitoringPolicy.UpgradeTimeout,
                    HealthCheckRetryTimeout = monitoringPolicy.HealthCheckRetryTimeout,
                    HealthCheckWaitDuration = monitoringPolicy.HealthCheckWaitDuration,
                    HealthCheckStableDuration = monitoringPolicy.HealthCheckStableDuration,
                    UpgradeReplicaSetCheckTimeout = policyDescription.UpgradeReplicaSetCheckTimeout,
                    ForceRestart = policyDescription.ForceRestart,
                    FailureAction = monitoringPolicy.FailureAction,
                    HealthPolicy = healthPolicy
                };
            }

            try
            {               

                Trace.WriteInfo(
                    TraceType,
                    "UpdateAsync: Updating application upgrade in progress. Name: {0}. Timeout: {1}",
                    upgradeUpdateDescription.ApplicationName,
                    context.OperationTimeout);
                await this.fabricClient.ApplicationManager.UpdateApplicationUpgradeAsync(
                    upgradeUpdateDescription,
                    context.OperationTimeout,
                    context.CancellationToken);
                Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade call accepted");
            }
            catch (FabricException fe)
            {
                if (fe.ErrorCode == FabricErrorCode.ApplicationNotUpgrading)
                {
                    Trace.WriteInfo(TraceType, "UpdateAsync: No application upgrade in progress");
                    return;
                }
            }
            catch (Exception e)
            {
                Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade encountered an exception: {0}", e);
                throw;
            }
        }
        protected override void ProcessRecord()
        {
            try
            {
                if (this.ReplicaQuorumTimeoutSec.HasValue)
                {
                    this.WriteWarning(StringResources.PowerShell_ReplicaQuorumTimeoutSec_Deprecated);

                    if (!this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
                    {
                        this.UpgradeReplicaSetCheckTimeoutSec = this.ReplicaQuorumTimeoutSec.Value;
                    }
                }

                if (this.RestartProcess)
                {
                    this.WriteWarning(StringResources.PowerShell_RestartProcess_Deprecated);

                    if (!this.ForceRestart)
                    {
                        this.ForceRestart = this.RestartProcess;
                    }
                }

                RollingUpgradePolicyDescription upgradePolicyDescription;
                if (this.Monitored)
                {
                    var monitoringPolicy = new RollingUpgradeMonitoringPolicy();

                    if (this.FailureAction != UpgradeFailureAction.Invalid)
                    {
                        monitoringPolicy.FailureAction = this.FailureAction;
                    }

                    if (this.HealthCheckRetryTimeoutSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value);
                    }

                    if (this.HealthCheckWaitDurationSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value);
                    }

                    if (this.HealthCheckStableDurationSec.HasValue)
                    {
                        monitoringPolicy.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value);
                    }

                    if (this.UpgradeDomainTimeoutSec.HasValue)
                    {
                        monitoringPolicy.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value);
                    }

                    if (this.UpgradeTimeoutSec.HasValue)
                    {
                        monitoringPolicy.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value);
                    }

                    var monitoredPolicyDescription = new MonitoredRollingFabricUpgradePolicyDescription
                    {
                        UpgradeMode      = RollingUpgradeMode.Monitored,
                        ForceRestart     = this.ForceRestart,
                        MonitoringPolicy = monitoringPolicy
                    };

                    upgradePolicyDescription = monitoredPolicyDescription;

                    if (this.IsUpdatingHealthPolicy())
                    {
                        if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess(
                                //// description shows up for "-WhatIf"
                                string.Format(
                                    CultureInfo.InvariantCulture,
                                    "{0} {1}",
                                    StringResources.PowerShell_HealthPolicyUpgradeCaption,
                                    StringResources.PowerShell_ClusterHealthPolicyUpdateWarning),
                                //// warning and caption show up when prompting for confirmation
                                StringResources.PowerShell_ClusterHealthPolicyUpdateWarning,
                                StringResources.PowerShell_HealthPolicyUpgradeCaption))
                        {
                            return;
                        }

                        var healthPolicy = new ClusterHealthPolicy();

                        if (this.ConsiderWarningAsError.HasValue)
                        {
                            healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value;
                        }

                        if (this.MaxPercentUnhealthyApplications.HasValue)
                        {
                            healthPolicy.MaxPercentUnhealthyApplications = this.MaxPercentUnhealthyApplications.Value;
                        }

                        if (this.MaxPercentUnhealthyNodes.HasValue)
                        {
                            healthPolicy.MaxPercentUnhealthyNodes = this.MaxPercentUnhealthyNodes.Value;
                        }

                        if (this.ApplicationTypeHealthPolicyMap != null)
                        {
                            foreach (var entry in this.ApplicationTypeHealthPolicyMap)
                            {
                                healthPolicy.ApplicationTypeHealthPolicyMap.Add(entry.Key, entry.Value);
                            }
                        }

                        monitoredPolicyDescription.HealthPolicy = healthPolicy;
                    }

                    monitoredPolicyDescription.EnableDeltaHealthEvaluation = this.EnableDeltaHealthEvaluation;

                    if (this.IsUpdatingUpgradeHealthPolicy())
                    {
                        if (!this.Force && !this.IsUpgradeHealthPolicyComplete() && !this.ShouldProcess(
                                //// description shows up for "-WhatIf"
                                string.Format(
                                    CultureInfo.InvariantCulture,
                                    "{0} {1}",
                                    StringResources.PowerShell_HealthPolicyUpgradeCaption,
                                    StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning),
                                //// warning and caption show up when prompting for confirmation
                                StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning,
                                StringResources.PowerShell_HealthPolicyUpgradeCaption))
                        {
                            return;
                        }

                        var upgradeHealthPolicy = new ClusterUpgradeHealthPolicy();

                        if (this.MaxPercentDeltaUnhealthyNodes.HasValue)
                        {
                            upgradeHealthPolicy.MaxPercentDeltaUnhealthyNodes = this.MaxPercentDeltaUnhealthyNodes.Value;
                        }

                        if (this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.HasValue)
                        {
                            upgradeHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes = this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.Value;
                        }

                        monitoredPolicyDescription.UpgradeHealthPolicy = upgradeHealthPolicy;
                    }

                    if (this.ApplicationHealthPolicyMap != null)
                    {
                        foreach (var entry in this.ApplicationHealthPolicyMap)
                        {
                            monitoredPolicyDescription.ApplicationHealthPolicyMap.Add(entry.Key, entry.Value);
                        }
                    }
                }
                else if (this.UnmonitoredManual)
                {
                    upgradePolicyDescription = new RollingUpgradePolicyDescription
                    {
                        UpgradeMode  = RollingUpgradeMode.UnmonitoredManual,
                        ForceRestart = this.ForceRestart,
                    };
                }
                else
                {
                    upgradePolicyDescription = new RollingUpgradePolicyDescription
                    {
                        UpgradeMode  = RollingUpgradeMode.UnmonitoredAuto,
                        ForceRestart = this.ForceRestart,
                    };
                }

                if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
                {
                    upgradePolicyDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value);
                }

                var upgradeDescription = new FabricUpgradeDescription
                {
                    UpgradePolicyDescription = upgradePolicyDescription,
                    TargetCodeVersion        = this.CodePackageVersion,
                    TargetConfigVersion      = this.ClusterManifestVersion
                };

                this.UpgradeCluster(upgradeDescription);
            }
            catch (Exception exception)
            {
                this.ThrowTerminatingError(
                    exception,
                    Constants.UpgradeClusterErrorId,
                    null);
            }
        }
Esempio n. 4
0
        /// <summary>
        /// This function starts the upgrade process
        /// </summary>
        /// <param name="commandDescription"></param>
        /// <param name="targetCodeVersion"></param>
        /// <param name="targetConfigVersion"></param>
        /// <param name="timeoutHelper"></param>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        private async Task StartUpgradeFabricAsync(
            CommandProcessorClusterUpgradeDescription commandDescription,
            string targetCodeVersion,
            string targetConfigVersion,
            TimeoutHelper timeoutHelper,
            CancellationToken cancellationToken)
        {
            Trace.WriteInfo(
                TraceType,
                "StartUpgradeFabricAsync - Started");

            var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
            {
                FailureAction = UpgradeFailureAction.Rollback
            };

            var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
            {
                UpgradeMode      = RollingUpgradeMode.Monitored,
                MonitoringPolicy = rollingUpgradeMonitoringPolicy,
            };

            if (commandDescription != null)
            {
                if (commandDescription.HealthCheckRetryTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
                }

                if (commandDescription.HealthCheckStableDuration.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
                }

                if (commandDescription.HealthCheckWaitDuration.HasValue)
                {
                    policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
                }

                if (commandDescription.UpgradeDomainTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
                }

                if (commandDescription.UpgradeTimeout.HasValue)
                {
                    policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
                }

                if (commandDescription.ForceRestart.HasValue)
                {
                    policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
                }

                if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
                {
                    policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
                }

                if (commandDescription.HealthPolicy != null)
                {
                    policyDescription.HealthPolicy = new ClusterHealthPolicy
                    {
                        MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
                        MaxPercentUnhealthyNodes        = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
                    };

                    if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
                    {
                        foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
                        {
                            CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
                                commandProcessorApplicationHealthPolicyKeyValue.Value;

                            if (commandProcessorApplicationHealthPolicy == null)
                            {
                                continue;
                            }

                            var applicationHealthPolicy = new ApplicationHealthPolicy();
                            if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
                            {
                                applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
                                {
                                    MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
                                };
                            }

                            if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
                            {
                                foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
                                {
                                    if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
                                    {
                                        continue;
                                    }

                                    ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
                                    {
                                        MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
                                    };

                                    applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
                                }
                            }

                            policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
                        }
                    }
                }

                if (commandDescription.DeltaHealthPolicy != null)
                {
                    policyDescription.EnableDeltaHealthEvaluation = true;
                    policyDescription.UpgradeHealthPolicy         = new ClusterUpgradeHealthPolicy()
                    {
                        MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
                        MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
                    };
                }
            }
            ;

            // Specify the target code and configuration version and upgrade mode.
            var upgradeDescription = new FabricUpgradeDescription()
            {
                TargetCodeVersion        = targetCodeVersion,
                TargetConfigVersion      = targetConfigVersion,
                UpgradePolicyDescription = policyDescription
            };

            Trace.WriteInfo(TraceType, "Start upgrade");

            await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
                                                                            this.fabricClient.ClusterManager.UpgradeFabricAsync(
                                                                                upgradeDescription,
                                                                                timeoutHelper.GetOperationTimeout(),
                                                                                cancellationToken),
                                                                            FabricClientRetryErrors.UpgradeFabricErrors.Value,
                                                                            timeoutHelper.GetOperationTimeout(),
                                                                            cancellationToken).ConfigureAwait(false);
        }