public async Task ProcessStartClusterConfigurationUpgradeAsync(ConfigurationUpgradeDescription configUpgradeDesc, TimeSpan timeout, CancellationToken cancellationToken)
        {
            /* The cancellation token passed in this API call is not used (by design). This token corresponds to the client side call.
             * The global this.cancellationToken is initialised in RunAsync() and is honored in every API call. */

            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessStartUpgradeAsync.");
            try
            {
                UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Deserializing input json config string.");
                StandAloneInstallerJsonModelBase targetJsonConfig = StandAloneInstallerJsonModelBase.GetJsonConfigFromString(configUpgradeDesc.ClusterConfiguration);
                if (targetJsonConfig == null)
                {
                    throw new ArgumentException("The input cluster configuration is not in a valid json format or supported apiVersion.");
                }

                UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Retrieve current cluster resource from StoreManager.");
                StandAloneCluster cluster = await this.storeManager.GetClusterResourceAsync(
                    Constants.ClusterReliableDictionaryKey, this.cancellationToken).ConfigureAwait(false);

                bool isInDataLossState = await DatalossHelper.IsInDatalossStateAsync(this.cancellationToken).ConfigureAwait(false);

                if (!isInDataLossState)
                {
                    if (cluster == null || cluster.Current == null)
                    {
                        UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "Persisted cluster resource is not ready: {0}", cluster == null ? "null" : "current = null");
                        throw new FabricException("UpgradeOrchestrationService is not ready.");
                    }

                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Setting target config and topology based on new input json config.");

                    if (cluster.Pending != null && !this.IsInterruptibleAsync(cluster.Pending).Result)
                    {
                        throw new FabricException(string.Format("Cluster configuration upgrade of type {0} is already in progress and cannot be interrupted.", cluster.Pending.GetType().Name));
                    }

                    StandaloneSettingsValidator validator = new StandaloneSettingsValidator(targetJsonConfig);

                    await UpgradeOrchestrationMessageProcessor.ValidateModel(targetJsonConfig, validator, cluster, true).ConfigureAwait(false);

                    var removedNodes = validator.GetRemovedNodes(cluster.Topology);
                    var addedNodes   = validator.GetAddedNodes(cluster.Topology);

                    if (addedNodes.Any() && StandaloneUtility.CheckFabricRunningAsGMSA(cluster.Current.CSMConfig))
                    {
                        /* Need to resolve assembly so that FabricDeployer can load the right binaries from FabricCodePath since not all binaries required by FabricDeployer are present in UOS.Current folder.*/
                        AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(this.LoadFromFabricCodePath);
                        try
                        {
                            await this.PerformAddNodeOperationGMSAAsync(addedNodes, this.fabricClient, validator.ClusterProperties.NodeTypes).ConfigureAwait(false);
                        }
                        catch (AggregateException ex)
                        {
                            UpgradeOrchestrationTrace.TraceSource.WriteError(TraceType, "Adding nodes for GMSA scenario failed with exception: {0}", ex);
                            throw;
                        }
                        finally
                        {
                            AppDomain.CurrentDomain.AssemblyResolve -= new ResolveEventHandler(this.LoadFromFabricCodePath);
                        }
                    }

                    if (addedNodes.Any())
                    {
                        cluster.TargetNodeConfig = GetTargetNodeConfigAddNode(validator.Topology, cluster.Current.NodeConfig.Version);
                    }

                    if (removedNodes.Any())
                    {
                        cluster.TargetNodeConfig = GetTargetNodeConfigRemoveNode(cluster.Topology, removedNodes, cluster.Current.NodeConfig.Version);
                    }
                    else
                    {
                        cluster.Topology = validator.Topology;
                    }

                    cluster.TargetCsmConfig = validator.ClusterProperties;

                    // Cluster is updated above so persist it.
                    await this.storeManager.PersistClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cluster, cancellationToken).ConfigureAwait(false);

                    await this.UpdatePersistedCodeUpgradePackage(validator).ConfigureAwait(false);

                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Invoking Orchestrator");

                    await this.orchestrator.StartUpgradeAsync(cluster, this.cancellationToken, configUpgradeDesc).ContinueWith(t =>
                    {
                        if (t.Exception != null)
                        {
                            UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "Orchestrator completed with status: {0} exception: {1}", t.Status, t.Exception);
                        }
                        else
                        {
                            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Orchestrator completed with status: {0}", t.Status);
                        }
                    });
                }
                else
                {
                    StandaloneSettingsValidator validator = new StandaloneSettingsValidator(targetJsonConfig);
                    await UpgradeOrchestrationMessageProcessor.ValidateModel(targetJsonConfig, validator, cluster, false).ConfigureAwait(false);

                    cluster = FabricUpgradeOrchestrationService.ConstructClusterFromJson(targetJsonConfig, FabricNativeConfigStore.FabricGetConfigStore());

                    DatalossHelper.DryRunConfigUpgrade(cluster);
                    await this.storeManager.PersistClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cluster, this.cancellationToken);

                    await DatalossHelper.UpdateHeathStateAsync(isHealthy : true).ConfigureAwait(false);
                }
            }
            catch (Exception e)
            {
                UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessStartUpgradeAsync exception: {0}", e);
                throw UpgradeOrchestrationMessageProcessor.ConvertToComException(e);
            }

            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exiting ProcessStartUpgradeAsync.");
        }
        protected override async Task RunAsync(CancellationToken cancellationToken)
        {
            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Enter RunAsync");

            try
            {
                var configStore = FabricNativeConfigStore.FabricGetConfigStore();

                StandAloneFabricSettingsActivator.InitializeConfigStore(configStore);

                UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "After InitializeAsync");

                UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Getting cluster resource from store manager.");
                StandAloneCluster cluster = await this.storeManager.GetClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cancellationToken).ConfigureAwait(false);

                bool isInDatalossState = await DatalossHelper.IsInDatalossStateAsync(cancellationToken).ConfigureAwait(false);

                if (!isInDatalossState)
                {
                    bool isBaselineUpgrade = false;
                    if (cluster == null)
                    {
                        // Cluster resource does not exist, e.g. first time baseline upgrade.
                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Cluster resource does not exist in store manager. Initiating new clsuter resource.");

                        cluster = FabricUpgradeOrchestrationService.InitClusterResource(configStore);
                        ReleaseAssert.AssertIf(cluster == null, "Cluster Resource cannot be initialized.");
                        isBaselineUpgrade = true;

                        await this.CompleteAndPersistBaselineStateAsync(cluster, cancellationToken).ConfigureAwait(false);
                    }

                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Setting a valid cancellation token for UpgradeOrchestrationMessageProcessor");
                    this.messageProcessor.CancellationToken = cancellationToken;

                    /* In version 5.7, we added Iron as a new entry to ReliabilityLevel enum. This entry was added as ReliabilityLevel = 1 and hence, it moved all the existing levels one level down.
                     * For clusters created in <=5.6, when upgraded to 5.7+ the ReliabilityLevel field would be interpreted differently. The below code fixes this issue by reading the
                     * UOS state and comparing it against the actual cluster state. If there is a mismatch it sets the UOS to the correct ReliabilityLevel.
                     */
                    if (!isBaselineUpgrade &&
                        cluster.Current != null &&
                        cluster.Current.CSMConfig != null)
                    {
                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Current ReliabilityLevel set in UOS state {0}", cluster.Current.CSMConfig.ReliabilityLevel);
                        var actualReliabilityLevelForCluster = await this.GetActualReliabilityLevelForCluster(cancellationToken).ConfigureAwait(false);

                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Actual ReliabilityLevel set for the cluster {0}", actualReliabilityLevelForCluster);
                        if (actualReliabilityLevelForCluster != cluster.Current.CSMConfig.ReliabilityLevel)
                        {
                            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "UOS ReliabilityLevel is inconsistent with actual reliability level for the cluster.. Setting UOS state to {0}", actualReliabilityLevelForCluster);
                            cluster.Current.CSMConfig.ReliabilityLevel = actualReliabilityLevelForCluster;
                            await this.storeManager.PersistClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cluster, cancellationToken).ConfigureAwait(false);

                            cluster = await this.storeManager.GetClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cancellationToken).ConfigureAwait(false);

                            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "UOS ReliabilityLevel set to {0}", cluster.Current.CSMConfig.ReliabilityLevel);
                        }
                    }

                    /* This is a workaround till actual admin config upgrades are implemented. In 5.7 we changed some properties in ClusterSettings
                     * but those will not take effect for clusters created with version < 5.7 since no upgrade reads those settings.
                     * This workaround initiates a WRP config upgrade in the form of SimpleClusterUpgradeState if the settings found are old (after the code upgrade completes to this version). */
                    string isAdminConfigUpgradeAttempted = await this.storeManager.GetStorageObjectAsync(Constants.AdminConfigUpgradeAttemptedDictionaryKey, cancellationToken).ConfigureAwait(false);

                    if (!isBaselineUpgrade && string.IsNullOrEmpty(isAdminConfigUpgradeAttempted) && this.IsFirewallRuleDisabled())
                    {
                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Old admin configuration settings detected.. Will initiate admin config upgrade after code upgrade completes..");
                        bool isCurrentCodeUpgradeCompleted = false;
                        while (!isCurrentCodeUpgradeCompleted)
                        {
                            isCurrentCodeUpgradeCompleted = await this.CheckCodeUpgradeCompletedAsync(cancellationToken).ConfigureAwait(false);

                            await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken).ConfigureAwait(false);
                        }

                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Setting targetWRPConfig to initiate admin config upgrade.");
                        var adminConfig = new StandaloneAdminConfig();
                        adminConfig.Version.ClusterSettingsVersion = "2.1";
                        cluster.TargetWrpConfig = adminConfig;
                        await this.storeManager.SetStorageObjectAsync(Constants.AdminConfigUpgradeAttemptedDictionaryKey, "true", cancellationToken).ConfigureAwait(false);
                    }

                    if (!isBaselineUpgrade)
                    {
                        // If the cluster manifest versions don't match, send a health warning for users to know.
                        Task manifestCheck = this.SetupClusterManifestVersionCheck(cancellationToken, cluster.Current.ExternalState.ClusterManifest.Version);

                        UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Invoking Orchestrator.StartUpgradeAsync");
                        Task t = this.Orchestrator.StartUpgradeAsync(cluster, cancellationToken, new ConfigurationUpgradeDescription());
                    }

                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Done with Orchestrator.StartUpgradeAsync");
                }

                var  goalStateProvisioner = new StandaloneGoalStateProvisioner(this.storeManager, this.Orchestrator, cancellationToken);
                Task goalStatePollTask    = (goalStateProvisioner.IsAutoupgradeEnabled() || goalStateProvisioner.IsAutoupgradeInstallEnabled())
                    ? goalStateProvisioner.SetUpGoalStatePoll(cancellationToken, this.IsSkipInitialGoalStateCheck())
                    : Task.Run(
                    () =>
                {
                    goalStateProvisioner.EmitGoalStateReachableHealth(this.fabricClient, true /*success*/);
                    goalStateProvisioner.EmitClusterVersionSupportedHealth(this.fabricClient, true /*success*/);
                },
                    cancellationToken);

                await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
            }
            catch (FabricNotPrimaryException ex)
            {
                UpgradeOrchestrationTrace.TraceSource.WriteError(TraceType, ex.ToString());
            }
            catch (FabricObjectClosedException ocex)
            {
                UpgradeOrchestrationTrace.TraceSource.WriteError(TraceType, ocex.ToString());
            }

            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exit RunAsync");
        }