public async Task <string> ProcessGetClusterConfigurationAsync(string apiVersion, TimeSpan timeout, CancellationToken cancellationToken)
        {
            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessGetClusterConfigurationAsync.");
            try
            {
                UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Retrieve current cluster resource from StoreManager.");
                StandAloneCluster cluster = await this.storeManager.GetClusterResourceAsync(
                    Constants.ClusterReliableDictionaryKey, this.cancellationToken).ConfigureAwait(false);

                if (cluster == null || cluster.Current == null)
                {
                    // read from fabric data root
                    string fabricDataRoot = FabricEnvironment.GetDataRoot();
                    string jsonConfigPath = Path.Combine(fabricDataRoot, System.Fabric.FabricDeployer.Constants.FileNames.BaselineJsonClusterConfig); // TODO: Needs to come from image store
                    return(File.ReadAllText(jsonConfigPath));
                }
                else
                {
                    var      nodesFromFM = new Dictionary <string, NodeDescription>();
                    NodeList nodes       = await StandaloneUtility.GetNodesFromFMAsync(this.fabricClient, this.cancellationToken).ConfigureAwait(false);

                    for (int i = 0; i < nodes.Count; ++i)
                    {
                        if (nodes[i].NodeStatus != System.Fabric.Query.NodeStatus.Invalid && !UpgradeOrchestrationMessageProcessor.IsGhostNode(nodes[i]))
                        {
                            NodeDescription nodeDesc = new NodeDescription()
                            {
                                FaultDomain   = nodes[i].FaultDomain.ToString(),
                                UpgradeDomain = nodes[i].UpgradeDomain,
                                IPAddress     = nodes[i].IpAddressOrFQDN,
                                NodeTypeRef   = nodes[i].NodeType,
                                NodeName      = nodes[i].NodeName
                            };
                            nodesFromFM.Add(nodes[i].NodeName, nodeDesc);
                        }
                    }

                    cluster.Topology.Nodes = nodesFromFM;

                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Construct StandAloneJsonModel from current cluster resource.");
                    var jsonModel = StandAloneInstallerJsonModelBase.ConstructByApiVersion(cluster.Current.CSMConfig, cluster.Topology, cluster.Current.CSMConfig.Version.Version, apiVersion);
                    UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Serializing current json model.");
                    var serializerSettings = new JsonSerializerSettings
                    {
                        Formatting                 = Formatting.Indented,
                        NullValueHandling          = NullValueHandling.Ignore,
                        PreserveReferencesHandling = PreserveReferencesHandling.None
                    };
                    serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
                    return(JsonConvert.SerializeObject(jsonModel, serializerSettings));
                }
            }
            catch (Exception e)
            {
                UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessStartUpgradeAsync exception: {0}", e);
                throw UpgradeOrchestrationMessageProcessor.ConvertToComException(e);
            }
        }
        internal async Task PerformAddNodeOperationGMSAAsync(List <NodeDescription> addedNodes, FabricClient fabricClient, List <NodeTypeDescription> allNodeTypes)
        {
            string destinationPackagePath = await this.DownloadProvisionedPackage().ConfigureAwait(false);

            var nodesFromFM = await StandaloneUtility.GetNodesFromFMAsync(fabricClient, this.cancellationToken).ConfigureAwait(false);

            string clusterManifestPath = await this.GenerateClusterManifestWithAddedNodes(addedNodes, nodesFromFM, allNodeTypes);

            List <string> machines = new List <string>();

            foreach (var addednode in addedNodes)
            {
                if (!nodesFromFM.Any(nodeFromFM => nodeFromFM.NodeName == addednode.NodeName))
                {
                    machines.Add(addednode.IPAddress);
                }
            }

            bool             successfulRun = true;
            List <Exception> exceptions    = new List <Exception>();

            try
            {
                await this.AddNodesAsync(clusterManifestPath, destinationPackagePath, machines, addedNodes);
            }
            catch (Exception ex)
            {
                successfulRun = false;
                exceptions.Add(ex);
                UpgradeOrchestrationTrace.TraceSource.WriteError("Adding nodes failed with exception {0}.", ex.ToString());
            }

            if (!successfulRun)
            {
                // Best-Effort Rollback
                try
                {
                    await DeploymentManagerInternal.RemoveNodeConfigurationAsync(machines, false, FabricPackageType.XCopyPackage, true);
                }
                catch (Exception ex)
                {
                    UpgradeOrchestrationTrace.TraceSource.WriteError("Rollback failed with {0}.", ex.ToString());
                    exceptions.Add(ex);
                }

                AggregateException ae = new AggregateException(exceptions);
                throw ae;
            }

            this.CleanUp(clusterManifestPath, destinationPackagePath);
            UpgradeOrchestrationTrace.TraceSource.WriteInfo("Nodes were succesfully added to the GMSA secured cluster");
        }
        private async Task <ReliabilityLevel> GetActualReliabilityLevelForCluster(CancellationToken cancellationToken)
        {
            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering GetCurrentReliabilityLevelForCluster");
            System.Fabric.Query.NodeList nodes = await StandaloneUtility.GetNodesFromFMAsync(this.fabricClient, cancellationToken).ConfigureAwait(false);

            int seedNodeCount = nodes.Count(p => p.IsSeedNode);

            ReliabilityLevel actualReliabilityLevelInCluster = ReliabilityLevelExtension.GetReliabilityLevel(seedNodeCount);

            UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "SeedNodeCount from cluster {0}, actualReliabilityLevelInCluster {1}", seedNodeCount, actualReliabilityLevelInCluster);

            return(actualReliabilityLevelInCluster);
        }