public ClusterCommandProcessor(
            IConfigStore configStore,
            string configSectionName,
            IFabricClientWrapper fabricClientWrapper,
            CommandParameterGenerator commandParameterGenerator,
            NodeStatusManager nodeStatusManager,
            JsonSerializer jsonSerializer)
        {
            configStore.ThrowIfNull(nameof(configStore));
            configSectionName.ThrowIfNullOrWhiteSpace(nameof(configSectionName));
            fabricClientWrapper.ThrowIfNull(nameof(fabricClientWrapper));
            commandParameterGenerator.ThrowIfNull(nameof(commandParameterGenerator));
            nodeStatusManager.ThrowIfNull(nameof(nodeStatusManager));
            jsonSerializer.ThrowIfNull(nameof(jsonSerializer));

            this.configStore               = configStore;
            this.configSectionName         = configSectionName;
            this.fabricClientWrapper       = fabricClientWrapper;
            this.commandParameterGenerator = commandParameterGenerator;
            this.nodeStatusManager         = nodeStatusManager;
            this.serializer = jsonSerializer;
        }
        public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token)
        {
            if (paasUpgradePolicy == null)
            {
                return(null);
            }

            CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription()
            {
                ForceRestart                  = paasUpgradePolicy.ForceRestart,
                HealthCheckRetryTimeout       = paasUpgradePolicy.HealthCheckRetryTimeout,
                HealthCheckStableDuration     = paasUpgradePolicy.HealthCheckStableDuration,
                HealthCheckWaitDuration       = paasUpgradePolicy.HealthCheckWaitDuration,
                UpgradeDomainTimeout          = paasUpgradePolicy.UpgradeDomainTimeout,
                UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout,
                UpgradeTimeout                = paasUpgradePolicy.UpgradeTimeout,
            };

            if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null)
            {
                return(upgradeDescription);
            }

            upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy();

            var paasHealthPolicy = paasUpgradePolicy.HealthPolicy;

            if (paasHealthPolicy != null)
            {
                upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications;
                upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes        = paasHealthPolicy.MaxPercentUnhealthyNodes;

                if (paasHealthPolicy.ApplicationHealthPolicies != null)
                {
                    upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary(
                        keyValuePair => keyValuePair.Key,
                        KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy());
                }
            }

            var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy;

            if (paasDeltaHealthPolicy != null)
            {
                upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy()
                {
                    MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
                    MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
                };

                if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100)
                {
                    upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications;
                }
                else
                {
                    int totalAppCount = 0, unhealthyAppCount = 0;

                    if (currentClusterHealth == null)
                    {
                        upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0;
                        Trace.WriteWarning(
                            TraceType,
                            "currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0");
                    }
                    else if (currentClusterHealth.ApplicationHealthStates != null)
                    {
                        var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where(
                            appHealthState =>
                        {
                            if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase))
                            {
                                return(false);
                            }

                            if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null &&
                                paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
                            {
                                return(false);
                            }

                            if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null &&
                                paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString))
                            {
                                return(false);
                            }

                            return(true);
                        });

                        unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error);
                        totalAppCount     = filteredAppHealthStates.Count();

                        upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage(
                            unhealthyAppCount,
                            totalAppCount,
                            paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
                    }

                    Trace.WriteInfo(
                        TraceType,
                        "Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.",
                        upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications,
                        totalAppCount,
                        unhealthyAppCount,
                        paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications);
                }

                if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null)
                {
                    foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies)
                    {
                        var applicationName = applicationDeltaHealthPolicy.Key;
                        var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value;

                        if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null)
                        {
                            // no policy provided
                            continue;
                        }

                        ApplicationHealthState matchingHealthState = null;
                        if (currentClusterHealth != null)
                        {
                            matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault(
                                appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
                        }

                        if (matchingHealthState == null)
                        {
                            Trace.WriteWarning(
                                TraceType,
                                "Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.",
                                applicationName);

                            // the application is not found in the cluster
                            continue;
                        }

                        Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();

                        var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token);

                        // Compute the total and unhealthy services by ServiceType for this application
                        Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>();
                        foreach (var service in serviceList)
                        {
                            HealthStats serviceTypeHealthstats;
                            if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats))
                            {
                                serviceTypeHealthstats = new HealthStats();
                                serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats);
                            }

                            if (service.HealthState == HealthState.Error)
                            {
                                serviceTypeHealthstats.UnhealthyCount++;
                            }

                            serviceTypeHealthstats.TotalCount++;
                        }

                        // For each service type specific healthy policy provided, compute the delta health policy
                        if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null)
                        {
                            foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies)
                            {
                                var serviceTypeName        = serviceTypeHealthPolicyKeyValue.Key;
                                var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value;

                                HealthStats stats;
                                if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats))
                                {
                                    byte maxUnhealthyPercentage =
                                        CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount,
                                                                                            stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);

                                    commandProcessorServiceTypeHealthPolicies.Add(
                                        serviceTypeName,
                                        new CommandProcessorServiceTypeHealthPolicy()
                                    {
                                        MaxPercentUnhealthyServices = maxUnhealthyPercentage
                                    });

                                    Trace.WriteInfo(
                                        TraceType,
                                        "Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
                                        serviceTypeName,
                                        applicationName,
                                        maxUnhealthyPercentage,
                                        stats.TotalCount,
                                        stats.UnhealthyCount,
                                        serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices);
                                }
                                else
                                {
                                    Trace.WriteWarning(
                                        TraceType,
                                        "ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.",
                                        serviceTypeName,
                                        applicationName);

                                    continue;
                                }
                            }
                        }

                        // If default service type delta policy is specified, compute the delta health policy for ServiceType
                        // which does not have an explicit policy
                        if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null)
                        {
                            foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary)
                            {
                                var serviceTypeName        = serviceTypeHealthStatsKeyValue.Key;
                                var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value;

                                if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName))
                                {
                                    // Explicit policy has been specified
                                    continue;
                                }

                                byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(
                                    serviceTypeHealthStats.UnhealthyCount,
                                    serviceTypeHealthStats.TotalCount,
                                    paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);

                                commandProcessorServiceTypeHealthPolicies.Add(
                                    serviceTypeName,
                                    new CommandProcessorServiceTypeHealthPolicy()
                                {
                                    MaxPercentUnhealthyServices = maxUnhealthyPercentage
                                });

                                Trace.WriteInfo(
                                    TraceType,
                                    "Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.",
                                    serviceTypeName,
                                    applicationName,
                                    maxUnhealthyPercentage,
                                    serviceTypeHealthStats.UnhealthyCount,
                                    serviceTypeHealthStats.UnhealthyCount,
                                    paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices);
                            }
                        }

                        if (commandProcessorServiceTypeHealthPolicies.Any())
                        {
                            if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null)
                            {
                                upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>();
                            }

                            CommandProcessorApplicationHealthPolicy applicationHealthPolicy;
                            if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy))
                            {
                                applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy()
                                {
                                    SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>()
                                };
                                upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add(
                                    applicationName,
                                    applicationHealthPolicy);
                            }

                            foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies)
                            {
                                if (applicationHealthPolicy.SerivceTypeHealthPolicies == null)
                                {
                                    applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>();
                                }

                                applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value;
                            }
                        }
                    }
                }
            }

            return(upgradeDescription);
        }