public ClusterCommandProcessor( IConfigStore configStore, string configSectionName, IFabricClientWrapper fabricClientWrapper, CommandParameterGenerator commandParameterGenerator, NodeStatusManager nodeStatusManager, JsonSerializer jsonSerializer) { configStore.ThrowIfNull(nameof(configStore)); configSectionName.ThrowIfNullOrWhiteSpace(nameof(configSectionName)); fabricClientWrapper.ThrowIfNull(nameof(fabricClientWrapper)); commandParameterGenerator.ThrowIfNull(nameof(commandParameterGenerator)); nodeStatusManager.ThrowIfNull(nameof(nodeStatusManager)); jsonSerializer.ThrowIfNull(nameof(jsonSerializer)); this.configStore = configStore; this.configSectionName = configSectionName; this.fabricClientWrapper = fabricClientWrapper; this.commandParameterGenerator = commandParameterGenerator; this.nodeStatusManager = nodeStatusManager; this.serializer = jsonSerializer; }
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token) { if (paasUpgradePolicy == null) { return(null); } CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription() { ForceRestart = paasUpgradePolicy.ForceRestart, HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout, HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration, HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration, UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout, UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout, UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout, }; if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null) { return(upgradeDescription); } upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy(); var paasHealthPolicy = paasUpgradePolicy.HealthPolicy; if (paasHealthPolicy != null) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications; upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes; if (paasHealthPolicy.ApplicationHealthPolicies != null) { upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary( keyValuePair => keyValuePair.Key, KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy()); } } var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy; if (paasDeltaHealthPolicy != null) { upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy() { MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes, MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes }; if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications; } else { int totalAppCount = 0, unhealthyAppCount = 0; if (currentClusterHealth == null) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0; Trace.WriteWarning( TraceType, "currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0"); } else if (currentClusterHealth.ApplicationHealthStates != null) { var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where( appHealthState => { if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase)) { return(false); } if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null && paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString)) { return(false); } if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null && paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString)) { return(false); } return(true); }); unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error); totalAppCount = filteredAppHealthStates.Count(); upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage( unhealthyAppCount, totalAppCount, paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications); } Trace.WriteInfo( TraceType, "Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.", upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications, totalAppCount, unhealthyAppCount, paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications); } if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null) { foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies) { var applicationName = applicationDeltaHealthPolicy.Key; var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value; if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null) { // no policy provided continue; } ApplicationHealthState matchingHealthState = null; if (currentClusterHealth != null) { matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault( appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase)); } if (matchingHealthState == null) { Trace.WriteWarning( TraceType, "Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.", applicationName); // the application is not found in the cluster continue; } Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>(); var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token); // Compute the total and unhealthy services by ServiceType for this application Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>(); foreach (var service in serviceList) { HealthStats serviceTypeHealthstats; if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats)) { serviceTypeHealthstats = new HealthStats(); serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats); } if (service.HealthState == HealthState.Error) { serviceTypeHealthstats.UnhealthyCount++; } serviceTypeHealthstats.TotalCount++; } // For each service type specific healthy policy provided, compute the delta health policy if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null) { foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies) { var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key; var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value; HealthStats stats; if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats)) { byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount, stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices); commandProcessorServiceTypeHealthPolicies.Add( serviceTypeName, new CommandProcessorServiceTypeHealthPolicy() { MaxPercentUnhealthyServices = maxUnhealthyPercentage }); Trace.WriteInfo( TraceType, "Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.", serviceTypeName, applicationName, maxUnhealthyPercentage, stats.TotalCount, stats.UnhealthyCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices); } else { Trace.WriteWarning( TraceType, "ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.", serviceTypeName, applicationName); continue; } } } // If default service type delta policy is specified, compute the delta health policy for ServiceType // which does not have an explicit policy if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null) { foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary) { var serviceTypeName = serviceTypeHealthStatsKeyValue.Key; var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value; if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName)) { // Explicit policy has been specified continue; } byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage( serviceTypeHealthStats.UnhealthyCount, serviceTypeHealthStats.TotalCount, paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices); commandProcessorServiceTypeHealthPolicies.Add( serviceTypeName, new CommandProcessorServiceTypeHealthPolicy() { MaxPercentUnhealthyServices = maxUnhealthyPercentage }); Trace.WriteInfo( TraceType, "Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.", serviceTypeName, applicationName, maxUnhealthyPercentage, serviceTypeHealthStats.UnhealthyCount, serviceTypeHealthStats.UnhealthyCount, paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices); } } if (commandProcessorServiceTypeHealthPolicies.Any()) { if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null) { upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>(); } CommandProcessorApplicationHealthPolicy applicationHealthPolicy; if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy)) { applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy() { SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>() }; upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add( applicationName, applicationHealthPolicy); } foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies) { if (applicationHealthPolicy.SerivceTypeHealthPolicies == null) { applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>(); } applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value; } } } } } return(upgradeDescription); }