public void TestApplcationWithNullDeltaHealthPolicy() { CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper()); PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy(); clusterUpgradePolicy.DeltaHealthPolicy = null; ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth( totalApplicationCount: 6, totalNodeCount: 5, unhealthyApplicationsCount: 0, unhealthyNodeCount: 0); CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync( clusterUpgradePolicy, defaultClusterHealth, CancellationToken.None).Result; VerifyCommandProcessorClusterUpgradeDescription( clusterUpgradePolicy, result, maxPercentageUnhealthyApplication: 100, expectedtMaxPercentUnhealthyPerServicesType: null); }
/// <summary> /// Serializes the object to JSON. /// </summary> /// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param> /// <param name="obj">The object to serialize to JSON.</param> internal static void Serialize(JsonWriter writer, ClusterHealth obj) { // Required properties are always serialized, optional properties are serialized when not null. writer.WriteStartObject(); writer.WriteProperty(obj.AggregatedHealthState, "AggregatedHealthState", HealthStateConverter.Serialize); if (obj.HealthEvents != null) { writer.WriteEnumerableProperty(obj.HealthEvents, "HealthEvents", HealthEventConverter.Serialize); } if (obj.UnhealthyEvaluations != null) { writer.WriteEnumerableProperty(obj.UnhealthyEvaluations, "UnhealthyEvaluations", HealthEvaluationWrapperConverter.Serialize); } if (obj.HealthStatistics != null) { writer.WriteProperty(obj.HealthStatistics, "HealthStatistics", HealthStatisticsConverter.Serialize); } if (obj.NodeHealthStates != null) { writer.WriteEnumerableProperty(obj.NodeHealthStates, "NodeHealthStates", NodeHealthStateConverter.Serialize); } if (obj.ApplicationHealthStates != null) { writer.WriteEnumerableProperty(obj.ApplicationHealthStates, "ApplicationHealthStates", ApplicationHealthStateConverter.Serialize); } writer.WriteEndObject(); }
public void ClusterHealthSerializationTest() { ClusterHealth clusterHealth = this.random.CreateRandom <ClusterHealth>(); var appHS = new ApplicationHealthState() { ApplicationName = new Uri("fabric:/" + "testAppManifest" + this.random.CreateRandom <string>()), AggregatedHealthState = this.random.CreateRandom <HealthState>() }; var nodeHS = new NodeHealthState() { NodeName = "NodeName_" + this.random.CreateRandom <string>(), AggregatedHealthState = this.random.CreateRandom <HealthState>() }; clusterHealth.ApplicationHealthStates = new List <ApplicationHealthState>() { appHS }; clusterHealth.NodeHealthStates = new List <NodeHealthState>() { nodeHS }; TestUsingSerializer(this.Serializer, clusterHealth); }
/// <summary> /// Reports the overall cluster health to the telemetry provider. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task ReportClusterHealthAsync(CancellationToken cancellationToken) { // Called from RunAsync, don't let an exception out so the service will start, but log the exception because the service won't work. try { ClusterHealth health = await _client.HealthManager.GetClusterHealthAsync(TimeSpan.FromSeconds(4), cancellationToken); if (null != health) { // Report the aggregated cluster health. await this._telemetry.ReportHealthAsync( this.Context.ServiceName.AbsoluteUri, this.Context.PartitionId.ToString(), this.Context.ReplicaOrInstanceId.ToString(), "Cluster", "Aggregated Cluster Health", health.AggregatedHealthState, cancellationToken); // Get the state of each of the applications running within the cluster. Report anything that is unhealthy. foreach (ApplicationHealthState appHealth in health.ApplicationHealthStates) { if (HealthState.Ok != appHealth.AggregatedHealthState) { await this._telemetry.ReportHealthAsync( appHealth.ApplicationName.AbsoluteUri, this.Context.ServiceName.AbsoluteUri, this.Context.PartitionId.ToString(), this.Context.ReplicaOrInstanceId.ToString(), this.Context.NodeContext.NodeName, appHealth.AggregatedHealthState, cancellationToken); } } // Get the state of each of the nodes running within the cluster. foreach (NodeHealthState nodeHealth in health.NodeHealthStates) { if (HealthState.Ok != nodeHealth.AggregatedHealthState) { await this._telemetry.ReportHealthAsync( this.Context.NodeContext.NodeName, this.Context.ServiceName.AbsoluteUri, this.Context.PartitionId.ToString(), this.Context.NodeContext.NodeType, this.Context.NodeContext.IPAddressOrFQDN, nodeHealth.AggregatedHealthState, cancellationToken); } } } } catch (Exception ex) { ServiceEventSource.Current.Error($"Exception: {ex.Message} at {ex.StackTrace}."); } }
private async Task ProcessClusterUpgradeAsync( PaasClusterUpgradeDescription upgradeDesc, CancellationToken token) { var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, token); var clusterHealthQueryTask = this.fabricClientWrapper.GetClusterHealthAsync(Constants.MaxOperationTimeout, token); await Task.WhenAll( upgradeProgressTask, clusterHealthQueryTask); FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask); ClusterHealth currentClusterHealth = GetResultFromTask(clusterHealthQueryTask); ClusterUpgradeCommandParameter upgradeCommandParameter = null; try { upgradeCommandParameter = await this.commandParameterGenerator.GetCommandParameterAsync( currentUpgradeProgress, currentClusterHealth, upgradeDesc, token); await UpgradeClusterAsync(upgradeCommandParameter, token); } finally { Cleanup(upgradeCommandParameter); } }
private static async Task DumpClusterHealthAsync(FabricClient fc) { FabricClient.HealthClient hm = fc.HealthManager; ClusterHealth clusterHealth = await hm.GetClusterHealthAsync(); WriteLine($"Cluster: State={clusterHealth.AggregatedHealthState}"); foreach (HealthEvent healthEvent in clusterHealth.HealthEvents) { healthEvent.WriteHealth(); } foreach (HealthEvaluation healthEval in clusterHealth.UnhealthyEvaluations) { WriteLine(healthEval); } foreach (NodeHealthState nodeHealth in clusterHealth.NodeHealthStates) { WriteLine($"Node: State={nodeHealth.AggregatedHealthState}, Name={nodeHealth.NodeName}"); } foreach (ApplicationHealthState appHealthState in clusterHealth.ApplicationHealthStates) { WriteLine($"App: State={appHealthState.AggregatedHealthState}, Name={appHealthState.ApplicationName}"); } //await hm.GetNodeHealthAsync() ApplicationHealth appHealth = await hm.GetApplicationHealthAsync(new Uri(@"fabric:/")); WriteLine($"App: State={appHealth.AggregatedHealthState}, Name={appHealth.ApplicationName}"); foreach (HealthEvent healthEvent in appHealth.HealthEvents) { healthEvent.WriteHealth(); } }
public async Task <ClusterInfo> Info() { NodeList nodes = await this.query.GetNodesAsync(); ApplicationTypeList appTypes = await this.query.GetApplicationTypesAsync(); ApplicationList applications = await this.query.GetApplicationsAsync(); ClusterLoadInformation clusterLoadInfo = await this.query.GetClusterLoadAsync(); ClusterHealth clusterHealth = await this.query.GetClusterHealthAsync(); ProvisionedFabricCodeVersion version = await this.query.GetFabricVersion(); long serviceCount = 0; long partitionCount = 0; long replicaCount = 0; foreach (Node node in nodes) { DeployedApplicationList deployedApplicationList = await this.query.GetDeployedApplicationsAsync(node.NodeName); foreach (DeployedApplication deployedApplication in deployedApplicationList) { DeployedServiceReplicaList deployedReplicas = await this.query.GetDeployedReplicasAsync(node.NodeName, deployedApplication.ApplicationName); replicaCount += deployedReplicas.Count; } } foreach (Application application in applications) { ServiceList services = await this.query.GetServicesAsync(application.ApplicationName); serviceCount += services.Count; } return(new ClusterInfo( clusterHealth.AggregatedHealthState.ToString(), version != null ? version.CodeVersion : "not based", nodes.Select(x => x.NodeType).Distinct().Count(), appTypes.Count(), nodes.Select(x => x.FaultDomain.ToString()).Distinct().Count(), nodes.Select(x => x.UpgradeDomain).Distinct().Count(), nodes.Count, applications.Count, serviceCount, partitionCount, // TODO: partition count replicaCount, clusterLoadInfo.LastBalancingStartTimeUtc, clusterLoadInfo.LastBalancingEndTimeUtc)); }
/// <summary> /// Overloaded ToString function for formatting the output on the console. /// </summary> /// <param name="clusterHealth"> Object of type ClusterHealth </param> /// <returns> /// Returns formatted string. /// </returns> public static string ToString(ClusterHealth clusterHealth) { var strBuilder = new StringBuilder(); strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "AggregatedHealthState", clusterHealth.AggregatedHealthState)); strBuilder.Append(Environment.NewLine); strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "NodeHealthStates", OutputFormatter.ToString(clusterHealth.NodeHealthStates.ToList()))); strBuilder.Append(Environment.NewLine); strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "ApplicationHealthStates", OutputFormatter.ToString(clusterHealth.ApplicationHealthStates.ToList()))); strBuilder.Append(Environment.NewLine); strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "HealthEvents", OutputFormatter.ToString(clusterHealth.HealthEvents.ToList()))); strBuilder.Append(Environment.NewLine); strBuilder.Append(string.Format(CultureInfo.CurrentCulture, "{0} : {1}", "HealthStatistics", OutputFormatter.ToString(clusterHealth.HealthStatistics))); strBuilder.Append(Environment.NewLine); return(strBuilder.ToString()); }
/// <summary> /// Compute cluster health /// </summary> protected virtual void ComputeClusterHealth() { var clusterHealth = new ClusterHealth(); foreach (var performanceCounter in _performanceCounters.ToList()) { if (!clusterHealth.PerformanceCounters.ContainsKey(performanceCounter.Key) && !clusterHealth.PerformanceCounters.TryAdd(performanceCounter.Key, Convert.ToInt64(performanceCounter.Value))) { Logger.LogWarning($"Could not add performance counter {performanceCounter.Key}."); } } ClusterMetrics.Health = clusterHealth; var clusterMetricSubmitted = ClusterMetricSubmitted; clusterMetricSubmitted?.Invoke(this, new ClusterMetricsEventArgs(ClusterMetrics)); }
public void TestDefaultHealthPolicy() { CommandParameterGenerator generator = new CommandParameterGenerator(new TestFabricClientWrapper()); PaasClusterUpgradePolicy defaultPaasClusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicy(); ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth(5, 5); CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync( defaultPaasClusterUpgradePolicy, defaultClusterHealth, CancellationToken.None).Result; VerifyCommandProcessorClusterUpgradeDescription( defaultPaasClusterUpgradePolicy, result, maxPercentageUnhealthyApplication: 0, expectedtMaxPercentUnhealthyPerServicesType: null); }
public void TestApplcationWithUnhealthySystemServiceHealthPolicy() { var testFabricClientWrapper = new TestFabricClientWrapper(); CommandParameterGenerator generator = new CommandParameterGenerator(testFabricClientWrapper); PaasClusterUpgradePolicy clusterUpgradePolicy = CreateDefaultPaasClusterUpgradePolicyWithSystemServicesPolicies(); ClusterHealth defaultClusterHealth = CreateDefaultClusterHealth( totalApplicationCount: 6, totalNodeCount: 5, unhealthyApplicationsCount: 0, unhealthyNodeCount: 0, isSystemServicesUnhealthy: true); Service fssService = new StatefulService(new Uri("fabric:/System/ImageStoreService"), "FileStoreService", "1.0", true, HealthState.Error, ServiceStatus.Active); Service cmService = new StatefulService(new Uri("fabric:/System/ClusterManagerService"), "ClusterManager", "1.0", true, HealthState.Ok, ServiceStatus.Active); Service hmService = new StatefulService(new Uri("fabric:/System/HealthManagerService"), "HealthManager", "1.0", true, HealthState.Ok, ServiceStatus.Active); testFabricClientWrapper.GetServicesResult = new ServiceList(new List <Service>() { fssService, cmService, hmService }); CommandProcessorClusterUpgradeDescription result = generator.GetClusterUpgradeDescriptionAsync( clusterUpgradePolicy, defaultClusterHealth, CancellationToken.None).Result; var expectedtMaxPercentUnhealthyPerServicesType = new Dictionary <string, byte>(); expectedtMaxPercentUnhealthyPerServicesType.Add("FileStoreService", 100); expectedtMaxPercentUnhealthyPerServicesType.Add("ClusterManager", 0); expectedtMaxPercentUnhealthyPerServicesType.Add("HealthManager", 0); VerifyCommandProcessorClusterUpgradeDescription( clusterUpgradePolicy, result, 0, expectedtMaxPercentUnhealthyPerServicesType); }
/// <summary> /// Post the cluster patching status as events on CoordinatorService /// </summary> public async Task PostClusterPatchingStatus(CancellationToken cancellationToken) { try { NodeList nodeList = await this.fabricClient.QueryManager.GetNodeListAsync(null, null, this.DefaultTimeoutForOperation, cancellationToken); IList <RepairTask> claimedTaskList = await this.GetClaimedRepairTasks(nodeList, cancellationToken); RepairTaskList processingTaskList = await this.GetRepairTasksUnderProcessing(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (claimedTaskList.Any()) { if (!processingTaskList.Any()) { // This means that repair tasks are not getting approved. ClusterHealth clusterHealth = await this.fabricClient.HealthManager.GetClusterHealthAsync(); if (clusterHealth.AggregatedHealthState == HealthState.Error) { // Reset Count postUpdateCount = 0; string warningDescription = " Cluster is currently unhealthy. Nodes are currently not getting patched by Patch Orchestration Application. Please ensure the cluster becomes healthy for patching to continue."; await PostWarningOnCoordinatorService(warningDescription, 1); } else { postUpdateCount++; if (postUpdateCount > 60) { // Reset Count and throw a warning on the service saying we dont know the reason. But POA not is not approving tasks. postUpdateCount = 0; string warningDescription = "Patch Orchestration Application is currently not patching nodes. This could be possible if there is some node which is stuck in disabling state for long time."; await PostWarningOnCoordinatorService(warningDescription, 61); } } } else { // Reset Count postUpdateCount = 0; await PostRMTaskNodeUpdate(cancellationToken); } } else { // Reset Count postUpdateCount = 0; if (processingTaskList.Any()) { await PostRMTaskNodeUpdate(cancellationToken); } else { // Post the health event saying that there is no repair task and things are working fine. string description = "No claimed tasks and no processing tasks are found."; HealthManagerHelper.PostNodeHealthReport(this.fabricClient, this.context.ServiceName, ClusterPatchingStatusProperty, description, HealthState.Ok, -1); } } } catch (Exception ex) { ServiceEventSource.Current.ErrorMessage("PostClusterPatchingStatus failed with exception {0}", ex.ToString()); } }
public async Task <ClusterUpgradeCommandParameter> GetCommandParameterAsync( FabricUpgradeProgress currentUpgradeProgress, ClusterHealth currentClusterHealth, PaasClusterUpgradeDescription response, CancellationToken token) { if (response == null || (response.TargetClusterManifestBase64 == null && response.TargetMsiUri == null)) { Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Null response"); return(null); } if (currentClusterHealth == null) { Trace.WriteWarning(TraceType, "currentClusterHealth is null. ContinousNullClusterHealthCount = {0}", this.continousNullClusterHealthCount); if (++continousNullClusterHealthCount < 5) { Trace.WriteWarning(TraceType, "Returning null since upgrade will not be initiated"); return(null); } Trace.WriteWarning(TraceType, "Upgrading even though currentClusterHealth is null."); } else { this.continousNullClusterHealthCount = 0; } Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Processing WRP response"); ClusterManifestType targetClusterManifest = FromBase64ClusterManifest(response.TargetClusterManifestBase64); Trace.WriteInfo(TraceType, "GetCommandParameterAsync: response.TargetMsiVersion = {0}, currentUpgradeProgress.TargetCodeVersion = {1}, targetClusterManifest.Version = {2}, currentUpgradeProgress.UpgradeState = {3}", response.TargetMsiVersion, currentUpgradeProgress != null ? currentUpgradeProgress.TargetCodeVersion : "NA", targetClusterManifest != null ? targetClusterManifest.Version : "NA", currentUpgradeProgress != null ? currentUpgradeProgress.UpgradeState.ToString() : "NA"); string currentCodeVersion = string.Empty; string currentConfigVersion = string.Empty; bool isUpgradeStateFailed = false; if (currentUpgradeProgress != null) { currentCodeVersion = currentUpgradeProgress.TargetCodeVersion; currentConfigVersion = currentUpgradeProgress.TargetConfigVersion; isUpgradeStateFailed = currentUpgradeProgress.UpgradeState == FabricUpgradeState.Failed; } if (response.TargetMsiVersion == currentCodeVersion && (targetClusterManifest != null && targetClusterManifest.Version == currentConfigVersion) && !isUpgradeStateFailed) { return(null); } string clusterConfigFilePath = null; string clusterMsiPath = null; string clusterConfigVersion = null; var info = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())); var tempPath = info.FullName; if (isUpgradeStateFailed || (targetClusterManifest != null && targetClusterManifest.Version != currentConfigVersion)) { clusterConfigFilePath = Path.Combine(tempPath, "ClusterManifest.xml"); using (var fs = new FileStream(clusterConfigFilePath, FileMode.Create)) { using (TextWriter writer = new StreamWriter(fs, new UTF8Encoding())) { // Serialize using the XmlTextWriter. this.clusterManifestSerializer.Serialize(writer, targetClusterManifest); } } clusterConfigVersion = targetClusterManifest.Version; } if (isUpgradeStateFailed || (response.TargetMsiUri != null && response.TargetMsiVersion != currentCodeVersion)) { clusterMsiPath = await DownloadMsiAsync(response.TargetMsiUri, response.TargetMsiVersion, tempPath); } Trace.WriteInfo(TraceType, "ClusterConfigFile {0}, ClusterMsiFile {1}", clusterConfigFilePath, clusterMsiPath); var parameter = new ClusterUpgradeCommandParameter() { CodeVersion = response.TargetMsiVersion, CodeFilePath = clusterMsiPath, ConfigFilePath = clusterConfigFilePath, ConfigVersion = clusterConfigVersion }; parameter.UpgradeDescription = await this.GetClusterUpgradeDescriptionAsync(response.UpgradePolicy, currentClusterHealth, token); return(parameter); }
public async Task <CommandProcessorClusterUpgradeDescription> GetClusterUpgradeDescriptionAsync(PaasClusterUpgradePolicy paasUpgradePolicy, ClusterHealth currentClusterHealth, CancellationToken token) { if (paasUpgradePolicy == null) { return(null); } CommandProcessorClusterUpgradeDescription upgradeDescription = new CommandProcessorClusterUpgradeDescription() { ForceRestart = paasUpgradePolicy.ForceRestart, HealthCheckRetryTimeout = paasUpgradePolicy.HealthCheckRetryTimeout, HealthCheckStableDuration = paasUpgradePolicy.HealthCheckStableDuration, HealthCheckWaitDuration = paasUpgradePolicy.HealthCheckWaitDuration, UpgradeDomainTimeout = paasUpgradePolicy.UpgradeDomainTimeout, UpgradeReplicaSetCheckTimeout = paasUpgradePolicy.UpgradeReplicaSetCheckTimeout, UpgradeTimeout = paasUpgradePolicy.UpgradeTimeout, }; if (paasUpgradePolicy.HealthPolicy == null && paasUpgradePolicy.DeltaHealthPolicy == null) { return(upgradeDescription); } upgradeDescription.HealthPolicy = new CommandProcessorClusterUpgradeHealthPolicy(); var paasHealthPolicy = paasUpgradePolicy.HealthPolicy; if (paasHealthPolicy != null) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasHealthPolicy.MaxPercentUnhealthyApplications; upgradeDescription.HealthPolicy.MaxPercentUnhealthyNodes = paasHealthPolicy.MaxPercentUnhealthyNodes; if (paasHealthPolicy.ApplicationHealthPolicies != null) { upgradeDescription.HealthPolicy.ApplicationHealthPolicies = paasHealthPolicy.ApplicationHealthPolicies.ToDictionary( keyValuePair => keyValuePair.Key, KeyValuePair => KeyValuePair.Value.ToCommondProcessorServiceTypeHealthPolicy()); } } var paasDeltaHealthPolicy = paasUpgradePolicy.DeltaHealthPolicy; if (paasDeltaHealthPolicy != null) { upgradeDescription.DeltaHealthPolicy = new CommandProcessorClusterUpgradeDeltaHealthPolicy() { MaxPercentDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes, MaxPercentUpgradeDomainDeltaUnhealthyNodes = paasDeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes }; if (paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications == 100) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications; } else { int totalAppCount = 0, unhealthyAppCount = 0; if (currentClusterHealth == null) { upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = 0; Trace.WriteWarning( TraceType, "currentClusterHealth is null. Setting MaxPercentUnhealthyApplications conservatively to 0"); } else if (currentClusterHealth.ApplicationHealthStates != null) { var filteredAppHealthStates = currentClusterHealth.ApplicationHealthStates.Where( appHealthState => { if (appHealthState.ApplicationName.OriginalString.Equals("fabric:/System", StringComparison.OrdinalIgnoreCase)) { return(false); } if (paasHealthPolicy != null && paasHealthPolicy.ApplicationHealthPolicies != null && paasHealthPolicy.ApplicationHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString)) { return(false); } if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null && paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies.ContainsKey(appHealthState.ApplicationName.OriginalString)) { return(false); } return(true); }); unhealthyAppCount = filteredAppHealthStates.Count(health => health.AggregatedHealthState == HealthState.Error); totalAppCount = filteredAppHealthStates.Count(); upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications = CommandParameterGenerator.GetMaxUnhealthyPercentage( unhealthyAppCount, totalAppCount, paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications); } Trace.WriteInfo( TraceType, "Delta health policy is specified. MaxPercentUnhealthyApplications is overwritten to {0}. TotalApps={1}, UnhealthyApps={2}, MaxPercentDeltaUnhealthyApplications={3}.", upgradeDescription.HealthPolicy.MaxPercentUnhealthyApplications, totalAppCount, unhealthyAppCount, paasDeltaHealthPolicy.MaxPercentDeltaUnhealthyApplications); } if (paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies != null) { foreach (var applicationDeltaHealthPolicy in paasDeltaHealthPolicy.ApplicationDeltaHealthPolicies) { var applicationName = applicationDeltaHealthPolicy.Key; var paasApplicationDeltaHealthPolicy = applicationDeltaHealthPolicy.Value; if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy == null && paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies == null) { // no policy provided continue; } ApplicationHealthState matchingHealthState = null; if (currentClusterHealth != null) { matchingHealthState = currentClusterHealth.ApplicationHealthStates.FirstOrDefault( appHealthState => appHealthState.ApplicationName.OriginalString.Equals(applicationName, StringComparison.OrdinalIgnoreCase)); } if (matchingHealthState == null) { Trace.WriteWarning( TraceType, "Application {0} is not found in the current cluster health. Ignoring the application since delta policy cannot be computed.", applicationName); // the application is not found in the cluster continue; } Dictionary <string, CommandProcessorServiceTypeHealthPolicy> commandProcessorServiceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>(); var serviceList = await this.fabricClientWrapper.GetServicesAsync(matchingHealthState.ApplicationName, Constants.MaxOperationTimeout, token); // Compute the total and unhealthy services by ServiceType for this application Dictionary <string, HealthStats> serviceTypeHealthStatsDictionary = new Dictionary <string, HealthStats>(); foreach (var service in serviceList) { HealthStats serviceTypeHealthstats; if (!serviceTypeHealthStatsDictionary.TryGetValue(service.ServiceTypeName, out serviceTypeHealthstats)) { serviceTypeHealthstats = new HealthStats(); serviceTypeHealthStatsDictionary.Add(service.ServiceTypeName, serviceTypeHealthstats); } if (service.HealthState == HealthState.Error) { serviceTypeHealthstats.UnhealthyCount++; } serviceTypeHealthstats.TotalCount++; } // For each service type specific healthy policy provided, compute the delta health policy if (paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies != null) { foreach (var serviceTypeHealthPolicyKeyValue in paasApplicationDeltaHealthPolicy.SerivceTypeDeltaHealthPolicies) { var serviceTypeName = serviceTypeHealthPolicyKeyValue.Key; var serviceTypeDeltaPolicy = serviceTypeHealthPolicyKeyValue.Value; HealthStats stats; if (serviceTypeHealthStatsDictionary.TryGetValue(serviceTypeName, out stats)) { byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage(stats.UnhealthyCount, stats.TotalCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices); commandProcessorServiceTypeHealthPolicies.Add( serviceTypeName, new CommandProcessorServiceTypeHealthPolicy() { MaxPercentUnhealthyServices = maxUnhealthyPercentage }); Trace.WriteInfo( TraceType, "Delta health policy is specified for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.", serviceTypeName, applicationName, maxUnhealthyPercentage, stats.TotalCount, stats.UnhealthyCount, serviceTypeDeltaPolicy.MaxPercentDeltaUnhealthyServices); } else { Trace.WriteWarning( TraceType, "ServiceType {0} in Application {1} is not found in the current application. Ignoring the ServiceType since delta policy cannot be computed.", serviceTypeName, applicationName); continue; } } } // If default service type delta policy is specified, compute the delta health policy for ServiceType // which does not have an explicit policy if (paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy != null) { foreach (var serviceTypeHealthStatsKeyValue in serviceTypeHealthStatsDictionary) { var serviceTypeName = serviceTypeHealthStatsKeyValue.Key; var serviceTypeHealthStats = serviceTypeHealthStatsKeyValue.Value; if (commandProcessorServiceTypeHealthPolicies.ContainsKey(serviceTypeName)) { // Explicit policy has been specified continue; } byte maxUnhealthyPercentage = CommandParameterGenerator.GetMaxUnhealthyPercentage( serviceTypeHealthStats.UnhealthyCount, serviceTypeHealthStats.TotalCount, paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices); commandProcessorServiceTypeHealthPolicies.Add( serviceTypeName, new CommandProcessorServiceTypeHealthPolicy() { MaxPercentUnhealthyServices = maxUnhealthyPercentage }); Trace.WriteInfo( TraceType, "Using default delta health policy for ServiceType {0} in Application {1}. MaxPercentUnhealthyServices is overwritten to {2}. TotalCount={3}, UnhealthyCount={4}, MaxPercentDeltaUnhealthyServices={5}.", serviceTypeName, applicationName, maxUnhealthyPercentage, serviceTypeHealthStats.UnhealthyCount, serviceTypeHealthStats.UnhealthyCount, paasApplicationDeltaHealthPolicy.DefaultServiceTypeDeltaHealthPolicy.MaxPercentDeltaUnhealthyServices); } } if (commandProcessorServiceTypeHealthPolicies.Any()) { if (upgradeDescription.HealthPolicy.ApplicationHealthPolicies == null) { upgradeDescription.HealthPolicy.ApplicationHealthPolicies = new Dictionary <string, CommandProcessorApplicationHealthPolicy>(); } CommandProcessorApplicationHealthPolicy applicationHealthPolicy; if (!upgradeDescription.HealthPolicy.ApplicationHealthPolicies.TryGetValue(applicationName, out applicationHealthPolicy)) { applicationHealthPolicy = new CommandProcessorApplicationHealthPolicy() { SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>() }; upgradeDescription.HealthPolicy.ApplicationHealthPolicies.Add( applicationName, applicationHealthPolicy); } foreach (var commandProcessorServiceTypeHealthPolicy in commandProcessorServiceTypeHealthPolicies) { if (applicationHealthPolicy.SerivceTypeHealthPolicies == null) { applicationHealthPolicy.SerivceTypeHealthPolicies = new Dictionary <string, CommandProcessorServiceTypeHealthPolicy>(); } applicationHealthPolicy.SerivceTypeHealthPolicies[commandProcessorServiceTypeHealthPolicy.Key] = commandProcessorServiceTypeHealthPolicy.Value; } } } } } return(upgradeDescription); }