protected void AddToPSObject(PSObject itemPSObj, ApplicationHealthPolicy healthPolicy)
{
if (healthPolicy == null)
{
return;
}
itemPSObj.Properties.Add(new PSNoteProperty(Constants.ConsiderWarningAsErrorPropertyName, healthPolicy.ConsiderWarningAsError));
itemPSObj.Properties.Add(
new PSNoteProperty(Constants.MaxPercentUnhealthyDeployedApplicationsPropertyName, healthPolicy.MaxPercentUnhealthyDeployedApplications));
if (healthPolicy.DefaultServiceTypeHealthPolicy != null)
{
itemPSObj.Properties.Add(
new PSNoteProperty(Constants.MaxPercentUnhealthyPartitionsPerServicePropertyName, healthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyPartitionsPerService));
itemPSObj.Properties.Add(
new PSNoteProperty(Constants.MaxPercentUnhealthyReplicasPerPartitionPropertyName, healthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyReplicasPerPartition));
itemPSObj.Properties.Add(
new PSNoteProperty(Constants.MaxPercentUnhealthyServicesPropertyName, healthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices));
}
if (healthPolicy.ServiceTypeHealthPolicyMap != null)
{
var serviceTypeHealthPolicyMapPropertyPSObj = new PSObject(healthPolicy.ServiceTypeHealthPolicyMap);
serviceTypeHealthPolicyMapPropertyPSObj.Members.Add(
new PSCodeMethod(
Constants.ToStringMethodName,
typeof(OutputFormatter).GetMethod(Constants.FormatObjectMethodName)));
itemPSObj.Properties.Add(
new PSNoteProperty(
Constants.ServiceTypeHealthPolicyMapPropertyName,
serviceTypeHealthPolicyMapPropertyPSObj));
}
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);
var applicationHealthPolicy = new ApplicationHealthPolicy(
considerWarningAsError: this.ConsiderWarningAsError,
maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);
var result = this.ServiceFabricClient.Applications.GetApplicationHealthUsingPolicyAsync(
applicationId: this.ApplicationId,
eventsHealthStateFilter: this.EventsHealthStateFilter,
deployedApplicationsHealthStateFilter: this.DeployedApplicationsHealthStateFilter,
servicesHealthStateFilter: this.ServicesHealthStateFilter,
excludeHealthStatistics: this.ExcludeHealthStatistics,
applicationHealthPolicy: applicationHealthPolicy,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
if (result != null)
{
this.WriteObject(this.FormatOutput(result));
}
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ApplicationHealthPolicy obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
if (obj.ConsiderWarningAsError != null)
{
writer.WriteProperty(obj.ConsiderWarningAsError, "ConsiderWarningAsError", JsonWriterExtensions.WriteBoolValue);
}
if (obj.MaxPercentUnhealthyDeployedApplications != null)
{
writer.WriteProperty(obj.MaxPercentUnhealthyDeployedApplications, "MaxPercentUnhealthyDeployedApplications", JsonWriterExtensions.WriteIntValue);
}
if (obj.DefaultServiceTypeHealthPolicy != null)
{
writer.WriteProperty(obj.DefaultServiceTypeHealthPolicy, "DefaultServiceTypeHealthPolicy", ServiceTypeHealthPolicyConverter.Serialize);
}
if (obj.ServiceTypeHealthPolicyMap != null)
{
writer.WriteEnumerableProperty(obj.ServiceTypeHealthPolicyMap, "ServiceTypeHealthPolicyMap", ServiceTypeHealthPolicyMapItemConverter.Serialize);
}
writer.WriteEndObject();
}
new internal static unsafe MonitoredRollingApplicationUpgradePolicyDescription FromNative(IntPtr policyPtr)
{
if (policyPtr == IntPtr.Zero)
{
return(null);
}
var castedPtr = (NativeTypes.FABRIC_ROLLING_UPGRADE_POLICY_DESCRIPTION *)policyPtr;
var monitoringPolicy = new MonitoredRollingApplicationUpgradePolicyDescription();
if (castedPtr->Reserved != IntPtr.Zero)
{
var castedEx1Ptr = (NativeTypes.FABRIC_ROLLING_UPGRADE_POLICY_DESCRIPTION_EX1 *)castedPtr->Reserved;
if (castedEx1Ptr->MonitoringPolicy != IntPtr.Zero)
{
monitoringPolicy.MonitoringPolicy = RollingUpgradeMonitoringPolicy.FromNative(castedEx1Ptr->MonitoringPolicy);
}
if (castedEx1Ptr->HealthPolicy != IntPtr.Zero)
{
monitoringPolicy.HealthPolicy = ApplicationHealthPolicy.FromNative(castedEx1Ptr->HealthPolicy);
}
}
monitoringPolicy.FromNativeHelper(policyPtr);
return(monitoringPolicy);
}
/// <inheritdoc />
public Task <DeployedApplicationHealth> GetDeployedApplicationHealthUsingPolicyAsync(
NodeName nodeName,
string applicationId,
int?eventsHealthStateFilter = 0,
int?deployedServicePackagesHealthStateFilter = 0,
ApplicationHealthPolicy applicationHealthPolicy = default(ApplicationHealthPolicy),
bool?excludeHealthStatistics = false,
long?serverTimeout = 60,
CancellationToken cancellationToken = default(CancellationToken))
{
nodeName.ThrowIfNull(nameof(nodeName));
applicationId.ThrowIfNull(nameof(applicationId));
serverTimeout?.ThrowIfOutOfInclusiveRange("serverTimeout", 1, 4294967295);
var requestId = Guid.NewGuid().ToString();
var url = "Nodes/{nodeName}/$/GetApplications/{applicationId}/$/GetHealth";
url = url.Replace("{nodeName}", Uri.EscapeDataString(nodeName.ToString()));
url = url.Replace("{applicationId}", applicationId);
var queryParams = new List <string>();
// Append to queryParams if not null.
eventsHealthStateFilter?.AddToQueryParameters(queryParams, $"EventsHealthStateFilter={eventsHealthStateFilter}");
deployedServicePackagesHealthStateFilter?.AddToQueryParameters(queryParams, $"DeployedServicePackagesHealthStateFilter={deployedServicePackagesHealthStateFilter}");
excludeHealthStatistics?.AddToQueryParameters(queryParams, $"ExcludeHealthStatistics={excludeHealthStatistics}");
serverTimeout?.AddToQueryParameters(queryParams, $"timeout={serverTimeout}");
queryParams.Add("api-version=6.0");
url += "?" + string.Join("&", queryParams);
string content;
using (var sw = new StringWriter())
{
if (applicationHealthPolicy != default(ApplicationHealthPolicy))
{
ApplicationHealthPolicyConverter.Serialize(new JsonTextWriter(sw), applicationHealthPolicy);
}
content = sw.ToString();
}
HttpRequestMessage RequestFunc()
{
var request = new HttpRequestMessage()
{
Method = HttpMethod.Post,
Content = new StringContent(content, Encoding.UTF8),
};
request.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
return(request);
}
return(this.httpClient.SendAsyncGetResponse(RequestFunc, url, DeployedApplicationHealthConverter.Deserialize, requestId, cancellationToken));
}
public void ApplicationHealthPolicySerializationTest()
{
// Default
ApplicationHealthPolicy healthPolicy = new ApplicationHealthPolicy();
TestUsingSerializer(this.Serializer, healthPolicy);
ApplicationHealthPolicy healthPolicy2 = this.random.CreateRandom <ApplicationHealthPolicy>();
TestUsingSerializer(this.Serializer, healthPolicy2);
}
/// <inheritdoc />
public Task <ReplicaHealth> GetReplicaHealthUsingPolicyAsync(
PartitionId partitionId,
ReplicaId replicaId,
int?eventsHealthStateFilter = 0,
ApplicationHealthPolicy applicationHealthPolicy = default(ApplicationHealthPolicy),
long?serverTimeout = 60,
CancellationToken cancellationToken = default(CancellationToken))
{
partitionId.ThrowIfNull(nameof(partitionId));
replicaId.ThrowIfNull(nameof(replicaId));
serverTimeout?.ThrowIfOutOfInclusiveRange("serverTimeout", 1, 4294967295);
var requestId = Guid.NewGuid().ToString();
var url = "Partitions/{partitionId}/$/GetReplicas/{replicaId}/$/GetHealth";
url = url.Replace("{partitionId}", partitionId.ToString());
url = url.Replace("{replicaId}", replicaId.ToString());
var queryParams = new List <string>();
// Append to queryParams if not null.
eventsHealthStateFilter?.AddToQueryParameters(queryParams, $"EventsHealthStateFilter={eventsHealthStateFilter}");
serverTimeout?.AddToQueryParameters(queryParams, $"timeout={serverTimeout}");
queryParams.Add("api-version=6.0");
url += "?" + string.Join("&", queryParams);
string content;
using (var sw = new StringWriter())
{
ApplicationHealthPolicyConverter.Serialize(new JsonTextWriter(sw), applicationHealthPolicy);
content = sw.ToString();
}
HttpRequestMessage RequestFunc()
{
var request = new HttpRequestMessage()
{
Method = HttpMethod.Post,
Content = new StringContent(content, Encoding.UTF8),
};
request.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
return(request);
}
return(this.httpClient.SendAsyncGetResponse(RequestFunc, url, ReplicaHealthConverter.Deserialize, requestId, cancellationToken));
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var registryCredential = new RegistryCredential(
registryUserName: this.RegistryUserName,
registryPassword: this.RegistryPassword,
passwordEncrypted: this.PasswordEncrypted);
var monitoringPolicyDescription = new MonitoringPolicyDescription(
failureAction: this.FailureAction,
healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds);
var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);
var applicationHealthPolicy = new ApplicationHealthPolicy(
considerWarningAsError: this.ConsiderWarningAsError,
maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);
var composeDeploymentUpgradeDescription = new ComposeDeploymentUpgradeDescription(
deploymentName: this.DeploymentName,
composeFileContent: this.ComposeFileContent,
upgradeKind: this.UpgradeKind,
registryCredential: registryCredential,
rollingUpgradeMode: this.RollingUpgradeMode,
upgradeReplicaSetCheckTimeoutInSeconds: this.UpgradeReplicaSetCheckTimeoutInSeconds,
forceRestart: this.ForceRestart,
monitoringPolicy: monitoringPolicyDescription,
applicationHealthPolicy: applicationHealthPolicy);
this.ServiceFabricClient.ComposeDeployments.StartComposeDeploymentUpgradeAsync(
deploymentName: this.DeploymentName,
composeDeploymentUpgradeDescription: composeDeploymentUpgradeDescription,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var monitoringPolicyDescription = new MonitoringPolicyDescription(
failureAction: this.FailureAction,
healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds);
var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);
var applicationHealthPolicy = new ApplicationHealthPolicy(
considerWarningAsError: this.ConsiderWarningAsError,
maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);
var applicationUpgradeDescription = new ApplicationUpgradeDescription(
name: $"fabric:/{this.ApplicationId}",
targetApplicationTypeVersion: this.TargetApplicationTypeVersion,
parameters: this.Parameters?.ToDictionary <string, string>(),
upgradeKind: this.UpgradeKind,
rollingUpgradeMode: this.RollingUpgradeMode,
upgradeReplicaSetCheckTimeoutInSeconds: this.UpgradeReplicaSetCheckTimeoutInSeconds,
forceRestart: this.ForceRestart,
monitoringPolicy: monitoringPolicyDescription,
applicationHealthPolicy: applicationHealthPolicy);
this.ServiceFabricClient.Applications.StartApplicationUpgradeAsync(
applicationId: this.ApplicationId,
applicationUpgradeDescription: applicationUpgradeDescription,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
var serviceTypeHealthPolicy = new ServiceTypeHealthPolicy(
maxPercentUnhealthyPartitionsPerService: this.MaxPercentUnhealthyPartitionsPerService,
maxPercentUnhealthyReplicasPerPartition: this.MaxPercentUnhealthyReplicasPerPartition,
maxPercentUnhealthyServices: this.MaxPercentUnhealthyServices);
var applicationHealthPolicy = new ApplicationHealthPolicy(
considerWarningAsError: this.ConsiderWarningAsError,
maxPercentUnhealthyDeployedApplications: this.MaxPercentUnhealthyDeployedApplications,
defaultServiceTypeHealthPolicy: serviceTypeHealthPolicy,
serviceTypeHealthPolicyMap: this.ServiceTypeHealthPolicyMap);
var rollingUpgradeUpdateDescription = new RollingUpgradeUpdateDescription(
rollingUpgradeMode: this.RollingUpgradeMode,
forceRestart: this.ForceRestart,
replicaSetCheckTimeoutInMilliseconds: this.ReplicaSetCheckTimeoutInMilliseconds,
failureAction: this.FailureAction,
healthCheckWaitDurationInMilliseconds: this.HealthCheckWaitDurationInMilliseconds,
healthCheckStableDurationInMilliseconds: this.HealthCheckStableDurationInMilliseconds,
healthCheckRetryTimeoutInMilliseconds: this.HealthCheckRetryTimeoutInMilliseconds,
upgradeTimeoutInMilliseconds: this.UpgradeTimeoutInMilliseconds,
upgradeDomainTimeoutInMilliseconds: this.UpgradeDomainTimeoutInMilliseconds,
instanceCloseDelayDurationInSeconds: this.InstanceCloseDelayDurationInSeconds);
var applicationUpgradeUpdateDescription = new ApplicationUpgradeUpdateDescription(
name: this.Name,
upgradeKind: this.UpgradeKind,
applicationHealthPolicy: applicationHealthPolicy,
updateDescription: rollingUpgradeUpdateDescription);
this.ServiceFabricClient.Applications.UpdateApplicationUpgradeAsync(
applicationId: this.ApplicationId,
applicationUpgradeUpdateDescription: applicationUpgradeUpdateDescription,
serverTimeout: this.ServerTimeout,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
Console.WriteLine("Success!");
}
protected override void ProcessRecord()
{
try
{
if (this.ReplicaQuorumTimeoutSec.HasValue)
{
this.WriteWarning(StringResources.PowerShell_ReplicaQuorumTimeoutSec_Deprecated);
if (!this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
{
this.UpgradeReplicaSetCheckTimeoutSec = this.ReplicaQuorumTimeoutSec.Value;
}
}
if (this.RestartProcess)
{
this.WriteWarning(StringResources.PowerShell_RestartProcess_Deprecated);
if (!this.ForceRestart)
{
this.ForceRestart = this.RestartProcess;
}
}
RollingUpgradePolicyDescription policyDescription;
if (this.Monitored)
{
var monitoringPolicy = new RollingUpgradeMonitoringPolicy();
if (this.FailureAction == UpgradeFailureAction.Invalid)
{
this.FailureAction = Constants.DefaultUpgradeFailureAction;
}
monitoringPolicy.FailureAction = this.FailureAction;
if (this.HealthCheckRetryTimeoutSec.HasValue)
{
monitoringPolicy.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value);
}
if (this.HealthCheckWaitDurationSec.HasValue)
{
monitoringPolicy.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value);
}
if (this.HealthCheckStableDurationSec.HasValue)
{
monitoringPolicy.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value);
}
if (this.UpgradeDomainTimeoutSec.HasValue)
{
monitoringPolicy.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value);
}
if (this.UpgradeTimeoutSec.HasValue)
{
monitoringPolicy.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value);
}
var monitoredPolicyDescription = new MonitoredRollingApplicationUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.Monitored,
ForceRestart = this.ForceRestart,
MonitoringPolicy = monitoringPolicy
};
policyDescription = monitoredPolicyDescription;
if (this.IsUpdatingHealthPolicy())
{
if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess(
//// description shows up for "-WhatIf"
string.Format(
CultureInfo.InvariantCulture,
"{0} {1}",
StringResources.PowerShell_HealthPolicyUpgradeCaption,
StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning),
//// warning and caption show up when prompting for confirmation
StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning,
StringResources.PowerShell_HealthPolicyUpgradeCaption))
{
return;
}
var healthPolicy = new ApplicationHealthPolicy();
if (this.ConsiderWarningAsError.HasValue)
{
healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value;
}
if (this.DefaultServiceTypeHealthPolicy != null)
{
healthPolicy.DefaultServiceTypeHealthPolicy = this.ParseServiceTypeHealthPolicy(this.DefaultServiceTypeHealthPolicy);
}
if (this.MaxPercentUnhealthyDeployedApplications.HasValue)
{
healthPolicy.MaxPercentUnhealthyDeployedApplications = this.MaxPercentUnhealthyDeployedApplications.Value;
}
if (this.ServiceTypeHealthPolicyMap != null)
{
foreach (DictionaryEntry entry in this.ServiceTypeHealthPolicyMap)
{
healthPolicy.ServiceTypeHealthPolicyMap.Add(entry.Key as string, this.ParseServiceTypeHealthPolicy(entry.Value as string));
}
}
monitoredPolicyDescription.HealthPolicy = healthPolicy;
}
}
else if (this.UnmonitoredManual)
{
policyDescription = new RollingUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.UnmonitoredManual,
ForceRestart = this.ForceRestart,
};
}
else
{
policyDescription = new RollingUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.UnmonitoredAuto,
ForceRestart = this.ForceRestart,
};
}
if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value);
}
var upgradeDescription = new ApplicationUpgradeDescription
{
ApplicationName = this.ApplicationName,
TargetApplicationTypeVersion = this.ApplicationTypeVersion,
UpgradePolicyDescription = policyDescription
};
upgradeDescription.ApplicationParameters.Add(this.GetNameValueCollection(this.ApplicationParameter));
this.UpgradeApplication(upgradeDescription);
}
catch (Exception exception)
{
this.ThrowTerminatingError(
exception,
Constants.UpgradeApplicationErrorId,
null);
}
}
protected override void ProcessRecord()
{
try
{
var updateDescription = new ApplicationUpgradeUpdateDescription()
{
ApplicationName = this.ApplicationName,
};
if (this.ForceRestart.HasValue)
{
updateDescription.ForceRestart = this.ForceRestart.Value;
}
if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
{
updateDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value);
}
if (this.UpgradeMode.HasValue)
{
updateDescription.UpgradeMode = this.UpgradeMode.Value;
}
if (this.FailureAction.HasValue)
{
updateDescription.FailureAction = this.FailureAction.Value;
}
if (this.HealthCheckWaitDurationSec.HasValue)
{
updateDescription.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value);
}
if (this.HealthCheckStableDurationSec.HasValue)
{
updateDescription.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value);
}
if (this.HealthCheckRetryTimeoutSec.HasValue)
{
updateDescription.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value);
}
if (this.UpgradeTimeoutSec.HasValue)
{
updateDescription.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value);
}
if (this.UpgradeDomainTimeoutSec.HasValue)
{
updateDescription.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value);
}
if (this.IsUpdatingHealthPolicy())
{
if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess(
//// description shows up for "-WhatIf"
string.Format(
CultureInfo.InvariantCulture,
"{0} {1}",
StringResources.PowerShell_HealthPolicyUpdateCaption,
StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning),
//// warning and caption show up when prompting for confirmation
StringResources.PowerShell_ApplicationHealthPolicyUpdateWarning,
StringResources.PowerShell_HealthPolicyUpdateCaption))
{
return;
}
var healthPolicy = new ApplicationHealthPolicy();
if (this.ConsiderWarningAsError.HasValue)
{
healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value;
}
if (this.DefaultServiceTypeHealthPolicy != null)
{
healthPolicy.DefaultServiceTypeHealthPolicy = this.ParseServiceTypeHealthPolicy(this.DefaultServiceTypeHealthPolicy);
}
if (this.MaxPercentUnhealthyDeployedApplications.HasValue)
{
healthPolicy.MaxPercentUnhealthyDeployedApplications = this.MaxPercentUnhealthyDeployedApplications.Value;
}
if (this.ServiceTypeHealthPolicyMap != null)
{
foreach (DictionaryEntry entry in this.ServiceTypeHealthPolicyMap)
{
healthPolicy.ServiceTypeHealthPolicyMap.Add(entry.Key as string, this.ParseServiceTypeHealthPolicy(entry.Value as string));
}
}
updateDescription.HealthPolicy = healthPolicy;
}
this.UpdateApplicationUpgrade(updateDescription);
}
catch (Exception exception)
{
this.ThrowTerminatingError(
exception,
Constants.UpdateApplicationUpgradeErrorId,
null);
}
}
public override async Task UpdateAsync(IOperationDescription description, IFabricOperationResult result, IOperationContext context)
{
Trace.WriteInfo(TraceType, "UpdateAsync called");
description.ThrowIfNull(nameof(description));
result.ThrowIfNull(nameof(result));
result.OperationStatus.ThrowIfNull(nameof(result.OperationStatus));
result.QueryResult.ThrowIfNull(nameof(result.QueryResult));
context.ThrowIfNull(nameof(context));
string errorMessage;
if (!this.ValidObjectType<ApplicationOperationDescription>(description, out errorMessage))
{
throw new InvalidCastException(errorMessage);
}
if (!this.ValidObjectType<ApplicationFabricQueryResult>(result.QueryResult, out errorMessage))
{
throw new InvalidCastException(errorMessage);
}
var appDescription = (ApplicationOperationDescription)description;
appDescription.TypeVersion.ThrowIfNullOrWhiteSpace(nameof(appDescription.TypeVersion));
appDescription.ApplicationUri.ThrowIfNull(nameof(appDescription.ApplicationUri));
var appResult = (ApplicationFabricQueryResult)result.QueryResult;
var healthPolicy = new ApplicationHealthPolicy();
var defaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy();
if (appDescription.UpgradePolicy?.ApplicationHealthPolicy != null)
{
healthPolicy.ConsiderWarningAsError =
appDescription.UpgradePolicy.ApplicationHealthPolicy.ConsiderWarningAsError;
healthPolicy.MaxPercentUnhealthyDeployedApplications =
appDescription.UpgradePolicy.ApplicationHealthPolicy.MaxPercentUnhealthyDeployedApplications;
if (appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
defaultServiceTypeHealthPolicy.MaxPercentUnhealthyPartitionsPerService =
appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
.MaxPercentUnhealthyPartitionsPerService;
defaultServiceTypeHealthPolicy.MaxPercentUnhealthyReplicasPerPartition =
appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
.MaxPercentUnhealthyReplicasPerPartition;
defaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices =
appDescription.UpgradePolicy.ApplicationHealthPolicy.DefaultServiceTypeHealthPolicy
.MaxPercentUnhealthyServices;
healthPolicy.DefaultServiceTypeHealthPolicy = defaultServiceTypeHealthPolicy;
}
}
var monitoringPolicy = new RollingUpgradeMonitoringPolicy();
if (appDescription.UpgradePolicy?.RollingUpgradeMonitoringPolicy != null)
{
monitoringPolicy.FailureAction =
(UpgradeFailureAction) Enum.Parse(
typeof (UpgradeFailureAction),
appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.FailureAction.ToString(),
true);
monitoringPolicy.HealthCheckRetryTimeout =
appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckRetryTimeout;
monitoringPolicy.HealthCheckStableDuration =
appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckStableDuration;
monitoringPolicy.HealthCheckWaitDuration =
appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.HealthCheckWaitDuration;
monitoringPolicy.UpgradeDomainTimeout =
appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeDomainTimeout;
monitoringPolicy.UpgradeTimeout = appDescription.UpgradePolicy.RollingUpgradeMonitoringPolicy.UpgradeTimeout;
}
var policyDescription = new MonitoredRollingApplicationUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
HealthPolicy = healthPolicy,
MonitoringPolicy = monitoringPolicy
};
if (appDescription.UpgradePolicy != null)
{
if (appDescription.UpgradePolicy.ForceRestart.HasValue)
{
policyDescription.ForceRestart = appDescription.UpgradePolicy.ForceRestart.Value;
}
if (appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout.Value;
}
}
// Update
var updateDescription = new ApplicationUpdateDescription(appDescription.ApplicationUri)
{
RemoveApplicationCapacity = appDescription.RemoveApplicationCapacity
};
if (appDescription.MinimumNodes.HasValue)
{
updateDescription.MinimumNodes = appDescription.MinimumNodes.Value;
}
if (appDescription.MaximumNodes.HasValue)
{
updateDescription.MaximumNodes = appDescription.MaximumNodes.Value;
}
if (appDescription.Metrics != null && appDescription.Metrics.Any())
{
updateDescription.Metrics = new List<ApplicationMetricDescription>();
foreach (var metric in appDescription.Metrics)
{
updateDescription.Metrics.Add(
new ApplicationMetricDescription()
{
Name = metric.Name,
MaximumNodeCapacity = metric.MaximumCapacity,
NodeReservationCapacity = metric.ReservationCapacity,
TotalApplicationCapacity = metric.TotalApplicationCapacity
});
}
}
Trace.WriteInfo(
TraceType,
"UpdateAsync: Updating application. Name: {0}. Timeout: {1}",
updateDescription.ApplicationName,
context.OperationTimeout);
await this.fabricClient.ApplicationManager.UpdateApplicationAsync(
updateDescription,
context.OperationTimeout,
context.CancellationToken);
Trace.WriteInfo(TraceType, "UpdateAsync: Application update call accepted");
// Upgrade
var upgradeDescription = new ApplicationUpgradeDescription()
{
ApplicationName = appDescription.ApplicationUri,
TargetApplicationTypeVersion = appDescription.TypeVersion,
UpgradePolicyDescription = policyDescription
};
if (appDescription.Parameters != null)
{
upgradeDescription.ApplicationParameters.Add(appDescription.Parameters.ToNameValueCollection());
}
try
{
Trace.WriteInfo(
TraceType,
"UpdateAsync: Upgrading application. Name: {0}. Version: {1}. TargetVersion: {2}. Timeout: {3}",
upgradeDescription.ApplicationName,
appResult.Application.ApplicationTypeVersion,
upgradeDescription.TargetApplicationTypeVersion,
context.OperationTimeout);
await this.fabricClient.ApplicationManager.UpgradeApplicationAsync(
upgradeDescription,
context.OperationTimeout,
context.CancellationToken);
Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade call accepted");
}
catch (FabricException fe)
{
if (fe.ErrorCode == FabricErrorCode.ApplicationAlreadyInTargetVersion)
{
Trace.WriteInfo(TraceType, "UpdateAsync: Application already in target version: {0}.", upgradeDescription.TargetApplicationTypeVersion);
return;
}
if (fe.ErrorCode == FabricErrorCode.ApplicationUpgradeInProgress)
{
Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade in progress with same version: {0}", upgradeDescription.TargetApplicationTypeVersion);
}
else
{
Trace.WriteError(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", fe);
throw;
}
}
catch (Exception e)
{
Trace.WriteInfo(TraceType, "UpdateAsync: Application upgrade encountered an exception: {0}", e);
throw;
}
// UpgradeUpdate
ApplicationUpgradeProgress upgradeProgress = null;
try
{
upgradeProgress = await this.fabricClient.ApplicationManager.GetApplicationUpgradeProgressAsync(
appDescription.ApplicationUri,
context.OperationTimeout,
context.CancellationToken);
}
catch (Exception)
{
Trace.WriteWarning(
TraceType,
"UpdateAsync: Failed to get the application upgrade progress to determine the parameters for upgrade update. Need to retry the operation. Application Name: {0}",
appDescription.ApplicationUri);
throw new FabricTransientException();
}
ApplicationUpgradeUpdateDescription upgradeUpdateDescription;
if (upgradeProgress.UpgradeState == ApplicationUpgradeState.RollingBackInProgress)
{
// During rollback we can only change below properties
upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
{
ApplicationName = appDescription.ApplicationUri,
UpgradeReplicaSetCheckTimeout = appDescription.UpgradePolicy.UpgradeReplicaSetCheckTimeout,
ForceRestart = appDescription.UpgradePolicy.ForceRestart
};
}
else
{
upgradeUpdateDescription = new ApplicationUpgradeUpdateDescription()
{
ApplicationName = appDescription.ApplicationUri,
UpgradeDomainTimeout = monitoringPolicy.UpgradeDomainTimeout,
UpgradeTimeout = monitoringPolicy.UpgradeTimeout,
HealthCheckRetryTimeout = monitoringPolicy.HealthCheckRetryTimeout,
HealthCheckWaitDuration = monitoringPolicy.HealthCheckWaitDuration,
HealthCheckStableDuration = monitoringPolicy.HealthCheckStableDuration,
UpgradeReplicaSetCheckTimeout = policyDescription.UpgradeReplicaSetCheckTimeout,
ForceRestart = policyDescription.ForceRestart,
FailureAction = monitoringPolicy.FailureAction,
HealthPolicy = healthPolicy
};
}
try
{
Trace.WriteInfo(
TraceType,
"UpdateAsync: Updating application upgrade in progress. Name: {0}. Timeout: {1}",
upgradeUpdateDescription.ApplicationName,
context.OperationTimeout);
await this.fabricClient.ApplicationManager.UpdateApplicationUpgradeAsync(
upgradeUpdateDescription,
context.OperationTimeout,
context.CancellationToken);
Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade call accepted");
}
catch (FabricException fe)
{
if (fe.ErrorCode == FabricErrorCode.ApplicationNotUpgrading)
{
Trace.WriteInfo(TraceType, "UpdateAsync: No application upgrade in progress");
return;
}
}
catch (Exception e)
{
Trace.WriteInfo(TraceType, "UpdateAsync: Update application upgrade encountered an exception: {0}", e);
throw;
}
}
/// <summary>
/// This function starts the upgrade process
/// </summary>
/// <param name="commandDescription"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task StartUpgradeFabricAsync(
CommandProcessorClusterUpgradeDescription commandDescription,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
Trace.WriteInfo(
TraceType,
"StartUpgradeFabricAsync - Started");
var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
{
FailureAction = UpgradeFailureAction.Rollback
};
var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
MonitoringPolicy = rollingUpgradeMonitoringPolicy,
};
if (commandDescription != null)
{
if (commandDescription.HealthCheckRetryTimeout.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
}
if (commandDescription.HealthCheckStableDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
}
if (commandDescription.HealthCheckWaitDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
}
if (commandDescription.UpgradeDomainTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
}
if (commandDescription.UpgradeTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
}
if (commandDescription.ForceRestart.HasValue)
{
policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
}
if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
}
if (commandDescription.HealthPolicy != null)
{
policyDescription.HealthPolicy = new ClusterHealthPolicy
{
MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
};
if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
{
foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
{
CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
commandProcessorApplicationHealthPolicyKeyValue.Value;
if (commandProcessorApplicationHealthPolicy == null)
{
continue;
}
var applicationHealthPolicy = new ApplicationHealthPolicy();
if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
};
}
if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
{
foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
{
if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
{
continue;
}
ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
};
applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
}
}
policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
}
}
}
if (commandDescription.DeltaHealthPolicy != null)
{
policyDescription.EnableDeltaHealthEvaluation = true;
policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
}
}
;
// Specify the target code and configuration version and upgrade mode.
var upgradeDescription = new FabricUpgradeDescription()
{
TargetCodeVersion = targetCodeVersion,
TargetConfigVersion = targetConfigVersion,
UpgradePolicyDescription = policyDescription
};
Trace.WriteInfo(TraceType, "Start upgrade");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.UpgradeFabricAsync(
upgradeDescription,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.UpgradeFabricErrors.Value,
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
}
public async Task <ValidationReport> ValidateHealthWithReportAsync(TimeSpan maximumStabilizationTimeout, TimeSpan retryWait, CancellationToken ct)
{
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Validating that '{0}' is healthy with timeout '{1}'.", this.serviceName, maximumStabilizationTimeout);
TimeoutHelper timer = new TimeoutHelper(maximumStabilizationTimeout);
bool success = false;
string healthinfo = string.Empty;
int retryCount = 1;
while (!success && timer.GetRemainingTime() > TimeSpan.Zero)
{
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "ValidateHealthWithReportAsync(): retryCount='{0}', timer.GetRemainingTime()='{1}'", retryCount, timer.GetRemainingTime());
healthinfo = string.Empty;
if (this.TestContext == null)
{
Console.WriteLine("testcontext is null");
}
ReleaseAssert.AssertIfNull(this.TestContext, "test context");
ReleaseAssert.AssertIfNull(this.serviceName, "serviceName");
ReleaseAssert.AssertIfNull(FabricClientRetryErrors.GetEntityHealthFabricErrors.Value, "health error code");
ApplicationHealthPolicy healthPolicy = new ApplicationHealthPolicy();
var serviceHealthResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
this.TestContext.FabricClient.HealthManager.GetServiceHealthAsync(
this.serviceName,
healthPolicy,
this.requestTimeout,
ct),
FabricClientRetryErrors.GetEntityHealthFabricErrors.Value,
timer.GetRemainingTime(),
ct).ConfigureAwait(false);
bool checkError = (this.checkFlags & ValidationCheckFlag.CheckError) != 0;
bool checkWarning = (this.checkFlags & ValidationCheckFlag.CheckWarning) != 0;
if ((checkError && serviceHealthResult.AggregatedHealthState == HealthState.Error) ||
(checkWarning && serviceHealthResult.AggregatedHealthState == HealthState.Warning) ||
serviceHealthResult.AggregatedHealthState == HealthState.Invalid ||
serviceHealthResult.AggregatedHealthState == HealthState.Unknown)
{
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0} is health state is {1}. Will Retry check", this.serviceName, serviceHealthResult.AggregatedHealthState);
healthinfo = await this.GetUnhealthyItemsAsync(serviceHealthResult, timer, ct).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceSource, healthinfo);
}
else
{
success = true;
}
if (!success)
{
if (retryCount % 10 == 0)
{
TestabilityTrace.TraceSource.WriteWarning(TraceSource, "Service {0} health validation failed due to issues below, will retry: \n{1}", this.serviceName, healthinfo);
}
// Delay before querying again so we allow some time for state to change - don't spam the node
await AsyncWaiter.WaitAsync(retryWait);
}
retryCount++;
}
if (!success)
{
return(new ValidationReport(true, StringHelper.Format(StringResources.Error_ServiceNotHealthy, serviceName, maximumStabilizationTimeout, healthinfo)));
}
else
{
return(ValidationReport.Default);
}
}
