protected void UpgradeCluster(FabricUpgradeDescription upgradeDescription)
{
var clusterConnection = this.GetClusterConnection();
try
{
clusterConnection.UpgradeFabricAsync(
upgradeDescription,
this.GetTimeout(),
this.GetCancellationToken()).Wait();
var currentProgress = this.GetClusterUpgradeProgress();
upgradeDescription.TargetCodeVersion = currentProgress.TargetCodeVersion;
upgradeDescription.TargetConfigVersion = currentProgress.TargetConfigVersion;
this.WriteObject(this.FormatOutput(upgradeDescription));
}
catch (AggregateException aggregateException)
{
aggregateException.Handle((ae) =>
{
this.ThrowTerminatingError(
ae,
Constants.UpgradeClusterErrorId,
clusterConnection);
return(true);
});
}
}
public UpgradeFabricRequest(IFabricClient fabricClient, FabricUpgradeDescription upgradeDescription, TimeSpan timeout)
: base(fabricClient, timeout)
{
ThrowIf.Null(upgradeDescription, "upgradeDescription");
this.UpgradeDescription = upgradeDescription;
this.ConfigureErrorCodes();
}
protected PSObject ToPSObject(FabricUpgradeDescription upgradeDescription)
{
var itemPSObj = new PSObject(upgradeDescription);
if (upgradeDescription.UpgradePolicyDescription == null)
{
return(itemPSObj);
}
var monitoredPolicy = upgradeDescription.UpgradePolicyDescription as MonitoredRollingFabricUpgradePolicyDescription;
if (monitoredPolicy != null)
{
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeKindPropertyName, monitoredPolicy.Kind));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.ForceRestartPropertyName, monitoredPolicy.ForceRestart));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeModePropertyName, monitoredPolicy.UpgradeMode));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeReplicaSetCheckTimeoutPropertyName, monitoredPolicy.UpgradeReplicaSetCheckTimeout));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.EnableDeltaHealthEvaluationPropertyName, monitoredPolicy.EnableDeltaHealthEvaluation));
if (monitoredPolicy.MonitoringPolicy != null)
{
itemPSObj.Properties.Add(new PSNoteProperty(Constants.FailureActionPropertyName, monitoredPolicy.MonitoringPolicy.FailureAction));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckWaitDurationPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckWaitDuration));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckStableDurationPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckStableDuration));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckRetryTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckRetryTimeout));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeDomainTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.UpgradeDomainTimeout));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.UpgradeTimeout));
}
if (monitoredPolicy.HealthPolicy != null)
{
this.AddToPSObject(itemPSObj, monitoredPolicy.HealthPolicy);
}
if (monitoredPolicy.UpgradeHealthPolicy != null)
{
this.AddToPSObject(itemPSObj, monitoredPolicy.UpgradeHealthPolicy);
}
this.AddToPSObject(itemPSObj, monitoredPolicy.ApplicationHealthPolicyMap);
}
else
{
var policy = upgradeDescription.UpgradePolicyDescription as RollingUpgradePolicyDescription;
if (policy != null)
{
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeKindPropertyName, policy.Kind));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.ForceRestartPropertyName, policy.ForceRestart));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeModePropertyName, policy.UpgradeMode));
itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeReplicaSetCheckTimeoutPropertyName, policy.UpgradeReplicaSetCheckTimeout));
}
}
return(itemPSObj);
}
internal static FabricUpgradeProgress CreateFromNative(NativeClient.IFabricUpgradeProgressResult3 nativeResult)
{
if (nativeResult == null)
{
return(null);
}
var progress = new FabricUpgradeProgress(nativeResult);
progress.TargetCodeVersion = NativeTypes.FromNativeString(nativeResult.get_TargetCodeVersion());
progress.TargetConfigVersion = NativeTypes.FromNativeString(nativeResult.get_TargetConfigVersion());
switch (nativeResult.get_UpgradeState())
{
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_IN_PROGRESS:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_COMPLETED:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_PENDING:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_PENDING:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_IN_PROGRESS:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_COMPLETED:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_FAILED:
progress.UpgradeState = (FabricUpgradeState)nativeResult.get_UpgradeState();
break;
default:
progress.UpgradeState = FabricUpgradeState.Invalid;
break;
}
uint count = 0;
IntPtr pointer = nativeResult.GetUpgradeDomains(out count);
progress.UpgradeDomains = UpgradeDomainHelper.CreateUpgradeDomainsFromNative(count, pointer);
switch (nativeResult.get_RollingUpgradeMode())
{
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_AUTO:
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_MANUAL:
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_MONITORED:
progress.RollingUpgradeMode = (RollingUpgradeMode)nativeResult.get_RollingUpgradeMode();
break;
default:
progress.RollingUpgradeMode = RollingUpgradeMode.Invalid;
break;
}
progress.NextUpgradeDomain = NativeTypes.FromNativeString(nativeResult.get_NextUpgradeDomain());
IntPtr upgradeProgressPtr = nativeResult.get_UpgradeProgress();
if (upgradeProgressPtr != IntPtr.Zero)
{
unsafe
{
var castedProgressPtr = (NativeTypes.FABRIC_UPGRADE_PROGRESS *)upgradeProgressPtr;
progress.UpgradeDescription = FabricUpgradeDescription.FromNative(castedProgressPtr->UpgradeDescription);
progress.UpgradeDuration = TimeSpan.FromSeconds(castedProgressPtr->UpgradeDurationInSeconds);
progress.CurrentUpgradeDomainDuration = TimeSpan.FromSeconds(castedProgressPtr->CurrentUpgradeDomainDurationInSeconds);
progress.UnhealthyEvaluations = HealthEvaluation.FromNativeList(castedProgressPtr->UnhealthyEvaluations);
progress.CurrentUpgradeDomainProgress = UpgradeDomainProgress.FromNative(castedProgressPtr->CurrentUpgradeDomainProgress);
if (castedProgressPtr->Reserved != IntPtr.Zero)
{
var castedProgressEx1Ptr = (NativeTypes.FABRIC_UPGRADE_PROGRESS_EX1 *)castedProgressPtr->Reserved;
progress.StartTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->StartTimestampUtc);
progress.FailureTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->FailureTimestampUtc);
if (progress.FailureTimestampUtc.HasValue)
{
switch (castedProgressEx1Ptr->FailureReason)
{
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_INTERRUPTED:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_HEALTH_CHECK:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_UPGRADE_DOMAIN_TIMEOUT:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_OVERALL_UPGRADE_TIMEOUT:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_PROCESSING_FAILURE:
progress.FailureReason = (UpgradeFailureReason)castedProgressEx1Ptr->FailureReason;
break;
default:
progress.FailureReason = UpgradeFailureReason.None;
break;
}
progress.UpgradeDomainProgressAtFailure = UpgradeDomainProgress.FromNative(
castedProgressEx1Ptr->UpgradeDomainProgressAtFailure);
}
}
}
}
GC.KeepAlive(nativeResult);
return(progress);
}
protected override void ProcessRecord()
{
try
{
if (this.ReplicaQuorumTimeoutSec.HasValue)
{
this.WriteWarning(StringResources.PowerShell_ReplicaQuorumTimeoutSec_Deprecated);
if (!this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
{
this.UpgradeReplicaSetCheckTimeoutSec = this.ReplicaQuorumTimeoutSec.Value;
}
}
if (this.RestartProcess)
{
this.WriteWarning(StringResources.PowerShell_RestartProcess_Deprecated);
if (!this.ForceRestart)
{
this.ForceRestart = this.RestartProcess;
}
}
RollingUpgradePolicyDescription upgradePolicyDescription;
if (this.Monitored)
{
var monitoringPolicy = new RollingUpgradeMonitoringPolicy();
if (this.FailureAction != UpgradeFailureAction.Invalid)
{
monitoringPolicy.FailureAction = this.FailureAction;
}
if (this.HealthCheckRetryTimeoutSec.HasValue)
{
monitoringPolicy.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value);
}
if (this.HealthCheckWaitDurationSec.HasValue)
{
monitoringPolicy.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value);
}
if (this.HealthCheckStableDurationSec.HasValue)
{
monitoringPolicy.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value);
}
if (this.UpgradeDomainTimeoutSec.HasValue)
{
monitoringPolicy.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value);
}
if (this.UpgradeTimeoutSec.HasValue)
{
monitoringPolicy.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value);
}
var monitoredPolicyDescription = new MonitoredRollingFabricUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.Monitored,
ForceRestart = this.ForceRestart,
MonitoringPolicy = monitoringPolicy
};
upgradePolicyDescription = monitoredPolicyDescription;
if (this.IsUpdatingHealthPolicy())
{
if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess(
//// description shows up for "-WhatIf"
string.Format(
CultureInfo.InvariantCulture,
"{0} {1}",
StringResources.PowerShell_HealthPolicyUpgradeCaption,
StringResources.PowerShell_ClusterHealthPolicyUpdateWarning),
//// warning and caption show up when prompting for confirmation
StringResources.PowerShell_ClusterHealthPolicyUpdateWarning,
StringResources.PowerShell_HealthPolicyUpgradeCaption))
{
return;
}
var healthPolicy = new ClusterHealthPolicy();
if (this.ConsiderWarningAsError.HasValue)
{
healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value;
}
if (this.MaxPercentUnhealthyApplications.HasValue)
{
healthPolicy.MaxPercentUnhealthyApplications = this.MaxPercentUnhealthyApplications.Value;
}
if (this.MaxPercentUnhealthyNodes.HasValue)
{
healthPolicy.MaxPercentUnhealthyNodes = this.MaxPercentUnhealthyNodes.Value;
}
if (this.ApplicationTypeHealthPolicyMap != null)
{
foreach (var entry in this.ApplicationTypeHealthPolicyMap)
{
healthPolicy.ApplicationTypeHealthPolicyMap.Add(entry.Key, entry.Value);
}
}
monitoredPolicyDescription.HealthPolicy = healthPolicy;
}
monitoredPolicyDescription.EnableDeltaHealthEvaluation = this.EnableDeltaHealthEvaluation;
if (this.IsUpdatingUpgradeHealthPolicy())
{
if (!this.Force && !this.IsUpgradeHealthPolicyComplete() && !this.ShouldProcess(
//// description shows up for "-WhatIf"
string.Format(
CultureInfo.InvariantCulture,
"{0} {1}",
StringResources.PowerShell_HealthPolicyUpgradeCaption,
StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning),
//// warning and caption show up when prompting for confirmation
StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning,
StringResources.PowerShell_HealthPolicyUpgradeCaption))
{
return;
}
var upgradeHealthPolicy = new ClusterUpgradeHealthPolicy();
if (this.MaxPercentDeltaUnhealthyNodes.HasValue)
{
upgradeHealthPolicy.MaxPercentDeltaUnhealthyNodes = this.MaxPercentDeltaUnhealthyNodes.Value;
}
if (this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.HasValue)
{
upgradeHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes = this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.Value;
}
monitoredPolicyDescription.UpgradeHealthPolicy = upgradeHealthPolicy;
}
if (this.ApplicationHealthPolicyMap != null)
{
foreach (var entry in this.ApplicationHealthPolicyMap)
{
monitoredPolicyDescription.ApplicationHealthPolicyMap.Add(entry.Key, entry.Value);
}
}
}
else if (this.UnmonitoredManual)
{
upgradePolicyDescription = new RollingUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.UnmonitoredManual,
ForceRestart = this.ForceRestart,
};
}
else
{
upgradePolicyDescription = new RollingUpgradePolicyDescription
{
UpgradeMode = RollingUpgradeMode.UnmonitoredAuto,
ForceRestart = this.ForceRestart,
};
}
if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue)
{
upgradePolicyDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value);
}
var upgradeDescription = new FabricUpgradeDescription
{
UpgradePolicyDescription = upgradePolicyDescription,
TargetCodeVersion = this.CodePackageVersion,
TargetConfigVersion = this.ClusterManifestVersion
};
this.UpgradeCluster(upgradeDescription);
}
catch (Exception exception)
{
this.ThrowTerminatingError(
exception,
Constants.UpgradeClusterErrorId,
null);
}
}
/// <summary>
/// This function starts the upgrade process
/// </summary>
/// <param name="commandDescription"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task StartUpgradeFabricAsync(
CommandProcessorClusterUpgradeDescription commandDescription,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
Trace.WriteInfo(
TraceType,
"StartUpgradeFabricAsync - Started");
var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
{
FailureAction = UpgradeFailureAction.Rollback
};
var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
MonitoringPolicy = rollingUpgradeMonitoringPolicy,
};
if (commandDescription != null)
{
if (commandDescription.HealthCheckRetryTimeout.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
}
if (commandDescription.HealthCheckStableDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
}
if (commandDescription.HealthCheckWaitDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
}
if (commandDescription.UpgradeDomainTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
}
if (commandDescription.UpgradeTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
}
if (commandDescription.ForceRestart.HasValue)
{
policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
}
if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
}
if (commandDescription.HealthPolicy != null)
{
policyDescription.HealthPolicy = new ClusterHealthPolicy
{
MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
};
if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
{
foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
{
CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
commandProcessorApplicationHealthPolicyKeyValue.Value;
if (commandProcessorApplicationHealthPolicy == null)
{
continue;
}
var applicationHealthPolicy = new ApplicationHealthPolicy();
if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
};
}
if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
{
foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
{
if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
{
continue;
}
ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
};
applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
}
}
policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
}
}
}
if (commandDescription.DeltaHealthPolicy != null)
{
policyDescription.EnableDeltaHealthEvaluation = true;
policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
}
}
;
// Specify the target code and configuration version and upgrade mode.
var upgradeDescription = new FabricUpgradeDescription()
{
TargetCodeVersion = targetCodeVersion,
TargetConfigVersion = targetConfigVersion,
UpgradePolicyDescription = policyDescription
};
Trace.WriteInfo(TraceType, "Start upgrade");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.UpgradeFabricAsync(
upgradeDescription,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.UpgradeFabricErrors.Value,
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
}
