protected void UpgradeCluster(FabricUpgradeDescription upgradeDescription) { var clusterConnection = this.GetClusterConnection(); try { clusterConnection.UpgradeFabricAsync( upgradeDescription, this.GetTimeout(), this.GetCancellationToken()).Wait(); var currentProgress = this.GetClusterUpgradeProgress(); upgradeDescription.TargetCodeVersion = currentProgress.TargetCodeVersion; upgradeDescription.TargetConfigVersion = currentProgress.TargetConfigVersion; this.WriteObject(this.FormatOutput(upgradeDescription)); } catch (AggregateException aggregateException) { aggregateException.Handle((ae) => { this.ThrowTerminatingError( ae, Constants.UpgradeClusterErrorId, clusterConnection); return(true); }); } }
public UpgradeFabricRequest(IFabricClient fabricClient, FabricUpgradeDescription upgradeDescription, TimeSpan timeout) : base(fabricClient, timeout) { ThrowIf.Null(upgradeDescription, "upgradeDescription"); this.UpgradeDescription = upgradeDescription; this.ConfigureErrorCodes(); }
protected PSObject ToPSObject(FabricUpgradeDescription upgradeDescription) { var itemPSObj = new PSObject(upgradeDescription); if (upgradeDescription.UpgradePolicyDescription == null) { return(itemPSObj); } var monitoredPolicy = upgradeDescription.UpgradePolicyDescription as MonitoredRollingFabricUpgradePolicyDescription; if (monitoredPolicy != null) { itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeKindPropertyName, monitoredPolicy.Kind)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.ForceRestartPropertyName, monitoredPolicy.ForceRestart)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeModePropertyName, monitoredPolicy.UpgradeMode)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeReplicaSetCheckTimeoutPropertyName, monitoredPolicy.UpgradeReplicaSetCheckTimeout)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.EnableDeltaHealthEvaluationPropertyName, monitoredPolicy.EnableDeltaHealthEvaluation)); if (monitoredPolicy.MonitoringPolicy != null) { itemPSObj.Properties.Add(new PSNoteProperty(Constants.FailureActionPropertyName, monitoredPolicy.MonitoringPolicy.FailureAction)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckWaitDurationPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckWaitDuration)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckStableDurationPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckStableDuration)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.HealthCheckRetryTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.HealthCheckRetryTimeout)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeDomainTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.UpgradeDomainTimeout)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeTimeoutPropertyName, monitoredPolicy.MonitoringPolicy.UpgradeTimeout)); } if (monitoredPolicy.HealthPolicy != null) { this.AddToPSObject(itemPSObj, monitoredPolicy.HealthPolicy); } if (monitoredPolicy.UpgradeHealthPolicy != null) { this.AddToPSObject(itemPSObj, monitoredPolicy.UpgradeHealthPolicy); } this.AddToPSObject(itemPSObj, monitoredPolicy.ApplicationHealthPolicyMap); } else { var policy = upgradeDescription.UpgradePolicyDescription as RollingUpgradePolicyDescription; if (policy != null) { itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeKindPropertyName, policy.Kind)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.ForceRestartPropertyName, policy.ForceRestart)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeModePropertyName, policy.UpgradeMode)); itemPSObj.Properties.Add(new PSNoteProperty(Constants.UpgradeReplicaSetCheckTimeoutPropertyName, policy.UpgradeReplicaSetCheckTimeout)); } } return(itemPSObj); }
internal static FabricUpgradeProgress CreateFromNative(NativeClient.IFabricUpgradeProgressResult3 nativeResult) { if (nativeResult == null) { return(null); } var progress = new FabricUpgradeProgress(nativeResult); progress.TargetCodeVersion = NativeTypes.FromNativeString(nativeResult.get_TargetCodeVersion()); progress.TargetConfigVersion = NativeTypes.FromNativeString(nativeResult.get_TargetConfigVersion()); switch (nativeResult.get_UpgradeState()) { case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_IN_PROGRESS: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_COMPLETED: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_PENDING: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_PENDING: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_IN_PROGRESS: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_COMPLETED: case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_FAILED: progress.UpgradeState = (FabricUpgradeState)nativeResult.get_UpgradeState(); break; default: progress.UpgradeState = FabricUpgradeState.Invalid; break; } uint count = 0; IntPtr pointer = nativeResult.GetUpgradeDomains(out count); progress.UpgradeDomains = UpgradeDomainHelper.CreateUpgradeDomainsFromNative(count, pointer); switch (nativeResult.get_RollingUpgradeMode()) { case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_AUTO: case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_MANUAL: case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_MONITORED: progress.RollingUpgradeMode = (RollingUpgradeMode)nativeResult.get_RollingUpgradeMode(); break; default: progress.RollingUpgradeMode = RollingUpgradeMode.Invalid; break; } progress.NextUpgradeDomain = NativeTypes.FromNativeString(nativeResult.get_NextUpgradeDomain()); IntPtr upgradeProgressPtr = nativeResult.get_UpgradeProgress(); if (upgradeProgressPtr != IntPtr.Zero) { unsafe { var castedProgressPtr = (NativeTypes.FABRIC_UPGRADE_PROGRESS *)upgradeProgressPtr; progress.UpgradeDescription = FabricUpgradeDescription.FromNative(castedProgressPtr->UpgradeDescription); progress.UpgradeDuration = TimeSpan.FromSeconds(castedProgressPtr->UpgradeDurationInSeconds); progress.CurrentUpgradeDomainDuration = TimeSpan.FromSeconds(castedProgressPtr->CurrentUpgradeDomainDurationInSeconds); progress.UnhealthyEvaluations = HealthEvaluation.FromNativeList(castedProgressPtr->UnhealthyEvaluations); progress.CurrentUpgradeDomainProgress = UpgradeDomainProgress.FromNative(castedProgressPtr->CurrentUpgradeDomainProgress); if (castedProgressPtr->Reserved != IntPtr.Zero) { var castedProgressEx1Ptr = (NativeTypes.FABRIC_UPGRADE_PROGRESS_EX1 *)castedProgressPtr->Reserved; progress.StartTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->StartTimestampUtc); progress.FailureTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->FailureTimestampUtc); if (progress.FailureTimestampUtc.HasValue) { switch (castedProgressEx1Ptr->FailureReason) { case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_INTERRUPTED: case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_HEALTH_CHECK: case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_UPGRADE_DOMAIN_TIMEOUT: case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_OVERALL_UPGRADE_TIMEOUT: case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_PROCESSING_FAILURE: progress.FailureReason = (UpgradeFailureReason)castedProgressEx1Ptr->FailureReason; break; default: progress.FailureReason = UpgradeFailureReason.None; break; } progress.UpgradeDomainProgressAtFailure = UpgradeDomainProgress.FromNative( castedProgressEx1Ptr->UpgradeDomainProgressAtFailure); } } } } GC.KeepAlive(nativeResult); return(progress); }
protected override void ProcessRecord() { try { if (this.ReplicaQuorumTimeoutSec.HasValue) { this.WriteWarning(StringResources.PowerShell_ReplicaQuorumTimeoutSec_Deprecated); if (!this.UpgradeReplicaSetCheckTimeoutSec.HasValue) { this.UpgradeReplicaSetCheckTimeoutSec = this.ReplicaQuorumTimeoutSec.Value; } } if (this.RestartProcess) { this.WriteWarning(StringResources.PowerShell_RestartProcess_Deprecated); if (!this.ForceRestart) { this.ForceRestart = this.RestartProcess; } } RollingUpgradePolicyDescription upgradePolicyDescription; if (this.Monitored) { var monitoringPolicy = new RollingUpgradeMonitoringPolicy(); if (this.FailureAction != UpgradeFailureAction.Invalid) { monitoringPolicy.FailureAction = this.FailureAction; } if (this.HealthCheckRetryTimeoutSec.HasValue) { monitoringPolicy.HealthCheckRetryTimeout = TimeSpan.FromSeconds(this.HealthCheckRetryTimeoutSec.Value); } if (this.HealthCheckWaitDurationSec.HasValue) { monitoringPolicy.HealthCheckWaitDuration = TimeSpan.FromSeconds(this.HealthCheckWaitDurationSec.Value); } if (this.HealthCheckStableDurationSec.HasValue) { monitoringPolicy.HealthCheckStableDuration = TimeSpan.FromSeconds(this.HealthCheckStableDurationSec.Value); } if (this.UpgradeDomainTimeoutSec.HasValue) { monitoringPolicy.UpgradeDomainTimeout = TimeSpan.FromSeconds(this.UpgradeDomainTimeoutSec.Value); } if (this.UpgradeTimeoutSec.HasValue) { monitoringPolicy.UpgradeTimeout = TimeSpan.FromSeconds(this.UpgradeTimeoutSec.Value); } var monitoredPolicyDescription = new MonitoredRollingFabricUpgradePolicyDescription { UpgradeMode = RollingUpgradeMode.Monitored, ForceRestart = this.ForceRestart, MonitoringPolicy = monitoringPolicy }; upgradePolicyDescription = monitoredPolicyDescription; if (this.IsUpdatingHealthPolicy()) { if (!this.Force && !this.IsHealthPolicyComplete() && !this.ShouldProcess( //// description shows up for "-WhatIf" string.Format( CultureInfo.InvariantCulture, "{0} {1}", StringResources.PowerShell_HealthPolicyUpgradeCaption, StringResources.PowerShell_ClusterHealthPolicyUpdateWarning), //// warning and caption show up when prompting for confirmation StringResources.PowerShell_ClusterHealthPolicyUpdateWarning, StringResources.PowerShell_HealthPolicyUpgradeCaption)) { return; } var healthPolicy = new ClusterHealthPolicy(); if (this.ConsiderWarningAsError.HasValue) { healthPolicy.ConsiderWarningAsError = this.ConsiderWarningAsError.Value; } if (this.MaxPercentUnhealthyApplications.HasValue) { healthPolicy.MaxPercentUnhealthyApplications = this.MaxPercentUnhealthyApplications.Value; } if (this.MaxPercentUnhealthyNodes.HasValue) { healthPolicy.MaxPercentUnhealthyNodes = this.MaxPercentUnhealthyNodes.Value; } if (this.ApplicationTypeHealthPolicyMap != null) { foreach (var entry in this.ApplicationTypeHealthPolicyMap) { healthPolicy.ApplicationTypeHealthPolicyMap.Add(entry.Key, entry.Value); } } monitoredPolicyDescription.HealthPolicy = healthPolicy; } monitoredPolicyDescription.EnableDeltaHealthEvaluation = this.EnableDeltaHealthEvaluation; if (this.IsUpdatingUpgradeHealthPolicy()) { if (!this.Force && !this.IsUpgradeHealthPolicyComplete() && !this.ShouldProcess( //// description shows up for "-WhatIf" string.Format( CultureInfo.InvariantCulture, "{0} {1}", StringResources.PowerShell_HealthPolicyUpgradeCaption, StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning), //// warning and caption show up when prompting for confirmation StringResources.PowerShell_ClusterUpgradeHealthPolicyUpdateWarning, StringResources.PowerShell_HealthPolicyUpgradeCaption)) { return; } var upgradeHealthPolicy = new ClusterUpgradeHealthPolicy(); if (this.MaxPercentDeltaUnhealthyNodes.HasValue) { upgradeHealthPolicy.MaxPercentDeltaUnhealthyNodes = this.MaxPercentDeltaUnhealthyNodes.Value; } if (this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.HasValue) { upgradeHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes = this.MaxPercentUpgradeDomainDeltaUnhealthyNodes.Value; } monitoredPolicyDescription.UpgradeHealthPolicy = upgradeHealthPolicy; } if (this.ApplicationHealthPolicyMap != null) { foreach (var entry in this.ApplicationHealthPolicyMap) { monitoredPolicyDescription.ApplicationHealthPolicyMap.Add(entry.Key, entry.Value); } } } else if (this.UnmonitoredManual) { upgradePolicyDescription = new RollingUpgradePolicyDescription { UpgradeMode = RollingUpgradeMode.UnmonitoredManual, ForceRestart = this.ForceRestart, }; } else { upgradePolicyDescription = new RollingUpgradePolicyDescription { UpgradeMode = RollingUpgradeMode.UnmonitoredAuto, ForceRestart = this.ForceRestart, }; } if (this.UpgradeReplicaSetCheckTimeoutSec.HasValue) { upgradePolicyDescription.UpgradeReplicaSetCheckTimeout = TimeSpan.FromSeconds(this.UpgradeReplicaSetCheckTimeoutSec.Value); } var upgradeDescription = new FabricUpgradeDescription { UpgradePolicyDescription = upgradePolicyDescription, TargetCodeVersion = this.CodePackageVersion, TargetConfigVersion = this.ClusterManifestVersion }; this.UpgradeCluster(upgradeDescription); } catch (Exception exception) { this.ThrowTerminatingError( exception, Constants.UpgradeClusterErrorId, null); } }
/// <summary> /// This function starts the upgrade process /// </summary> /// <param name="commandDescription"></param> /// <param name="targetCodeVersion"></param> /// <param name="targetConfigVersion"></param> /// <param name="timeoutHelper"></param> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task StartUpgradeFabricAsync( CommandProcessorClusterUpgradeDescription commandDescription, string targetCodeVersion, string targetConfigVersion, TimeoutHelper timeoutHelper, CancellationToken cancellationToken) { Trace.WriteInfo( TraceType, "StartUpgradeFabricAsync - Started"); var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy() { FailureAction = UpgradeFailureAction.Rollback }; var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription() { UpgradeMode = RollingUpgradeMode.Monitored, MonitoringPolicy = rollingUpgradeMonitoringPolicy, }; if (commandDescription != null) { if (commandDescription.HealthCheckRetryTimeout.HasValue) { policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value; } if (commandDescription.HealthCheckStableDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value; } if (commandDescription.HealthCheckWaitDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value; } if (commandDescription.UpgradeDomainTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value; } if (commandDescription.UpgradeTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value; } if (commandDescription.ForceRestart.HasValue) { policyDescription.ForceRestart = commandDescription.ForceRestart.Value; } if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue) { policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value; } if (commandDescription.HealthPolicy != null) { policyDescription.HealthPolicy = new ClusterHealthPolicy { MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications, MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes }; if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null) { foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies) { CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy = commandProcessorApplicationHealthPolicyKeyValue.Value; if (commandProcessorApplicationHealthPolicy == null) { continue; } var applicationHealthPolicy = new ApplicationHealthPolicy(); if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null) { applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices }; } if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null) { foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies) { if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null) { continue; } ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices }; applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy); } } policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy); } } } if (commandDescription.DeltaHealthPolicy != null) { policyDescription.EnableDeltaHealthEvaluation = true; policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy() { MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes, MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes }; } } ; // Specify the target code and configuration version and upgrade mode. var upgradeDescription = new FabricUpgradeDescription() { TargetCodeVersion = targetCodeVersion, TargetConfigVersion = targetConfigVersion, UpgradePolicyDescription = policyDescription }; Trace.WriteInfo(TraceType, "Start upgrade"); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.UpgradeFabricAsync( upgradeDescription, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.UpgradeFabricErrors.Value, timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); }