/// <summary>
/// Get the progress of the current upgrade
/// </summary>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <FabricOrchestrationUpgradeProgress> ProcessGetClusterConfigurationUpgradeProgressAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessGetUpgradeProgressAsync.");
FabricOrchestrationUpgradeProgress upgradeProgress = null;
try
{
string configVersion = await this.GetCurrentJsonConfigVersionAsync(this.cancellationToken).ConfigureAwait(false);
FabricUpgradeProgress fabricUpgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(Constants.UpgradeServiceMaxOperationTimeout, this.cancellationToken),
Constants.UpgradeServiceMaxOperationTimeout,
this.cancellationToken).ConfigureAwait(false);
ConfigUpgradeErrorDetail errorDetails = await this.storeManager.GetConfigUpgradeErrorDetailsAsync(Constants.ConfigUpgradeErrorDetails, cancellationToken);
uint manifestVersion;
upgradeProgress = new FabricOrchestrationUpgradeProgress()
{
UpgradeState = fabricUpgradeProgress.UpgradeState,
ProgressStatus = uint.TryParse(fabricUpgradeProgress.TargetConfigVersion, out manifestVersion) ? manifestVersion : 0,
ConfigVersion = configVersion,
Details = (errorDetails != null) ? errorDetails.ToString() : null
};
}
catch (Exception e)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessGetUpgradeProgressAsync exception: {0}", e);
throw;
}
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exiting ProcessGetUpgradeProgressAsync.");
return(upgradeProgress);
}
private async Task ProcessClusterUpgradeAsync(
PaasClusterUpgradeDescription upgradeDesc,
CancellationToken token)
{
var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, token);
var clusterHealthQueryTask = this.fabricClientWrapper.GetClusterHealthAsync(Constants.MaxOperationTimeout, token);
await Task.WhenAll(
upgradeProgressTask,
clusterHealthQueryTask);
FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask);
ClusterHealth currentClusterHealth = GetResultFromTask(clusterHealthQueryTask);
ClusterUpgradeCommandParameter upgradeCommandParameter = null;
try
{
upgradeCommandParameter = await this.commandParameterGenerator.GetCommandParameterAsync(
currentUpgradeProgress,
currentClusterHealth,
upgradeDesc,
token);
await UpgradeClusterAsync(upgradeCommandParameter, token);
}
finally
{
Cleanup(upgradeCommandParameter);
}
}
protected FabricUpgradeProgress GetClusterUpgradeProgress()
{
var clusterConnection = this.GetClusterConnection();
FabricUpgradeProgress currentProgress = null;
try
{
currentProgress =
clusterConnection.GetFabricUpgradeProgressAsync(
this.GetTimeout(),
this.GetCancellationToken()).Result;
}
catch (AggregateException aggregateException)
{
aggregateException.Handle((ae) =>
{
this.ThrowTerminatingError(
ae,
Constants.GetClusterUpgradeProgressErrorId,
clusterConnection);
return(true);
});
}
return(currentProgress);
}
/// <summary>
/// Returns a task which will complete when underlying upgrade completes.
/// </summary>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <FabricUpgradeProgress> GetCurrentRunningUpgradeTaskAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
FabricUpgradeProgress fabricUpgradeProgress = null;
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
while (true)
{
fabricUpgradeProgress = await this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(
timeoutHelper.GetOperationTimeout(),
cancellationToken);
Trace.WriteInfo(TraceType, "Current status: {0}", fabricUpgradeProgress.UpgradeState);
if (IsUpgradeCompleted(fabricUpgradeProgress.UpgradeState))
{
break;
}
if (IsUpgradeInUnSupportedState(fabricUpgradeProgress.UpgradeState))
{
throw new NotSupportedException(string.Format("Cluster Fabric upgrade is in state {0}", fabricUpgradeProgress.UpgradeState));
}
var delayTime = timeoutHelper.GetRemainingTime();
if (delayTime.CompareTo(TimeSpan.FromSeconds(DelayTimeoutInSeconds)) > 0)
{
delayTime = TimeSpan.FromSeconds(DelayTimeoutInSeconds);
}
await Task.Delay(delayTime, cancellationToken);
}
return(fabricUpgradeProgress);
}
internal static async Task <string> GetCurrentCodeVersion()
{
FabricUpgradeProgress fabricUpgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
new FabricClient().ClusterManager.GetFabricUpgradeProgressAsync()).ConfigureAwait(false);
return(fabricUpgradeProgress.TargetCodeVersion);
}
internal async Task <bool> IsInterruptibleAsync(ClusterUpgradeStateBase clusterUpgradeState)
{
if (!clusterUpgradeState.CanInterruptUpgrade())
{
return(false);
}
FabricUpgradeProgress fabricUpgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(Constants.UpgradeServiceMaxOperationTimeout, this.cancellationToken),
Constants.UpgradeServiceMaxOperationTimeout,
this.cancellationToken).ConfigureAwait(false);
return(fabricUpgradeProgress.UpgradeState != FabricUpgradeState.RollingBackInProgress);
}
private async Task UpdateReplicaStoreAfterUpgradeStartAsync(
FabricUpgradeProgress upgradeProgress,
CancellationToken token)
{
try
{
await this.WaitForPartitionReadyAsync(token);
using (var tx = await this.store.CreateTransactionWithRetryAsync(this.exceptionPolicy, token))
{
KeyValueStoreItem storeItem = null;
CurrentUpgradeInformation updateInformation = null;
if (this.store.Contains(tx, CurrentUpgradeKey))
{
try
{
storeItem = this.store.Get(tx, CurrentUpgradeKey);
updateInformation = CurrentUpgradeInformation.Deserialize(storeItem.Value);
if (!updateInformation.UpgradeStarted)
{
updateInformation.UpgradeStarted = true;
}
}
catch (Exception ex)
{
Trace.WriteError(TraceType, "Update state deserialize failed with exception {0}", ex);
updateInformation = null;
}
}
if (updateInformation != null &&
!updateInformation.UpgradeStarted &&
updateInformation.TargetCodeVersion == upgradeProgress.TargetCodeVersion &&
(updateInformation.TargetConfigVersion == null ||
updateInformation.TargetConfigVersion == upgradeProgress.TargetConfigVersion))
{
updateInformation.UpgradeStarted = true;
this.store.Update(tx, CurrentUpgradeKey, updateInformation.Serialize(),
storeItem.Metadata.SequenceNumber);
await tx.CommitAsync();
}
}
}
catch (Exception ex)
{
Trace.WriteError(TraceType, "UpdateReplicaStoreAfterUpgradeStartAsync: Error : {0}", ex);
}
}
private async Task <TimeSpan> GetWaitDurationAndUpdateHealth(FabricUpgradeProgress upgradeProgress, CancellationToken token)
{
TimeSpan waitDuration = TimeSpan.MinValue;
using (var tx = await this.store.CreateTransactionWithRetryAsync(this.exceptionPolicy, token))
{
KeyValueStoreItem storeItem = null;
CurrentUpgradeInformation updateInformation = null;
if (this.store.Contains(tx, this.CurrentUpgradeKey))
{
try
{
storeItem = this.store.Get(tx, this.CurrentUpgradeKey);
updateInformation = CurrentUpgradeInformation.Deserialize(storeItem.Value);
}
catch (Exception ex)
{
Trace.WriteError(TraceType, "Update state deserialize failed with exception {0}", ex);
updateInformation = null;
}
}
if (updateInformation != null &&
(upgradeProgress.UpgradeState == FabricUpgradeState.Failed ||
upgradeProgress.UpgradeState == FabricUpgradeState.RollingBackCompleted))
{
if (updateInformation.CurrentCodeVersion == upgradeProgress.TargetCodeVersion &&
updateInformation.CurrentConfigVersion == upgradeProgress.TargetConfigVersion)
{
// Error so report Warning health for the service
var description = string.Format(
CultureInfo.InvariantCulture,
"Upgrade failed for Update Id:{0} Target code version: {1}",
updateInformation.UpdateId,
updateInformation.TargetCodeVersion);
this.ReportHealthEvent(description, HealthState.Warning);
waitDuration = this.waitTimeBeforePolling;
}
}
else
{
this.ReportHealthEvent("Health", HealthState.Ok);
}
}
return(waitDuration);
}
private async Task CheckClusterManifestVersion(string clusterManifestVersion, CancellationToken cancellationToken)
{
// If an upgrade is still in progress, don't do anything
FabricUpgradeProgress upgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <FabricUpgradeProgress>(
() =>
this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(FabricClient.DefaultTimeout, cancellationToken)).ConfigureAwait(false);
if (upgradeProgress == null || !FabricClientWrapper.IsUpgradeCompleted(upgradeProgress.UpgradeState))
{
return;
}
string clusterManifestContent = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.fabricClient.ClusterManager.GetClusterManifestAsync(FabricClient.DefaultTimeout, cancellationToken)).ConfigureAwait(false);
var clusterManifest = XMLHelper.ReadClusterManifestFromContent(clusterManifestContent);
var actualManifestVersion = clusterManifest.Version;
if (!actualManifestVersion.Equals(clusterManifestVersion, StringComparison.OrdinalIgnoreCase))
{
this.fabricClient.HealthManager.ReportHealth(new ClusterHealthReport(new HealthInformation(
Constants.UpgradeOrchestrationHealthSourceId,
Constants.ManifestVersionMismatchHealthProperty,
HealthState.Warning)
{
Description = string.Format(
StringResources.WarningHealth_ClusterManifestVersionMismatch, actualManifestVersion, clusterManifestVersion),
TimeToLive = TimeSpan.FromHours(48),
RemoveWhenExpired = true
}));
}
else
{
this.fabricClient.HealthManager.ReportHealth(new ClusterHealthReport(new HealthInformation(
Constants.UpgradeOrchestrationHealthSourceId,
Constants.ManifestVersionMismatchHealthProperty,
HealthState.Ok)
{
TimeToLive = TimeSpan.FromMilliseconds(1),
RemoveWhenExpired = true
}));
}
}
public Task <ClusterUpgradeCommandParameter> GetUpgradeCommandParamterAsync(
FabricUpgradeProgress currentUpgradeProgress,
TimeSpan timeout,
CancellationToken token)
{
Trace.ConsoleWriteLine(TraceType, "In Test");
ClusterUpgradeCommandParameter parameter = null;
var tcs = new TaskCompletionSource <ClusterUpgradeCommandParameter>();
if (Directory.Exists(@"c:\TempTest"))
{
parameter = new ClusterUpgradeCommandParameter();
if (File.Exists(@"c:\TempTest\ClusterManifest.xml"))
{
parameter.ConfigFilePath = @"c:\TempTest\ClusterManifest.xml";
}
var files = Directory.EnumerateFiles(@"c:\TempTest", "WindowsFabric.*.msi").ToArray();
if (files.Count() == 1)
{
var msi = Path.GetFileName(files[0]);
var name = Path.GetFileNameWithoutExtension(msi);
var version = name.Substring(14);
parameter.CodeFilePath = files[0];
parameter.CodeVersion = version;
}
if (currentUpgradeProgress.TargetCodeVersion == parameter.CodeVersion ||
(string.IsNullOrWhiteSpace(parameter.CodeFilePath) && string.IsNullOrWhiteSpace(parameter.ConfigFilePath)))
{
parameter = null;
}
}
if (parameter != null)
{
Trace.ConsoleWriteLine(TraceType, "codepath:{0} configPath:{1}", parameter.CodeFilePath, parameter.ConfigFilePath);
}
tcs.SetResult(parameter);
return(tcs.Task);
}
public async Task <ClusterUpgradeCommandParameter> GetCommandParameterAsync(
FabricUpgradeProgress currentUpgradeProgress,
ClusterHealth currentClusterHealth,
PaasClusterUpgradeDescription response,
CancellationToken token)
{
if (response == null ||
(response.TargetClusterManifestBase64 == null && response.TargetMsiUri == null))
{
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Null response");
return(null);
}
if (currentClusterHealth == null)
{
Trace.WriteWarning(TraceType, "currentClusterHealth is null. ContinousNullClusterHealthCount = {0}", this.continousNullClusterHealthCount);
if (++continousNullClusterHealthCount < 5)
{
Trace.WriteWarning(TraceType, "Returning null since upgrade will not be initiated");
return(null);
}
Trace.WriteWarning(TraceType, "Upgrading even though currentClusterHealth is null.");
}
else
{
this.continousNullClusterHealthCount = 0;
}
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: Processing WRP response");
ClusterManifestType targetClusterManifest = FromBase64ClusterManifest(response.TargetClusterManifestBase64);
Trace.WriteInfo(TraceType, "GetCommandParameterAsync: response.TargetMsiVersion = {0}, currentUpgradeProgress.TargetCodeVersion = {1}, targetClusterManifest.Version = {2}, currentUpgradeProgress.UpgradeState = {3}",
response.TargetMsiVersion,
currentUpgradeProgress != null ? currentUpgradeProgress.TargetCodeVersion : "NA",
targetClusterManifest != null ? targetClusterManifest.Version : "NA",
currentUpgradeProgress != null ? currentUpgradeProgress.UpgradeState.ToString() : "NA");
string currentCodeVersion = string.Empty;
string currentConfigVersion = string.Empty;
bool isUpgradeStateFailed = false;
if (currentUpgradeProgress != null)
{
currentCodeVersion = currentUpgradeProgress.TargetCodeVersion;
currentConfigVersion = currentUpgradeProgress.TargetConfigVersion;
isUpgradeStateFailed = currentUpgradeProgress.UpgradeState == FabricUpgradeState.Failed;
}
if (response.TargetMsiVersion == currentCodeVersion &&
(targetClusterManifest != null && targetClusterManifest.Version == currentConfigVersion) &&
!isUpgradeStateFailed)
{
return(null);
}
string clusterConfigFilePath = null;
string clusterMsiPath = null;
string clusterConfigVersion = null;
var info = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
var tempPath = info.FullName;
if (isUpgradeStateFailed || (targetClusterManifest != null && targetClusterManifest.Version != currentConfigVersion))
{
clusterConfigFilePath = Path.Combine(tempPath, "ClusterManifest.xml");
using (var fs = new FileStream(clusterConfigFilePath, FileMode.Create))
{
using (TextWriter writer = new StreamWriter(fs, new UTF8Encoding()))
{
// Serialize using the XmlTextWriter.
this.clusterManifestSerializer.Serialize(writer, targetClusterManifest);
}
}
clusterConfigVersion = targetClusterManifest.Version;
}
if (isUpgradeStateFailed || (response.TargetMsiUri != null && response.TargetMsiVersion != currentCodeVersion))
{
clusterMsiPath = await DownloadMsiAsync(response.TargetMsiUri, response.TargetMsiVersion, tempPath);
}
Trace.WriteInfo(TraceType, "ClusterConfigFile {0}, ClusterMsiFile {1}", clusterConfigFilePath, clusterMsiPath);
var parameter = new ClusterUpgradeCommandParameter()
{
CodeVersion = response.TargetMsiVersion,
CodeFilePath = clusterMsiPath,
ConfigFilePath = clusterConfigFilePath,
ConfigVersion = clusterConfigVersion
};
parameter.UpgradeDescription = await this.GetClusterUpgradeDescriptionAsync(response.UpgradePolicy, currentClusterHealth, token);
return(parameter);
}
private async Task <IOperationStatus> BuildStatusAsync(ClusterOperationDescription description, IOperationContext context, ClusterErrorDetails errorDetails)
{
var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, context.CancellationToken);
var nodeListQueryTask = this.fabricClientWrapper.GetNodeListAsync(Constants.MaxOperationTimeout, context.CancellationToken);
// Only query for system services when WRP needs to adjust the replica set size
Task <Dictionary <string, ServiceRuntimeDescription> > systemServiceSizeQueryTask = null;
if (description != null &&
description.SystemServiceDescriptionsToSet != null &&
description.SystemServiceDescriptionsToSet.Any())
{
systemServiceSizeQueryTask =
this.fabricClientWrapper.GetSystemServiceRuntimeDescriptionsAsync(Constants.MaxOperationTimeout, context.CancellationToken);
}
await Task.WhenAll(
nodeListQueryTask,
upgradeProgressTask,
systemServiceSizeQueryTask ?? Task.FromResult <Dictionary <string, ServiceRuntimeDescription> >(null));
FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask);
NodeList nodeList = FilterPrimaryNodeTypesStatus(GetResultFromTask(nodeListQueryTask), description?.PrimaryNodeTypes);
Dictionary <string, ServiceRuntimeDescription> systemServicesRuntimeDescriptions = GetResultFromTask(systemServiceSizeQueryTask);
List <PaasNodeStatusInfo> nodesDisabled = null;
List <PaasNodeStatusInfo> nodesEnabled = null;
if (description != null &&
nodeList != null)
{
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.ProcessNodeQuery.");
await this.nodeStatusManager.ProcessNodeQuery(nodeList, Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.ProcessNodeQuery.");
// Send back the list of nodes that are disabled
if (description.NodesToDisabled != null)
{
nodesDisabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested disabled nodes
foreach (var nodeToDisable in description.NodesToDisabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToDisable.NodeName,
StringComparison.OrdinalIgnoreCase));
if (matchingNodeStatus != null && matchingNodeStatus.NodeStatus == NodeStatus.Disabled)
{
var nodeDisabled = new PaasNodeStatusInfo(nodeToDisable)
{
NodeState = NodeState.Disabled
};
nodesDisabled.Add(nodeDisabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully disabled. {0}", nodeDisabled);
}
}
}
// Send back the list of nodes that are Enabled
if (description.NodesToEnabled != null)
{
nodesEnabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested up nodes
foreach (var nodeToEnable in description.NodesToEnabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToEnable.NodeName,
StringComparison.OrdinalIgnoreCase));
// Since a node can be enabled and can still be down, we infer enabled status instead.
if (matchingNodeStatus != null &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabling) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabled) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Enabling))
{
var nodeEnabled = new PaasNodeStatusInfo(nodeToEnable);
nodeEnabled.NodeState = NodeState.Enabled;
nodesEnabled.Add(nodeEnabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully enabled. {0}", nodeEnabled);
}
}
}
}
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.GetNodeStates.");
var nodesStatus = await this.nodeStatusManager.GetNodeStates(Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.GetNodeStates.");
var status = new ClusterOperationStatus(description)
{
DisabledNodes = nodesDisabled,
EnabledNodes = nodesEnabled,
NodesStatus = nodesStatus,
SystemServiceDescriptions = systemServicesRuntimeDescriptions,
};
if (currentUpgradeProgress != null)
{
status.Progress = JObject.FromObject(currentUpgradeProgress, this.serializer);
}
if (errorDetails != null)
{
status.ErrorDetails = JObject.FromObject(errorDetails, this.serializer);
}
return(status);
}
internal static FabricUpgradeProgress CreateFromNative(NativeClient.IFabricUpgradeProgressResult3 nativeResult)
{
if (nativeResult == null)
{
return(null);
}
var progress = new FabricUpgradeProgress(nativeResult);
progress.TargetCodeVersion = NativeTypes.FromNativeString(nativeResult.get_TargetCodeVersion());
progress.TargetConfigVersion = NativeTypes.FromNativeString(nativeResult.get_TargetConfigVersion());
switch (nativeResult.get_UpgradeState())
{
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_IN_PROGRESS:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_COMPLETED:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLBACK_PENDING:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_PENDING:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_IN_PROGRESS:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_ROLLFORWARD_COMPLETED:
case NativeTypes.FABRIC_UPGRADE_STATE.FABRIC_UPGRADE_STATE_FAILED:
progress.UpgradeState = (FabricUpgradeState)nativeResult.get_UpgradeState();
break;
default:
progress.UpgradeState = FabricUpgradeState.Invalid;
break;
}
uint count = 0;
IntPtr pointer = nativeResult.GetUpgradeDomains(out count);
progress.UpgradeDomains = UpgradeDomainHelper.CreateUpgradeDomainsFromNative(count, pointer);
switch (nativeResult.get_RollingUpgradeMode())
{
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_AUTO:
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_UNMONITORED_MANUAL:
case NativeTypes.FABRIC_ROLLING_UPGRADE_MODE.FABRIC_ROLLING_UPGRADE_MODE_MONITORED:
progress.RollingUpgradeMode = (RollingUpgradeMode)nativeResult.get_RollingUpgradeMode();
break;
default:
progress.RollingUpgradeMode = RollingUpgradeMode.Invalid;
break;
}
progress.NextUpgradeDomain = NativeTypes.FromNativeString(nativeResult.get_NextUpgradeDomain());
IntPtr upgradeProgressPtr = nativeResult.get_UpgradeProgress();
if (upgradeProgressPtr != IntPtr.Zero)
{
unsafe
{
var castedProgressPtr = (NativeTypes.FABRIC_UPGRADE_PROGRESS *)upgradeProgressPtr;
progress.UpgradeDescription = FabricUpgradeDescription.FromNative(castedProgressPtr->UpgradeDescription);
progress.UpgradeDuration = TimeSpan.FromSeconds(castedProgressPtr->UpgradeDurationInSeconds);
progress.CurrentUpgradeDomainDuration = TimeSpan.FromSeconds(castedProgressPtr->CurrentUpgradeDomainDurationInSeconds);
progress.UnhealthyEvaluations = HealthEvaluation.FromNativeList(castedProgressPtr->UnhealthyEvaluations);
progress.CurrentUpgradeDomainProgress = UpgradeDomainProgress.FromNative(castedProgressPtr->CurrentUpgradeDomainProgress);
if (castedProgressPtr->Reserved != IntPtr.Zero)
{
var castedProgressEx1Ptr = (NativeTypes.FABRIC_UPGRADE_PROGRESS_EX1 *)castedProgressPtr->Reserved;
progress.StartTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->StartTimestampUtc);
progress.FailureTimestampUtc = NativeTypes.FromNullableNativeFILETIME(castedProgressEx1Ptr->FailureTimestampUtc);
if (progress.FailureTimestampUtc.HasValue)
{
switch (castedProgressEx1Ptr->FailureReason)
{
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_INTERRUPTED:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_HEALTH_CHECK:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_UPGRADE_DOMAIN_TIMEOUT:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_OVERALL_UPGRADE_TIMEOUT:
case NativeTypes.FABRIC_UPGRADE_FAILURE_REASON.FABRIC_UPGRADE_FAILURE_REASON_PROCESSING_FAILURE:
progress.FailureReason = (UpgradeFailureReason)castedProgressEx1Ptr->FailureReason;
break;
default:
progress.FailureReason = UpgradeFailureReason.None;
break;
}
progress.UpgradeDomainProgressAtFailure = UpgradeDomainProgress.FromNative(
castedProgressEx1Ptr->UpgradeDomainProgressAtFailure);
}
}
}
}
GC.KeepAlive(nativeResult);
return(progress);
}
private async Task <ClusterUpgradeCommandParameter> UpdateReplicaStoreBeforeUpgradeStartAsync(
ClusterUpgradeCommandParameter commandParameter,
FabricUpgradeProgress upgradeProgress,
string updateId,
CancellationToken token)
{
try
{
await this.WaitForPartitionReadyAsync(token);
Trace.WriteInfo(TraceType, "UpdateReplicaStoreBeforeUpgradeStartAsync: Update replica store.");
KeyValueStoreItem storeItem = null;
using (var tx = await this.store.CreateTransactionWithRetryAsync(this.exceptionPolicy, token))
{
CurrentUpgradeInformation updateInformation = null;
if (this.store.Contains(tx, CurrentUpgradeKey))
{
storeItem = this.store.Get(tx, CurrentUpgradeKey);
try
{
updateInformation = CurrentUpgradeInformation.Deserialize(storeItem.Value);
if (updateInformation.UpgradeStarted && updateId == updateInformation.UpdateId)
{
updateInformation.Retry++;
updateInformation.UpgradeStarted = false;
}
}
catch (Exception ex)
{
Trace.WriteError(TraceType, "Deserialize failed with exception {0}", ex);
updateInformation = null;
}
}
if (updateInformation == null || updateId != updateInformation.UpdateId)
{
updateInformation = new CurrentUpgradeInformation
{
Retry = 0,
TargetCodeVersion = commandParameter.CodeVersion,
TargetConfigVersion = commandParameter.ConfigVersion,
CurrentCodeVersion = upgradeProgress.TargetCodeVersion,
CurrentConfigVersion = upgradeProgress.TargetConfigVersion,
UpdateId = updateId,
UpgradeStarted = false
};
}
Trace.WriteInfo(TraceType, "Updating store for update Id: {0}, Retry: {1} ", updateId, updateInformation.Retry);
if (storeItem != null)
{
this.store.Update(tx, CurrentUpgradeKey, updateInformation.Serialize(), storeItem.Metadata.SequenceNumber);
}
else
{
this.store.Add(tx, CurrentUpgradeKey, updateInformation.Serialize());
}
await tx.CommitAsync();
Trace.WriteInfo(TraceType, "UpdateReplicaStoreBeforeUpgradeStartAsync: replica store updated");
}
}
catch (Exception ex)
{
Trace.WriteError(TraceType, "UpdateReplicaStoreBeforeUpgradeStartAsync: Error : {0}", ex);
commandParameter = null;
}
if (commandParameter != null)
{
if (commandParameter.UpgradeDescription == null)
{
commandParameter.UpgradeDescription = new CommandProcessorClusterUpgradeDescription();
}
commandParameter.UpgradeDescription.ForceRestart = true;
}
return(commandParameter);
}
// Return true if upgrade started other wise false
private async Task <bool> OnGettingUpdateCollectionAsync(
NativeMethods.IUpdateCollection updateCollection,
FabricUpgradeProgress upgradeProgress,
CancellationToken token)
{
ClusterUpgradeCommandParameter commandParameter = null;
string updateId = string.Empty;
if (updateCollection != null)
{
NativeMethods.IUpdate updateToDownload = this.GetUpdateToDownload(updateCollection);
if (updateToDownload != null)
{
Trace.WriteInfo(TraceType, "OnGettingUpdateCollectionAsync: Update to download: {0}", updateToDownload.Identity.UpdateID);
commandParameter = await this.packageRetriever.DownloadWindowsUpdate(updateToDownload, this.windowsUpdateApiTimeout, token);
updateId = updateToDownload.Identity.UpdateID;
}
else
{
Trace.WriteInfo(TraceType, "OnGettingUpdateCollectionAsync: No update found.");
}
}
else
{
Trace.WriteInfo(TraceType, "OnGettingUpdateCollectionAsync: update collection is null.");
if (this.testMode && Directory.Exists(this.testSrcDir))
{
var srcCabFile = Directory.GetFiles(this.testSrcDir, "*.cab", SearchOption.TopDirectoryOnly).FirstOrDefault();
if (!string.IsNullOrWhiteSpace(srcCabFile))
{
Trace.WriteWarning(TraceType, "OnGettingUpdateCollectionAsync: Test cab file {0}", srcCabFile);
var dir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
var info = new DirectoryInfo(dir);
if (!info.Exists)
{
info.Create();
}
updateId = Guid.NewGuid().ToString();
var destCabFile = Path.Combine(dir, "test.cab");
File.Copy(srcCabFile, destCabFile);
Trace.WriteWarning(TraceType, "OnGettingUpdateCollectionAsync: Test dest file {0}", destCabFile);
var cabVersion = CabFileOperations.GetCabVersion(destCabFile);
Trace.WriteWarning(TraceType, "OnGettingUpdateCollectionAsync: Cab version {0}", cabVersion);
commandParameter = new ClusterUpgradeCommandParameter()
{
CodeFilePath = destCabFile,
CodeVersion = cabVersion
};
}
}
}
if (commandParameter == null)
{
return(false);
}
var updatedCommandParameter = await this.UpdateReplicaStoreBeforeUpgradeStartAsync(
commandParameter,
upgradeProgress,
updateId,
token);
var upgradeTask = await this.commandProcessor.ClusterUpgradeAsync(updatedCommandParameter, TimeSpan.MaxValue, token).ContinueWith(
(task) =>
{
if (commandParameter != null)
{
DeleteFileDirectory(commandParameter.CodeFilePath);
DeleteFileDirectory(commandParameter.ConfigFilePath);
}
return(task);
});
await upgradeTask;
return(true);
}
