private static async Task ProvisionPackageAsync(string targetCodeVersion, string localPackagePath, CancellationToken cancellationToken)
{
var commandParameter = new ClusterUpgradeCommandParameter
{
ConfigFilePath = null,
ConfigVersion = null,
CodeFilePath = localPackagePath,
CodeVersion = targetCodeVersion
};
var timeoutHelper = new System.Fabric.UpgradeService.TimeoutHelper(TimeSpan.MaxValue, System.Fabric.UpgradeService.Constants.MaxOperationTimeout);
var commandProcessor = new FabricClientWrapper();
await commandProcessor.CopyAndProvisionUpgradePackageAsync(
commandParameter,
commandParameter.CodeVersion,
null,
timeoutHelper,
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Copy and provision upgrade package.
/// </summary>
/// <param name="commandParameter"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
internal async Task CopyAndProvisionUpgradePackageAsync(
ClusterUpgradeCommandParameter commandParameter,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
var imageStorePath = string.IsNullOrWhiteSpace(targetCodeVersion) ? string.Empty : targetCodeVersion;
imageStorePath += string.IsNullOrWhiteSpace(targetConfigVersion)
? string.Empty
: "_" + targetConfigVersion;
imageStorePath = imageStorePath.Trim('_');
Trace.WriteInfo(TraceType, "CopyAndProvision");
var configFileName = string.Empty;
var codeFileName = string.Empty;
var configPathInImageStore = string.Empty;
var codePathInImageStore = string.Empty;
if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath))
{
configFileName = Path.GetFileName(commandParameter.ConfigFilePath);
configPathInImageStore = Path.Combine(imageStorePath, configFileName);
}
if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath))
{
codeFileName = Path.GetFileName(commandParameter.CodeFilePath);
codePathInImageStore = Path.Combine(imageStorePath, codeFileName);
}
if (string.IsNullOrWhiteSpace(configFileName) &&
string.IsNullOrWhiteSpace(codeFileName))
{
return;
}
var configFilePath = commandParameter.ConfigFilePath;
var codeFilePath = commandParameter.CodeFilePath;
if (!string.IsNullOrWhiteSpace(targetCodeVersion))
{
var provisionedCodeList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.QueryManager.GetProvisionedFabricCodeVersionListAsync(targetCodeVersion),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
if (provisionedCodeList.Count != 0)
{
Trace.WriteInfo(TraceType, "Code Already provisioned: {0}", targetCodeVersion);
codePathInImageStore = null;
codeFilePath = null;
}
}
if (!string.IsNullOrWhiteSpace(targetConfigVersion))
{
var provisionedConfigList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(targetConfigVersion),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
if (provisionedConfigList.Count != 0)
{
Trace.WriteInfo(TraceType, "Config Already provisioned: {0}", targetConfigVersion);
configPathInImageStore = null;
configFilePath = null;
}
}
// Get current cluster manifest file
if (string.IsNullOrWhiteSpace(this.imageStoreConnectionString))
{
var clusterManifest = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.GetClusterManifestAsync(
timeoutHelper.GetOperationTimeout(),
cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
// Get image store connection string if it not yet done.
this.imageStoreConnectionString = GetImageStoreConnectionString(clusterManifest);
}
if (!string.IsNullOrWhiteSpace(configPathInImageStore) ||
!string.IsNullOrWhiteSpace(codePathInImageStore))
{
// CopyClusterPackage is not an async API, so ideally we should create a sync version of ExecuteFabricActionWithRetry and don't spawn a new task thread here.
// However, the copy and provision call doesn't happen all the time so there's no perf consideration here to have an extra thread.
// Also, I don't see why CopyClusterPackage shouldn't expose an async version in the future.
// So, don't bother to create a sync version of ExecuteFabricActionWithRetryAsync.
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
Task.Run(
delegate
{
Trace.WriteInfo(TraceType, "Copy to ImageStorePath: {0}", imageStorePath);
this.fabricClient.ClusterManager.CopyClusterPackage(
this.imageStoreConnectionString,
configFilePath,
configPathInImageStore,
codeFilePath,
codePathInImageStore,
timeoutHelper.GetOperationTimeout());
}),
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Trace.WriteInfo(TraceType, "Completed Copy to ImageStorePath: {0}", imageStorePath);
// Provision the code and config
Trace.WriteInfo(
TraceType,
"Provision codePath:{0} configPath:{1}",
codePathInImageStore,
configPathInImageStore);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.ProvisionFabricAsync(
codePathInImageStore,
configPathInImageStore,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.ProvisionFabricErrors.Value,
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Trace.WriteInfo(
TraceType,
"Completed Provision codePath:{0} configPath:{1}",
codePathInImageStore,
configPathInImageStore);
}
}
/// <summary>
/// This function starts the upgrade process
/// </summary>
/// <param name="commandDescription"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task StartUpgradeFabricAsync(
CommandProcessorClusterUpgradeDescription commandDescription,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
Trace.WriteInfo(
TraceType,
"StartUpgradeFabricAsync - Started");
var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
{
FailureAction = UpgradeFailureAction.Rollback
};
var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
MonitoringPolicy = rollingUpgradeMonitoringPolicy,
};
if (commandDescription != null)
{
if (commandDescription.HealthCheckRetryTimeout.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
}
if (commandDescription.HealthCheckStableDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
}
if (commandDescription.HealthCheckWaitDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
}
if (commandDescription.UpgradeDomainTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
}
if (commandDescription.UpgradeTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
}
if (commandDescription.ForceRestart.HasValue)
{
policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
}
if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
}
if (commandDescription.HealthPolicy != null)
{
policyDescription.HealthPolicy = new ClusterHealthPolicy
{
MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
};
if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
{
foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
{
CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
commandProcessorApplicationHealthPolicyKeyValue.Value;
if (commandProcessorApplicationHealthPolicy == null)
{
continue;
}
var applicationHealthPolicy = new ApplicationHealthPolicy();
if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
};
}
if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
{
foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
{
if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
{
continue;
}
ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
};
applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
}
}
policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
}
}
}
if (commandDescription.DeltaHealthPolicy != null)
{
policyDescription.EnableDeltaHealthEvaluation = true;
policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
}
}
;
// Specify the target code and configuration version and upgrade mode.
var upgradeDescription = new FabricUpgradeDescription()
{
TargetCodeVersion = targetCodeVersion,
TargetConfigVersion = targetConfigVersion,
UpgradePolicyDescription = policyDescription
};
Trace.WriteInfo(TraceType, "Start upgrade");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.UpgradeFabricAsync(
upgradeDescription,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.UpgradeFabricErrors.Value,
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// This function upgrades the cluster. It returns the task which completes when the upgrade completes.
/// This function does monitored upgrade with failure action as Rollback.
/// </summary>
/// <param name="commandParameter"></param>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task ClusterUpgradeAsync(ClusterUpgradeCommandParameter commandParameter, TimeSpan timeout, CancellationToken cancellationToken)
{
// No upgrade is required
if (commandParameter == null)
{
Trace.WriteInfo(TraceType, $"No upgrade is required. {nameof(commandParameter)} is Null.");
return;
}
if (string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) &&
string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath))
{
return;
}
#region parameter check
// Validate parameter
if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) &&
string.IsNullOrWhiteSpace(commandParameter.CodeVersion))
{
throw new ArgumentException("CodeFilePath is not null but its version is empty");
}
if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath))
{
Version version;
if (!Version.TryParse(commandParameter.CodeVersion, out version))
{
throw new ArgumentException("CodeVersion is not a valid version");
}
}
if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) &&
string.IsNullOrWhiteSpace(Path.GetFileName(commandParameter.CodeFilePath)))
{
throw new ArgumentException("commandParameter.CodeFilePath should end with a file name");
}
if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath) &&
string.IsNullOrWhiteSpace(Path.GetFileName(commandParameter.ConfigFilePath)))
{
throw new ArgumentException("commandParameter.ConfigFilePath should end with a file name");
}
#endregion
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
var targetCodeVersion = commandParameter.CodeVersion;
var targetConfigVersion = commandParameter.ConfigVersion;
if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath) &&
string.IsNullOrWhiteSpace(targetConfigVersion))
{
// If config version is not mentioned then read it from given manifest file
targetConfigVersion = GetConfigVersion(commandParameter.ConfigFilePath);
}
Trace.WriteInfo(
TraceType,
"ClusterUpgradeAsync: CodePath {0}, CodeVersion {1}, ConfigPath {2}, ConfigVersion {3}",
commandParameter.CodeFilePath,
targetCodeVersion,
commandParameter.ConfigFilePath,
targetConfigVersion);
await this.CopyAndProvisionUpgradePackageAsync(
commandParameter,
targetCodeVersion,
targetConfigVersion,
timeoutHelper,
cancellationToken);
Trace.WriteInfo(
TraceType,
"ClusterUpgradeAsync: Completed CopyAndProvisionUpgradePackageAsync");
await this.StartUpgradeFabricAsync(commandParameter.UpgradeDescription,
targetCodeVersion,
targetConfigVersion,
timeoutHelper,
cancellationToken);
Trace.WriteInfo(
TraceType,
"ClusterUpgradeAsync: Completed StartUpgradeFabricAsync");
}
