public Task <ClusterHealth> GetClusterHealthAsync(TimeSpan timeout, CancellationToken token)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.fabricClient.HealthManager.GetClusterHealthAsync(timeoutHelper.GetOperationTimeout(), token),
timeoutHelper.GetOperationTimeout(),
token));
}
public async Task EnableNodeAsync(PaasNodeStatusInfo nodeToEnable, TimeSpan timeout, CancellationToken cancellationToken)
{
Trace.WriteInfo(TraceType, "Calling ActivateNode for {0}", nodeToEnable.NodeName);
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ClusterManager.ActivateNodeAsync(
nodeToEnable.NodeName,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken);
}
/// <summary>
/// Returns a task which will complete when underlying upgrade completes.
/// </summary>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <FabricUpgradeProgress> GetCurrentRunningUpgradeTaskAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
FabricUpgradeProgress fabricUpgradeProgress = null;
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
while (true)
{
fabricUpgradeProgress = await this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(
timeoutHelper.GetOperationTimeout(),
cancellationToken);
Trace.WriteInfo(TraceType, "Current status: {0}", fabricUpgradeProgress.UpgradeState);
if (IsUpgradeCompleted(fabricUpgradeProgress.UpgradeState))
{
break;
}
if (IsUpgradeInUnSupportedState(fabricUpgradeProgress.UpgradeState))
{
throw new NotSupportedException(string.Format("Cluster Fabric upgrade is in state {0}", fabricUpgradeProgress.UpgradeState));
}
var delayTime = timeoutHelper.GetRemainingTime();
if (delayTime.CompareTo(TimeSpan.FromSeconds(DelayTimeoutInSeconds)) > 0)
{
delayTime = TimeSpan.FromSeconds(DelayTimeoutInSeconds);
}
await Task.Delay(delayTime, cancellationToken);
}
return(fabricUpgradeProgress);
}
public async Task UpdateServiceDescriptionAsync(Uri serviceName, ServiceRuntimeDescription description, TimeSpan timeout, CancellationToken cancellationToken)
{
if (serviceName == null ||
description == null)
{
return;
}
var updateDesc = new StatefulServiceUpdateDescription()
{
MinReplicaSetSize = description.MinReplicaSetSize,
TargetReplicaSetSize = description.TargetReplicaSetSize,
PlacementConstraints = description.PlacementConstraints
};
Trace.WriteInfo(
TraceType,
"Calling UpdateServiceDescription for {0}. TargetReplicaSetSize={1}, MinReplicaSetSize={2}. PlacementConstraints={3}",
serviceName,
updateDesc.TargetReplicaSetSize,
updateDesc.MinReplicaSetSize,
updateDesc.PlacementConstraints ?? "null");
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.UpdateServiceAsync(serviceName, updateDesc, timeoutHelper.GetOperationTimeout(), cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken);
}
public async Task <ServiceList> GetServicesAsync(Uri applicationName, TimeSpan timeout, CancellationToken token)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.fabricClient.QueryManager.GetServiceListAsync(applicationName, null, timeout, token),
timeoutHelper.GetOperationTimeout(),
token));
}
/// <summary>
/// Gets details for a specific cluster config version provisioned in the system.
/// </summary>
/// <param name="configVersionFilter"></param>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <ProvisionedFabricConfigVersionList> GetProvisionedFabricConfigVersionListAsync(string configVersionFilter, TimeSpan timeout, CancellationToken cancellationToken)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(configVersionFilter),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false));
}
public async Task <ApplicationHealth> GetApplicationHealthAsync(Uri applicationName, TimeSpan timeout, CancellationToken token)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.fabricClient.HealthManager.GetApplicationHealthAsync(applicationName, timeoutHelper.GetOperationTimeout(), token),
timeoutHelper.GetOperationTimeout(),
token));
}
/// <summary>
/// Gets the XML contents of the current running cluster manifest.
/// </summary>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <string> GetClusterManifestAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.GetClusterManifestAsync(
timeoutHelper.GetOperationTimeout(),
cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// It queries cluster manager to get FabricUpgradeProgress
/// </summary>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task <FabricUpgradeProgress> GetFabricUpgradeProgressAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
return(FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(timeoutHelper.GetOperationTimeout(), cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken));
}
/// <summary>
/// Provisions the Service Fabric by using the specified timeout and cancellation token.
/// </summary>
/// <param name="codePathInImageStore"></param>
/// <param name="configPathInImageStore"></param>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task ProvisionFabricAsync(string codePathInImageStore, string configPathInImageStore, TimeSpan timeout, CancellationToken cancellationToken)
{
var timeoutHelper = new TimeoutHelper(timeout, timeout);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.ProvisionFabricAsync(
codePathInImageStore,
configPathInImageStore,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.ProvisionFabricErrors.Value,
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
public async Task UpdateServiceDescriptionAsync(Uri serviceName, ReplicaSetSize replicaSetSize, TimeSpan timeout, CancellationToken cancellationToken)
{
var updateDesc = new StatefulServiceUpdateDescription()
{
MinReplicaSetSize = replicaSetSize.MinReplicaSetSize,
TargetReplicaSetSize = replicaSetSize.TargetReplicaSetSize
};
Trace.WriteInfo(
TraceType,
"Calling UpdateServiceDescription for {0}. Target={1}, Min={2}.",
serviceName,
updateDesc.TargetReplicaSetSize,
updateDesc.MinReplicaSetSize);
var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.UpdateServiceAsync(serviceName, updateDesc, timeoutHelper.GetOperationTimeout(), cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken);
}
/// <summary>
/// This function starts the upgrade process
/// </summary>
/// <param name="commandDescription"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task StartUpgradeFabricAsync(
CommandProcessorClusterUpgradeDescription commandDescription,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
Trace.WriteInfo(
TraceType,
"StartUpgradeFabricAsync - Started");
var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy()
{
FailureAction = UpgradeFailureAction.Rollback
};
var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription()
{
UpgradeMode = RollingUpgradeMode.Monitored,
MonitoringPolicy = rollingUpgradeMonitoringPolicy,
};
if (commandDescription != null)
{
if (commandDescription.HealthCheckRetryTimeout.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value;
}
if (commandDescription.HealthCheckStableDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value;
}
if (commandDescription.HealthCheckWaitDuration.HasValue)
{
policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value;
}
if (commandDescription.UpgradeDomainTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value;
}
if (commandDescription.UpgradeTimeout.HasValue)
{
policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value;
}
if (commandDescription.ForceRestart.HasValue)
{
policyDescription.ForceRestart = commandDescription.ForceRestart.Value;
}
if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue)
{
policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value;
}
if (commandDescription.HealthPolicy != null)
{
policyDescription.HealthPolicy = new ClusterHealthPolicy
{
MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications,
MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes
};
if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null)
{
foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies)
{
CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy =
commandProcessorApplicationHealthPolicyKeyValue.Value;
if (commandProcessorApplicationHealthPolicy == null)
{
continue;
}
var applicationHealthPolicy = new ApplicationHealthPolicy();
if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null)
{
applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices
};
}
if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null)
{
foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies)
{
if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null)
{
continue;
}
ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy
{
MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices
};
applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy);
}
}
policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy);
}
}
}
if (commandDescription.DeltaHealthPolicy != null)
{
policyDescription.EnableDeltaHealthEvaluation = true;
policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy()
{
MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes,
MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes
};
}
}
;
// Specify the target code and configuration version and upgrade mode.
var upgradeDescription = new FabricUpgradeDescription()
{
TargetCodeVersion = targetCodeVersion,
TargetConfigVersion = targetConfigVersion,
UpgradePolicyDescription = policyDescription
};
Trace.WriteInfo(TraceType, "Start upgrade");
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.UpgradeFabricAsync(
upgradeDescription,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.UpgradeFabricErrors.Value,
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Copies the cluster manifest file and/or Service Fabric code package to the image store.
/// </summary>
/// <param name="imageStoreConnectionString"></param>
/// <param name="clusterManifestPath"></param>
/// <param name="clusterManifestPathInImageStore"></param>
/// <param name="codePackagePath"></param>
/// <param name="codePackagePathInImageStore"></param>
/// <param name="timeout"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task CopyClusterPackageAsync(
string imageStoreConnectionString,
string clusterManifestPath,
string clusterManifestPathInImageStore,
string codePackagePath,
string codePackagePathInImageStore,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var timeoutHelper = new TimeoutHelper(timeout, timeout);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
Task.Run(() => this.fabricClient.ClusterManager.CopyClusterPackage(
imageStoreConnectionString,
clusterManifestPath,
clusterManifestPathInImageStore,
codePackagePath,
codePackagePathInImageStore,
timeoutHelper.GetOperationTimeout())
),
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Copy and provision upgrade package.
/// </summary>
/// <param name="commandParameter"></param>
/// <param name="targetCodeVersion"></param>
/// <param name="targetConfigVersion"></param>
/// <param name="timeoutHelper"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
internal async Task CopyAndProvisionUpgradePackageAsync(
ClusterUpgradeCommandParameter commandParameter,
string targetCodeVersion,
string targetConfigVersion,
TimeoutHelper timeoutHelper,
CancellationToken cancellationToken)
{
var imageStorePath = string.IsNullOrWhiteSpace(targetCodeVersion) ? string.Empty : targetCodeVersion;
imageStorePath += string.IsNullOrWhiteSpace(targetConfigVersion)
? string.Empty
: "_" + targetConfigVersion;
imageStorePath = imageStorePath.Trim('_');
Trace.WriteInfo(TraceType, "CopyAndProvision");
var configFileName = string.Empty;
var codeFileName = string.Empty;
var configPathInImageStore = string.Empty;
var codePathInImageStore = string.Empty;
if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath))
{
configFileName = Path.GetFileName(commandParameter.ConfigFilePath);
configPathInImageStore = Path.Combine(imageStorePath, configFileName);
}
if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath))
{
codeFileName = Path.GetFileName(commandParameter.CodeFilePath);
codePathInImageStore = Path.Combine(imageStorePath, codeFileName);
}
if (string.IsNullOrWhiteSpace(configFileName) &&
string.IsNullOrWhiteSpace(codeFileName))
{
return;
}
var configFilePath = commandParameter.ConfigFilePath;
var codeFilePath = commandParameter.CodeFilePath;
if (!string.IsNullOrWhiteSpace(targetCodeVersion))
{
var provisionedCodeList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.QueryManager.GetProvisionedFabricCodeVersionListAsync(targetCodeVersion),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
if (provisionedCodeList.Count != 0)
{
Trace.WriteInfo(TraceType, "Code Already provisioned: {0}", targetCodeVersion);
codePathInImageStore = null;
codeFilePath = null;
}
}
if (!string.IsNullOrWhiteSpace(targetConfigVersion))
{
var provisionedConfigList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(targetConfigVersion),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
if (provisionedConfigList.Count != 0)
{
Trace.WriteInfo(TraceType, "Config Already provisioned: {0}", targetConfigVersion);
configPathInImageStore = null;
configFilePath = null;
}
}
// Get current cluster manifest file
if (string.IsNullOrWhiteSpace(this.imageStoreConnectionString))
{
var clusterManifest = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.GetClusterManifestAsync(
timeoutHelper.GetOperationTimeout(),
cancellationToken),
timeoutHelper.GetOperationTimeout(),
cancellationToken).ConfigureAwait(false);
// Get image store connection string if it not yet done.
this.imageStoreConnectionString = GetImageStoreConnectionString(clusterManifest);
}
if (!string.IsNullOrWhiteSpace(configPathInImageStore) ||
!string.IsNullOrWhiteSpace(codePathInImageStore))
{
// CopyClusterPackage is not an async API, so ideally we should create a sync version of ExecuteFabricActionWithRetry and don't spawn a new task thread here.
// However, the copy and provision call doesn't happen all the time so there's no perf consideration here to have an extra thread.
// Also, I don't see why CopyClusterPackage shouldn't expose an async version in the future.
// So, don't bother to create a sync version of ExecuteFabricActionWithRetryAsync.
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
Task.Run(
delegate
{
Trace.WriteInfo(TraceType, "Copy to ImageStorePath: {0}", imageStorePath);
this.fabricClient.ClusterManager.CopyClusterPackage(
this.imageStoreConnectionString,
configFilePath,
configPathInImageStore,
codeFilePath,
codePathInImageStore,
timeoutHelper.GetOperationTimeout());
}),
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Trace.WriteInfo(TraceType, "Completed Copy to ImageStorePath: {0}", imageStorePath);
// Provision the code and config
Trace.WriteInfo(
TraceType,
"Provision codePath:{0} configPath:{1}",
codePathInImageStore,
configPathInImageStore);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() =>
this.fabricClient.ClusterManager.ProvisionFabricAsync(
codePathInImageStore,
configPathInImageStore,
timeoutHelper.GetOperationTimeout(),
cancellationToken),
FabricClientRetryErrors.ProvisionFabricErrors.Value,
timeoutHelper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
Trace.WriteInfo(
TraceType,
"Completed Provision codePath:{0} configPath:{1}",
codePathInImageStore,
configPathInImageStore);
}
}
