public Task <ClusterHealth> GetClusterHealthAsync(TimeSpan timeout, CancellationToken token) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.HealthManager.GetClusterHealthAsync(timeoutHelper.GetOperationTimeout(), token), timeoutHelper.GetOperationTimeout(), token)); }
public async Task EnableNodeAsync(PaasNodeStatusInfo nodeToEnable, TimeSpan timeout, CancellationToken cancellationToken) { Trace.WriteInfo(TraceType, "Calling ActivateNode for {0}", nodeToEnable.NodeName); var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.FabricClient.ClusterManager.ActivateNodeAsync( nodeToEnable.NodeName, timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken); }
/// <summary> /// Returns a task which will complete when underlying upgrade completes. /// </summary> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <FabricUpgradeProgress> GetCurrentRunningUpgradeTaskAsync( TimeSpan timeout, CancellationToken cancellationToken) { FabricUpgradeProgress fabricUpgradeProgress = null; var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); while (true) { fabricUpgradeProgress = await this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync( timeoutHelper.GetOperationTimeout(), cancellationToken); Trace.WriteInfo(TraceType, "Current status: {0}", fabricUpgradeProgress.UpgradeState); if (IsUpgradeCompleted(fabricUpgradeProgress.UpgradeState)) { break; } if (IsUpgradeInUnSupportedState(fabricUpgradeProgress.UpgradeState)) { throw new NotSupportedException(string.Format("Cluster Fabric upgrade is in state {0}", fabricUpgradeProgress.UpgradeState)); } var delayTime = timeoutHelper.GetRemainingTime(); if (delayTime.CompareTo(TimeSpan.FromSeconds(DelayTimeoutInSeconds)) > 0) { delayTime = TimeSpan.FromSeconds(DelayTimeoutInSeconds); } await Task.Delay(delayTime, cancellationToken); } return(fabricUpgradeProgress); }
public async Task UpdateServiceDescriptionAsync(Uri serviceName, ServiceRuntimeDescription description, TimeSpan timeout, CancellationToken cancellationToken) { if (serviceName == null || description == null) { return; } var updateDesc = new StatefulServiceUpdateDescription() { MinReplicaSetSize = description.MinReplicaSetSize, TargetReplicaSetSize = description.TargetReplicaSetSize, PlacementConstraints = description.PlacementConstraints }; Trace.WriteInfo( TraceType, "Calling UpdateServiceDescription for {0}. TargetReplicaSetSize={1}, MinReplicaSetSize={2}. PlacementConstraints={3}", serviceName, updateDesc.TargetReplicaSetSize, updateDesc.MinReplicaSetSize, updateDesc.PlacementConstraints ?? "null"); var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.FabricClient.ServiceManager.UpdateServiceAsync(serviceName, updateDesc, timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken); }
public async Task <ServiceList> GetServicesAsync(Uri applicationName, TimeSpan timeout, CancellationToken token) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.QueryManager.GetServiceListAsync(applicationName, null, timeout, token), timeoutHelper.GetOperationTimeout(), token)); }
/// <summary> /// Gets details for a specific cluster config version provisioned in the system. /// </summary> /// <param name="configVersionFilter"></param> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <ProvisionedFabricConfigVersionList> GetProvisionedFabricConfigVersionListAsync(string configVersionFilter, TimeSpan timeout, CancellationToken cancellationToken) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(configVersionFilter), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false)); }
public async Task <ApplicationHealth> GetApplicationHealthAsync(Uri applicationName, TimeSpan timeout, CancellationToken token) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.HealthManager.GetApplicationHealthAsync(applicationName, timeoutHelper.GetOperationTimeout(), token), timeoutHelper.GetOperationTimeout(), token)); }
/// <summary> /// Gets the XML contents of the current running cluster manifest. /// </summary> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <string> GetClusterManifestAsync(TimeSpan timeout, CancellationToken cancellationToken) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.GetClusterManifestAsync( timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false)); }
/// <summary> /// It queries cluster manager to get FabricUpgradeProgress /// </summary> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public Task <FabricUpgradeProgress> GetFabricUpgradeProgressAsync( TimeSpan timeout, CancellationToken cancellationToken) { var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); return(FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken)); }
/// <summary> /// Provisions the Service Fabric by using the specified timeout and cancellation token. /// </summary> /// <param name="codePathInImageStore"></param> /// <param name="configPathInImageStore"></param> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task ProvisionFabricAsync(string codePathInImageStore, string configPathInImageStore, TimeSpan timeout, CancellationToken cancellationToken) { var timeoutHelper = new TimeoutHelper(timeout, timeout); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.ProvisionFabricAsync( codePathInImageStore, configPathInImageStore, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.ProvisionFabricErrors.Value, timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); }
public async Task UpdateServiceDescriptionAsync(Uri serviceName, ReplicaSetSize replicaSetSize, TimeSpan timeout, CancellationToken cancellationToken) { var updateDesc = new StatefulServiceUpdateDescription() { MinReplicaSetSize = replicaSetSize.MinReplicaSetSize, TargetReplicaSetSize = replicaSetSize.TargetReplicaSetSize }; Trace.WriteInfo( TraceType, "Calling UpdateServiceDescription for {0}. Target={1}, Min={2}.", serviceName, updateDesc.TargetReplicaSetSize, updateDesc.MinReplicaSetSize); var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.FabricClient.ServiceManager.UpdateServiceAsync(serviceName, updateDesc, timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken); }
/// <summary> /// This function starts the upgrade process /// </summary> /// <param name="commandDescription"></param> /// <param name="targetCodeVersion"></param> /// <param name="targetConfigVersion"></param> /// <param name="timeoutHelper"></param> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task StartUpgradeFabricAsync( CommandProcessorClusterUpgradeDescription commandDescription, string targetCodeVersion, string targetConfigVersion, TimeoutHelper timeoutHelper, CancellationToken cancellationToken) { Trace.WriteInfo( TraceType, "StartUpgradeFabricAsync - Started"); var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy() { FailureAction = UpgradeFailureAction.Rollback }; var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription() { UpgradeMode = RollingUpgradeMode.Monitored, MonitoringPolicy = rollingUpgradeMonitoringPolicy, }; if (commandDescription != null) { if (commandDescription.HealthCheckRetryTimeout.HasValue) { policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value; } if (commandDescription.HealthCheckStableDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value; } if (commandDescription.HealthCheckWaitDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value; } if (commandDescription.UpgradeDomainTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value; } if (commandDescription.UpgradeTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value; } if (commandDescription.ForceRestart.HasValue) { policyDescription.ForceRestart = commandDescription.ForceRestart.Value; } if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue) { policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value; } if (commandDescription.HealthPolicy != null) { policyDescription.HealthPolicy = new ClusterHealthPolicy { MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications, MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes }; if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null) { foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies) { CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy = commandProcessorApplicationHealthPolicyKeyValue.Value; if (commandProcessorApplicationHealthPolicy == null) { continue; } var applicationHealthPolicy = new ApplicationHealthPolicy(); if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null) { applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices }; } if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null) { foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies) { if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null) { continue; } ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices }; applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy); } } policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy); } } } if (commandDescription.DeltaHealthPolicy != null) { policyDescription.EnableDeltaHealthEvaluation = true; policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy() { MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes, MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes }; } } ; // Specify the target code and configuration version and upgrade mode. var upgradeDescription = new FabricUpgradeDescription() { TargetCodeVersion = targetCodeVersion, TargetConfigVersion = targetConfigVersion, UpgradePolicyDescription = policyDescription }; Trace.WriteInfo(TraceType, "Start upgrade"); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.UpgradeFabricAsync( upgradeDescription, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.UpgradeFabricErrors.Value, timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); }
/// <summary> /// Copies the cluster manifest file and/or Service Fabric code package to the image store. /// </summary> /// <param name="imageStoreConnectionString"></param> /// <param name="clusterManifestPath"></param> /// <param name="clusterManifestPathInImageStore"></param> /// <param name="codePackagePath"></param> /// <param name="codePackagePathInImageStore"></param> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task CopyClusterPackageAsync( string imageStoreConnectionString, string clusterManifestPath, string clusterManifestPathInImageStore, string codePackagePath, string codePackagePathInImageStore, TimeSpan timeout, CancellationToken cancellationToken) { var timeoutHelper = new TimeoutHelper(timeout, timeout); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => Task.Run(() => this.fabricClient.ClusterManager.CopyClusterPackage( imageStoreConnectionString, clusterManifestPath, clusterManifestPathInImageStore, codePackagePath, codePackagePathInImageStore, timeoutHelper.GetOperationTimeout()) ), timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); }
/// <summary> /// Copy and provision upgrade package. /// </summary> /// <param name="commandParameter"></param> /// <param name="targetCodeVersion"></param> /// <param name="targetConfigVersion"></param> /// <param name="timeoutHelper"></param> /// <param name="cancellationToken"></param> /// <returns></returns> internal async Task CopyAndProvisionUpgradePackageAsync( ClusterUpgradeCommandParameter commandParameter, string targetCodeVersion, string targetConfigVersion, TimeoutHelper timeoutHelper, CancellationToken cancellationToken) { var imageStorePath = string.IsNullOrWhiteSpace(targetCodeVersion) ? string.Empty : targetCodeVersion; imageStorePath += string.IsNullOrWhiteSpace(targetConfigVersion) ? string.Empty : "_" + targetConfigVersion; imageStorePath = imageStorePath.Trim('_'); Trace.WriteInfo(TraceType, "CopyAndProvision"); var configFileName = string.Empty; var codeFileName = string.Empty; var configPathInImageStore = string.Empty; var codePathInImageStore = string.Empty; if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath)) { configFileName = Path.GetFileName(commandParameter.ConfigFilePath); configPathInImageStore = Path.Combine(imageStorePath, configFileName); } if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath)) { codeFileName = Path.GetFileName(commandParameter.CodeFilePath); codePathInImageStore = Path.Combine(imageStorePath, codeFileName); } if (string.IsNullOrWhiteSpace(configFileName) && string.IsNullOrWhiteSpace(codeFileName)) { return; } var configFilePath = commandParameter.ConfigFilePath; var codeFilePath = commandParameter.CodeFilePath; if (!string.IsNullOrWhiteSpace(targetCodeVersion)) { var provisionedCodeList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.QueryManager.GetProvisionedFabricCodeVersionListAsync(targetCodeVersion), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); if (provisionedCodeList.Count != 0) { Trace.WriteInfo(TraceType, "Code Already provisioned: {0}", targetCodeVersion); codePathInImageStore = null; codeFilePath = null; } } if (!string.IsNullOrWhiteSpace(targetConfigVersion)) { var provisionedConfigList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(targetConfigVersion), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); if (provisionedConfigList.Count != 0) { Trace.WriteInfo(TraceType, "Config Already provisioned: {0}", targetConfigVersion); configPathInImageStore = null; configFilePath = null; } } // Get current cluster manifest file if (string.IsNullOrWhiteSpace(this.imageStoreConnectionString)) { var clusterManifest = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.GetClusterManifestAsync( timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); // Get image store connection string if it not yet done. this.imageStoreConnectionString = GetImageStoreConnectionString(clusterManifest); } if (!string.IsNullOrWhiteSpace(configPathInImageStore) || !string.IsNullOrWhiteSpace(codePathInImageStore)) { // CopyClusterPackage is not an async API, so ideally we should create a sync version of ExecuteFabricActionWithRetry and don't spawn a new task thread here. // However, the copy and provision call doesn't happen all the time so there's no perf consideration here to have an extra thread. // Also, I don't see why CopyClusterPackage shouldn't expose an async version in the future. // So, don't bother to create a sync version of ExecuteFabricActionWithRetryAsync. await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => Task.Run( delegate { Trace.WriteInfo(TraceType, "Copy to ImageStorePath: {0}", imageStorePath); this.fabricClient.ClusterManager.CopyClusterPackage( this.imageStoreConnectionString, configFilePath, configPathInImageStore, codeFilePath, codePathInImageStore, timeoutHelper.GetOperationTimeout()); }), timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); Trace.WriteInfo(TraceType, "Completed Copy to ImageStorePath: {0}", imageStorePath); // Provision the code and config Trace.WriteInfo( TraceType, "Provision codePath:{0} configPath:{1}", codePathInImageStore, configPathInImageStore); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.ProvisionFabricAsync( codePathInImageStore, configPathInImageStore, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.ProvisionFabricErrors.Value, timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); Trace.WriteInfo( TraceType, "Completed Provision codePath:{0} configPath:{1}", codePathInImageStore, configPathInImageStore); } }