private static async Task ProvisionPackageAsync(string targetCodeVersion, string localPackagePath, CancellationToken cancellationToken) { var commandParameter = new ClusterUpgradeCommandParameter { ConfigFilePath = null, ConfigVersion = null, CodeFilePath = localPackagePath, CodeVersion = targetCodeVersion }; var timeoutHelper = new System.Fabric.UpgradeService.TimeoutHelper(TimeSpan.MaxValue, System.Fabric.UpgradeService.Constants.MaxOperationTimeout); var commandProcessor = new FabricClientWrapper(); await commandProcessor.CopyAndProvisionUpgradePackageAsync( commandParameter, commandParameter.CodeVersion, null, timeoutHelper, cancellationToken).ConfigureAwait(false); }
/// <summary> /// Copy and provision upgrade package. /// </summary> /// <param name="commandParameter"></param> /// <param name="targetCodeVersion"></param> /// <param name="targetConfigVersion"></param> /// <param name="timeoutHelper"></param> /// <param name="cancellationToken"></param> /// <returns></returns> internal async Task CopyAndProvisionUpgradePackageAsync( ClusterUpgradeCommandParameter commandParameter, string targetCodeVersion, string targetConfigVersion, TimeoutHelper timeoutHelper, CancellationToken cancellationToken) { var imageStorePath = string.IsNullOrWhiteSpace(targetCodeVersion) ? string.Empty : targetCodeVersion; imageStorePath += string.IsNullOrWhiteSpace(targetConfigVersion) ? string.Empty : "_" + targetConfigVersion; imageStorePath = imageStorePath.Trim('_'); Trace.WriteInfo(TraceType, "CopyAndProvision"); var configFileName = string.Empty; var codeFileName = string.Empty; var configPathInImageStore = string.Empty; var codePathInImageStore = string.Empty; if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath)) { configFileName = Path.GetFileName(commandParameter.ConfigFilePath); configPathInImageStore = Path.Combine(imageStorePath, configFileName); } if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath)) { codeFileName = Path.GetFileName(commandParameter.CodeFilePath); codePathInImageStore = Path.Combine(imageStorePath, codeFileName); } if (string.IsNullOrWhiteSpace(configFileName) && string.IsNullOrWhiteSpace(codeFileName)) { return; } var configFilePath = commandParameter.ConfigFilePath; var codeFilePath = commandParameter.CodeFilePath; if (!string.IsNullOrWhiteSpace(targetCodeVersion)) { var provisionedCodeList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.QueryManager.GetProvisionedFabricCodeVersionListAsync(targetCodeVersion), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); if (provisionedCodeList.Count != 0) { Trace.WriteInfo(TraceType, "Code Already provisioned: {0}", targetCodeVersion); codePathInImageStore = null; codeFilePath = null; } } if (!string.IsNullOrWhiteSpace(targetConfigVersion)) { var provisionedConfigList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.QueryManager.GetProvisionedFabricConfigVersionListAsync(targetConfigVersion), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); if (provisionedConfigList.Count != 0) { Trace.WriteInfo(TraceType, "Config Already provisioned: {0}", targetConfigVersion); configPathInImageStore = null; configFilePath = null; } } // Get current cluster manifest file if (string.IsNullOrWhiteSpace(this.imageStoreConnectionString)) { var clusterManifest = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.GetClusterManifestAsync( timeoutHelper.GetOperationTimeout(), cancellationToken), timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); // Get image store connection string if it not yet done. this.imageStoreConnectionString = GetImageStoreConnectionString(clusterManifest); } if (!string.IsNullOrWhiteSpace(configPathInImageStore) || !string.IsNullOrWhiteSpace(codePathInImageStore)) { // CopyClusterPackage is not an async API, so ideally we should create a sync version of ExecuteFabricActionWithRetry and don't spawn a new task thread here. // However, the copy and provision call doesn't happen all the time so there's no perf consideration here to have an extra thread. // Also, I don't see why CopyClusterPackage shouldn't expose an async version in the future. // So, don't bother to create a sync version of ExecuteFabricActionWithRetryAsync. await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => Task.Run( delegate { Trace.WriteInfo(TraceType, "Copy to ImageStorePath: {0}", imageStorePath); this.fabricClient.ClusterManager.CopyClusterPackage( this.imageStoreConnectionString, configFilePath, configPathInImageStore, codeFilePath, codePathInImageStore, timeoutHelper.GetOperationTimeout()); }), timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); Trace.WriteInfo(TraceType, "Completed Copy to ImageStorePath: {0}", imageStorePath); // Provision the code and config Trace.WriteInfo( TraceType, "Provision codePath:{0} configPath:{1}", codePathInImageStore, configPathInImageStore); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.ProvisionFabricAsync( codePathInImageStore, configPathInImageStore, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.ProvisionFabricErrors.Value, timeoutHelper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); Trace.WriteInfo( TraceType, "Completed Provision codePath:{0} configPath:{1}", codePathInImageStore, configPathInImageStore); } }
/// <summary> /// This function starts the upgrade process /// </summary> /// <param name="commandDescription"></param> /// <param name="targetCodeVersion"></param> /// <param name="targetConfigVersion"></param> /// <param name="timeoutHelper"></param> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task StartUpgradeFabricAsync( CommandProcessorClusterUpgradeDescription commandDescription, string targetCodeVersion, string targetConfigVersion, TimeoutHelper timeoutHelper, CancellationToken cancellationToken) { Trace.WriteInfo( TraceType, "StartUpgradeFabricAsync - Started"); var rollingUpgradeMonitoringPolicy = new RollingUpgradeMonitoringPolicy() { FailureAction = UpgradeFailureAction.Rollback }; var policyDescription = new MonitoredRollingFabricUpgradePolicyDescription() { UpgradeMode = RollingUpgradeMode.Monitored, MonitoringPolicy = rollingUpgradeMonitoringPolicy, }; if (commandDescription != null) { if (commandDescription.HealthCheckRetryTimeout.HasValue) { policyDescription.MonitoringPolicy.HealthCheckRetryTimeout = commandDescription.HealthCheckRetryTimeout.Value; } if (commandDescription.HealthCheckStableDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckStableDuration = commandDescription.HealthCheckStableDuration.Value; } if (commandDescription.HealthCheckWaitDuration.HasValue) { policyDescription.MonitoringPolicy.HealthCheckWaitDuration = commandDescription.HealthCheckWaitDuration.Value; } if (commandDescription.UpgradeDomainTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeDomainTimeout = commandDescription.UpgradeDomainTimeout.Value; } if (commandDescription.UpgradeTimeout.HasValue) { policyDescription.MonitoringPolicy.UpgradeTimeout = commandDescription.UpgradeTimeout.Value; } if (commandDescription.ForceRestart.HasValue) { policyDescription.ForceRestart = commandDescription.ForceRestart.Value; } if (commandDescription.UpgradeReplicaSetCheckTimeout.HasValue) { policyDescription.UpgradeReplicaSetCheckTimeout = commandDescription.UpgradeReplicaSetCheckTimeout.Value; } if (commandDescription.HealthPolicy != null) { policyDescription.HealthPolicy = new ClusterHealthPolicy { MaxPercentUnhealthyApplications = commandDescription.HealthPolicy.MaxPercentUnhealthyApplications, MaxPercentUnhealthyNodes = commandDescription.HealthPolicy.MaxPercentUnhealthyNodes }; if (commandDescription.HealthPolicy.ApplicationHealthPolicies != null) { foreach (var commandProcessorApplicationHealthPolicyKeyValue in commandDescription.HealthPolicy.ApplicationHealthPolicies) { CommandProcessorApplicationHealthPolicy commandProcessorApplicationHealthPolicy = commandProcessorApplicationHealthPolicyKeyValue.Value; if (commandProcessorApplicationHealthPolicy == null) { continue; } var applicationHealthPolicy = new ApplicationHealthPolicy(); if (commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy != null) { applicationHealthPolicy.DefaultServiceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorApplicationHealthPolicy.DefaultServiceTypeHealthPolicy.MaxPercentUnhealthyServices }; } if (commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies != null) { foreach (var commandProcessorServiceTypeHealthPolicyKeyValue in commandProcessorApplicationHealthPolicy.SerivceTypeHealthPolicies) { if (commandProcessorServiceTypeHealthPolicyKeyValue.Value == null) { continue; } ServiceTypeHealthPolicy serviceTypeHealthPolicy = new ServiceTypeHealthPolicy { MaxPercentUnhealthyServices = commandProcessorServiceTypeHealthPolicyKeyValue.Value.MaxPercentUnhealthyServices }; applicationHealthPolicy.ServiceTypeHealthPolicyMap.Add(commandProcessorServiceTypeHealthPolicyKeyValue.Key, serviceTypeHealthPolicy); } } policyDescription.ApplicationHealthPolicyMap.Add(new Uri(commandProcessorApplicationHealthPolicyKeyValue.Key), applicationHealthPolicy); } } } if (commandDescription.DeltaHealthPolicy != null) { policyDescription.EnableDeltaHealthEvaluation = true; policyDescription.UpgradeHealthPolicy = new ClusterUpgradeHealthPolicy() { MaxPercentDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentDeltaUnhealthyNodes, MaxPercentUpgradeDomainDeltaUnhealthyNodes = commandDescription.DeltaHealthPolicy.MaxPercentUpgradeDomainDeltaUnhealthyNodes }; } } ; // Specify the target code and configuration version and upgrade mode. var upgradeDescription = new FabricUpgradeDescription() { TargetCodeVersion = targetCodeVersion, TargetConfigVersion = targetConfigVersion, UpgradePolicyDescription = policyDescription }; Trace.WriteInfo(TraceType, "Start upgrade"); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(() => this.fabricClient.ClusterManager.UpgradeFabricAsync( upgradeDescription, timeoutHelper.GetOperationTimeout(), cancellationToken), FabricClientRetryErrors.UpgradeFabricErrors.Value, timeoutHelper.GetOperationTimeout(), cancellationToken).ConfigureAwait(false); }
/// <summary> /// This function upgrades the cluster. It returns the task which completes when the upgrade completes. /// This function does monitored upgrade with failure action as Rollback. /// </summary> /// <param name="commandParameter"></param> /// <param name="timeout"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task ClusterUpgradeAsync(ClusterUpgradeCommandParameter commandParameter, TimeSpan timeout, CancellationToken cancellationToken) { // No upgrade is required if (commandParameter == null) { Trace.WriteInfo(TraceType, $"No upgrade is required. {nameof(commandParameter)} is Null."); return; } if (string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) && string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath)) { return; } #region parameter check // Validate parameter if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) && string.IsNullOrWhiteSpace(commandParameter.CodeVersion)) { throw new ArgumentException("CodeFilePath is not null but its version is empty"); } if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath)) { Version version; if (!Version.TryParse(commandParameter.CodeVersion, out version)) { throw new ArgumentException("CodeVersion is not a valid version"); } } if (!string.IsNullOrWhiteSpace(commandParameter.CodeFilePath) && string.IsNullOrWhiteSpace(Path.GetFileName(commandParameter.CodeFilePath))) { throw new ArgumentException("commandParameter.CodeFilePath should end with a file name"); } if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath) && string.IsNullOrWhiteSpace(Path.GetFileName(commandParameter.ConfigFilePath))) { throw new ArgumentException("commandParameter.ConfigFilePath should end with a file name"); } #endregion var timeoutHelper = new TimeoutHelper(timeout, Constants.MaxOperationTimeout); var targetCodeVersion = commandParameter.CodeVersion; var targetConfigVersion = commandParameter.ConfigVersion; if (!string.IsNullOrWhiteSpace(commandParameter.ConfigFilePath) && string.IsNullOrWhiteSpace(targetConfigVersion)) { // If config version is not mentioned then read it from given manifest file targetConfigVersion = GetConfigVersion(commandParameter.ConfigFilePath); } Trace.WriteInfo( TraceType, "ClusterUpgradeAsync: CodePath {0}, CodeVersion {1}, ConfigPath {2}, ConfigVersion {3}", commandParameter.CodeFilePath, targetCodeVersion, commandParameter.ConfigFilePath, targetConfigVersion); await this.CopyAndProvisionUpgradePackageAsync( commandParameter, targetCodeVersion, targetConfigVersion, timeoutHelper, cancellationToken); Trace.WriteInfo( TraceType, "ClusterUpgradeAsync: Completed CopyAndProvisionUpgradePackageAsync"); await this.StartUpgradeFabricAsync(commandParameter.UpgradeDescription, targetCodeVersion, targetConfigVersion, timeoutHelper, cancellationToken); Trace.WriteInfo( TraceType, "ClusterUpgradeAsync: Completed StartUpgradeFabricAsync"); }